plugin/forward using pkg/up (#1493)

* plugin/forward: on demand healtchecking

Only start doing health checks when we encouner an error (any error).
This uses the new pluing/pkg/up package to abstract away the actual
checking. This reduces the LOC quite a bit; does need more testing, unit
testing and tcpdumping a bit.

* fix tests

* Fix readme

* Use pkg/up for healthchecks

* remove unused channel

* more cleanups

* update readme

* * Again do go generate and go build; still referencing the wrong forward
  repo? Anyway fixed.
* Use pkg/up for doing the healtchecks to cut back on unwanted queries
  * Change up.Func to return an error instead of a boolean.
  * Drop the string target argument as it doesn't make sense.
* Add healthcheck test on failing to get an upstream answer.

TODO(miek): double check Forward and Lookup and how they interact with
HC, and if we correctly call close() on those

* actual test

* Tests here

* more tests

* try getting rid of host

* Get rid of the host indirection

* Finish removing hosts

* moar testing

* import fmt

* field is not used

* docs

* move some stuff

* bring back health_check

* maxfails=0 test

* git and merging, bah

* review

plugin/forward/lookup.go

@@ -5,10 +5,6 @@
 package forward
 
 import (
-	"crypto/tls"
-	"log"
-	"time"
-
 	"github.com/coredns/coredns/request"
 
 	"github.com/miekg/dns"
@@ -32,12 +28,10 @@ func (f *Forward) Forward(state request.Request) (*dns.Msg, error) {
 			// All upstream proxies are dead, assume healtcheck is complete broken and randomly
 			// select an upstream to connect to.
 			proxy = f.list()[0]
-			log.Printf("[WARNING] All upstreams down, picking random one to connect to %s", proxy.host.addr)
 		}
 
 		ret, err := proxy.connect(context.Background(), state, f.forceTCP, true)
 		if err != nil {
-			log.Printf("[WARNING] Failed to connect to %s: %s", proxy.host.addr, err)
 			if fails < len(f.proxies) {
 				continue
 			}
@@ -68,10 +62,11 @@ func (f *Forward) Lookup(state request.Request, name string, typ uint16) (*dns.M
 }
 
 // NewLookup returns a Forward that can be used for plugin that need an upstream to resolve external names.
+// Note that the caller must run Close on the forward to stop the health checking goroutines.
 func NewLookup(addr []string) *Forward {
-	f := &Forward{maxfails: 2, tlsConfig: new(tls.Config), expire: defaultExpire, hcInterval: 2 * time.Second}
+	f := New()
 	for i := range addr {
-		p := NewProxy(addr[i])
+		p := NewProxy(addr[i], nil)
 		f.SetProxy(p)
 	}
 	return f