plugin/forward: add it (#1447)

* plugin/forward: add it

This moves coredns/forward into CoreDNS. Fixes as a few bugs, adds a
policy option and more tests to the plugin.

Update the documentation, test IPv6 address and add persistent tests.

* Always use random policy when spraying

* include scrub fix here as well

* use correct var name

* Code review

* go vet

* Move logging to metrcs

* Small readme updates

* Fix readme

plugin/forward/metrics.go

@@ -0,0 +1,52 @@
+package forward
+
+import (
+	"sync"
+
+	"github.com/coredns/coredns/plugin"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Variables declared for monitoring.
+var (
+	RequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "request_count_total",
+		Help:      "Counter of requests made per upstream.",
+	}, []string{"to"})
+	RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "response_rcode_count_total",
+		Help:      "Counter of requests made per upstream.",
+	}, []string{"rcode", "to"})
+	RequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "request_duration_seconds",
+		Buckets:   plugin.TimeBuckets,
+		Help:      "Histogram of the time each request took.",
+	}, []string{"to"})
+	HealthcheckFailureCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "healthcheck_failure_count_total",
+		Help:      "Counter of the number of failed healtchecks.",
+	}, []string{"to"})
+	HealthcheckBrokenCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "healthcheck_broken_count_total",
+		Help:      "Counter of the number of complete failures of the healtchecks.",
+	})
+	SocketGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: plugin.Namespace,
+		Subsystem: "forward",
+		Name:      "socket_count_total",
+		Help:      "Guage of open sockets per upstream.",
+	}, []string{"to"})
+)
+
+var once sync.Once