Metrics registered on wrong prometheus registry (#2246)

* - UT on metrics verifying that all plugins of all blocs have their metrics collectors declared

* - fix error msg

* - redirect Registry of metric to the one that handle the listener
- allow duplicate of metrics collector on the same Registry (case of same plugin in 2 blocs listening metrics on the same address)

* - fix change of signature

* - ensure cleaning metrics before starting the test (metrics collectors are global vars .. and re-used by several tests)

* - I think I fixed this test. Ensure correct mn of hits and clean metrics before test.

* - fix typo in error msg - proposed at review

* - fix typo in comment

* - remove ResetMetrics functions
- change a way to test the numeric metrics : get the diff between begin and end of test

* - oops. removing debug logs

plugin/metrics/metrics.go

@@ -57,7 +57,15 @@ func New(addr string) *Metrics {
 }
 
 // MustRegister wraps m.Reg.MustRegister.
-func (m *Metrics) MustRegister(c prometheus.Collector) { m.Reg.MustRegister(c) }
+func (m *Metrics) MustRegister(c prometheus.Collector) {
+	err := m.Reg.Register(c)
+	if err != nil {
+		// ignore any duplicate error, but fatal on any other kind of error
+		if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
+			log.Fatalf("Cannot register metrics collector: %s", err)
+		}
+	}
+}
 
 // AddZone adds zone z to m.
 func (m *Metrics) AddZone(z string) {

plugin/metrics/setup.go

@@ -31,6 +31,13 @@ func setup(c *caddy.Controller) error {
 		return plugin.Error("prometheus", err)
 	}
 
+	// register the metrics to its address (ensure only one active metrics per address)
+	obj := uniqAddr.Set(m.Addr, m.OnStartup, m)
+	//propagate the real active Registry to current metrics
+	if om, ok := obj.(*Metrics); ok {
+		m.Reg = om.Reg
+	}
+
 	dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
 		m.Next = next
 		return m
@@ -55,10 +62,6 @@ func setup(c *caddy.Controller) error {
 func prometheusParse(c *caddy.Controller) (*Metrics, error) {
 	var met = New(defaultAddr)
 
-	defer func() {
-		uniqAddr.Set(met.Addr, met.OnStartup)
-	}()
-
 	i := 0
 	for c.Next() {
 		if i > 0 {

plugin/metrics/test/scrape.go

@@ -27,6 +27,7 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"strconv"
 	"testing"
 
 	"github.com/matttproud/golang_protobuf_extensions/pbutil"
@@ -78,6 +79,47 @@ func Scrape(t *testing.T, url string) []*MetricFamily {
 	return result
 }
 
+// ScrapeMetricAsInt provide a sum of all metrics collected for the name and label provided.
+// if the metric is not a numeric value, it will be counted a 0.
+func ScrapeMetricAsInt(t *testing.T, addr string, name string, label string, nometricvalue int) int {
+
+	valueToInt := func(m metric) int {
+		v := m.Value
+		r, err := strconv.Atoi(v)
+		if err != nil {
+			return 0
+		}
+		return r
+	}
+
+	met := Scrape(t, fmt.Sprintf("http://%s/metrics", addr))
+	found := false
+	tot := 0
+	for _, mf := range met {
+		if mf.Name == name {
+			// Sum all metrics available
+			for _, m := range mf.Metrics {
+				if label == "" {
+					tot += valueToInt(m.(metric))
+					found = true
+					continue
+				}
+				for _, v := range m.(metric).Labels {
+					if v == label {
+						tot += valueToInt(m.(metric))
+						found = true
+					}
+				}
+			}
+		}
+	}
+
+	if !found {
+		return nometricvalue
+	}
+	return tot
+}
+
 // MetricValue returns the value associated with name as a string as well as the labels.
 // It only returns the first metrics of the slice.
 func MetricValue(name string, mfs []*MetricFamily) (string, map[string]string) {