middleware/metrics: cleanup (#355)

* middleware/metrics: add more metrics middleware/cache: Add metrics for number of elements in the cache. Also export the total size. Update README to detail the new metrics. middleware/metrics Move metrics into subpackage called "vars". This breaks the import cycle and is cleaner. This allows vars.Report to be used in the the dnsserver to log refused queries. middleware/metrics: tests Add tests to the metrics framework. The metrics/test subpackage allows scraping of the local server. Do a few test scrape of the metrics that are defined in the metrics middleware. This also allows metrics integration tests to check if the caching and dnssec middleware export their metrics correctly. * update README * typos * fix tests
2026-01-22 08:31:18 -05:00 · 2016-10-26 10:01:52 +01:00
parent 6d9d60081d
commit 219bfd0493
39 changed files with 828 additions and 259 deletions
--- a/middleware/metrics/README.md
+++ b/middleware/metrics/README.md
@@ -7,12 +7,10 @@ The following metrics are exported:

 * coredns_dns_request_count_total{zone, proto, family}
 * coredns_dns_request_duration_milliseconds{zone}
-* coredns_dns_request_size_bytes{zone,, proto}
-* coredns_dns_request_transfer_size_bytes{zone,, proto}
+* coredns_dns_request_size_bytes{zone, proto}
 * coredns_dns_request_do_count_total{zone}
 * coredns_dns_request_type_count_total{zone, type}
 * coredns_dns_response_size_bytes{zone, proto}
-* coredns_dns_response_transfer_size_bytes{zone, proto}
 * coredns_dns_response_rcode_count_total{zone, rcode}

 Each counter has a label `zone` which is the zonename used for the request/response.
@@ -27,10 +25,7 @@ Extra labels used are:
 * The `response_rcode_count_total` has an extra label `rcode` which holds the rcode of the response.

 If monitoring is enabled, queries that do not enter the middleware chain are exported under the fake
-domain "dropped" (without a closing dot).
-
-Restarting CoreDNS will stop the monitoring. This is a bug. Also [this upstream
-Caddy bug](https://github.com/mholt/caddy/issues/675).
+name "dropped" (without a closing dot - this is never a valid domain name).

 ## Syntax

@@ -44,3 +39,9 @@ It optionally takes an address to which the metrics are exported; the default
 is `localhost:9153`. The metrics path is fixed to `/metrics`.

 ## Examples
+
+Use an alternative address:
+
+~~~
+prometheus localhost:9253
+~~~
--- a/middleware/metrics/handler.go
+++ b/middleware/metrics/handler.go
@@ -1,9 +1,8 @@
 package metrics

 import (
-	"time"
-
 	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/metrics/vars"
 	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/pkg/rcode"
 	"github.com/miekg/coredns/request"
@@ -17,7 +16,7 @@ func (m *Metrics) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg
 	state := request.Request{W: w, Req: r}

 	qname := state.QName()
-	zone := middleware.Zones(m.ZoneNames).Matches(qname)
+	zone := middleware.Zones(m.ZoneNames()).Matches(qname)
 	if zone == "" {
 		zone = "."
 	}
@@ -26,71 +25,9 @@ func (m *Metrics) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg
 	rw := dnsrecorder.New(w)
 	status, err := m.Next.ServeDNS(ctx, rw, r)

-	Report(state, zone, rcode.ToString(rw.Rcode), rw.Size, rw.Start)
+	vars.Report(state, zone, rcode.ToString(rw.Rcode), rw.Size, rw.Start)

 	return status, err
 }

-// Report is a plain reporting function that the server can use for REFUSED and other
-// queries that are turned down because they don't match any middleware.
-func Report(req request.Request, zone, rcode string, size int, start time.Time) {
-	if requestCount == nil {
-		// no metrics are enabled
-		return
-	}
-
-	// Proto and Family
-	net := req.Proto()
-	fam := "1"
-	if req.Family() == 2 {
-		fam = "2"
-	}
-
-	typ := req.QType()
-
-	requestCount.WithLabelValues(zone, net, fam).Inc()
-	requestDuration.WithLabelValues(zone).Observe(float64(time.Since(start) / time.Millisecond))
-
-	if req.Do() {
-		requestDo.WithLabelValues(zone).Inc()
-	}
-
-	if _, known := monitorType[typ]; known {
-		requestType.WithLabelValues(zone, dns.Type(typ).String()).Inc()
-	} else {
-		requestType.WithLabelValues(zone, other).Inc()
-	}
-
-	if typ == dns.TypeIXFR || typ == dns.TypeAXFR {
-		responseTransferSize.WithLabelValues(zone, net).Observe(float64(size))
-		requestTransferSize.WithLabelValues(zone, net).Observe(float64(req.Size()))
-	} else {
-		responseSize.WithLabelValues(zone, net).Observe(float64(size))
-		requestSize.WithLabelValues(zone, net).Observe(float64(req.Size()))
-	}
-
-	responseRcode.WithLabelValues(zone, rcode).Inc()
-}
-
-var monitorType = map[uint16]bool{
-	dns.TypeAAAA:   true,
-	dns.TypeA:      true,
-	dns.TypeCNAME:  true,
-	dns.TypeDNSKEY: true,
-	dns.TypeDS:     true,
-	dns.TypeMX:     true,
-	dns.TypeNSEC3:  true,
-	dns.TypeNSEC:   true,
-	dns.TypeNS:     true,
-	dns.TypePTR:    true,
-	dns.TypeRRSIG:  true,
-	dns.TypeSOA:    true,
-	dns.TypeSRV:    true,
-	dns.TypeTXT:    true,
-	// Meta Qtypes
-	dns.TypeIXFR: true,
-	dns.TypeAXFR: true,
-	dns.TypeANY:  true,
-}
-
-const other = "other"
+func (m *Metrics) Name() string { return "prometheus" }
--- a/middleware/metrics/metrics.go
+++ b/middleware/metrics/metrics.go
@@ -1,5 +1,4 @@
-// Package metrics implement a handler and middleware that provides Prometheus
-// metrics.
+// Package metrics implement a handler and middleware that provides Prometheus metrics.
 package metrics

 import (
@@ -9,37 +8,51 @@ import (
 	"sync"

 	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/metrics/vars"

 	"github.com/prometheus/client_golang/prometheus"
 )

-var (
-	requestCount        *prometheus.CounterVec
-	requestDuration     *prometheus.HistogramVec
-	requestSize         *prometheus.HistogramVec
-	requestTransferSize *prometheus.HistogramVec
-	requestDo           *prometheus.CounterVec
-	requestType         *prometheus.CounterVec
-
-	responseSize         *prometheus.HistogramVec
-	responseTransferSize *prometheus.HistogramVec
-	responseRcode        *prometheus.CounterVec
-)
-
 // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics
 type Metrics struct {
-	Next      middleware.Handler
-	Addr      string
-	ln        net.Listener
-	mux       *http.ServeMux
-	Once      sync.Once
-	ZoneNames []string
+	Next middleware.Handler
+	Addr string
+	ln   net.Listener
+	mux  *http.ServeMux
+	Once sync.Once
+
+	zoneNames []string
+	zoneMap   map[string]bool
+	zoneMu    sync.RWMutex
+}
+
+// AddZone adds zone z to m.
+func (m *Metrics) AddZone(z string) {
+	m.zoneMu.Lock()
+	m.zoneMap[z] = true
+	m.zoneNames = keys(m.zoneMap)
+	m.zoneMu.Unlock()
+}
+
+// RemoveZone remove zone z from m.
+func (m *Metrics) RemoveZone(z string) {
+	m.zoneMu.Lock()
+	delete(m.zoneMap, z)
+	m.zoneNames = keys(m.zoneMap)
+	m.zoneMu.Unlock()
+}
+
+// ZoneNames returns the zones of m.
+func (m *Metrics) ZoneNames() []string {
+	m.zoneMu.RLock()
+	s := m.zoneNames
+	m.zoneMu.RUnlock()
+	return s
 }

 // OnStartup sets up the metrics on startup.
 func (m *Metrics) OnStartup() error {
 	m.Once.Do(func() {
-		define()

 		ln, err := net.Listen("tcp", m.Addr)
 		if err != nil {
@@ -51,18 +64,16 @@ func (m *Metrics) OnStartup() error {

 		m.mux = http.NewServeMux()

-		prometheus.MustRegister(requestCount)
-		prometheus.MustRegister(requestDuration)
-		prometheus.MustRegister(requestSize)
-		prometheus.MustRegister(requestTransferSize)
-		prometheus.MustRegister(requestDo)
-		prometheus.MustRegister(requestType)
+		prometheus.MustRegister(vars.RequestCount)
+		prometheus.MustRegister(vars.RequestDuration)
+		prometheus.MustRegister(vars.RequestSize)
+		prometheus.MustRegister(vars.RequestDo)
+		prometheus.MustRegister(vars.RequestType)

-		prometheus.MustRegister(responseSize)
-		prometheus.MustRegister(responseTransferSize)
-		prometheus.MustRegister(responseRcode)
+		prometheus.MustRegister(vars.ResponseSize)
+		prometheus.MustRegister(vars.ResponseRcode)

-		m.mux.Handle(path, prometheus.Handler())
+		m.mux.Handle("/metrics", prometheus.Handler())

 		go func() {
 			http.Serve(m.ln, m.mux)
@@ -79,79 +90,10 @@ func (m *Metrics) OnShutdown() error {
 	return nil
 }

-func define() {
-	requestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_count_total",
-		Help:      "Counter of DNS requests made per zone, protocol and family.",
-	}, []string{"zone", "proto", "family"})
-
-	requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_duration_milliseconds",
-		Buckets:   append(prometheus.DefBuckets, []float64{50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000}...),
-		Help:      "Histogram of the time (in milliseconds) each request took.",
-	}, []string{"zone"})
-
-	requestSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_size_bytes",
-		Help:      "Size of the EDNS0 UDP buffer in bytes (64K for TCP).",
-		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
-	}, []string{"zone", "proto"})
-
-	requestTransferSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_transfer_size_bytes",
-		Help:      "Size of the incoming zone transfer in bytes.",
-		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
-	}, []string{"zone", "proto"})
-
-	requestDo = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_do_count_total",
-		Help:      "Counter of DNS requests with DO bit set per zone.",
-	}, []string{"zone"})
-
-	requestType = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "request_type_count_total",
-		Help:      "Counter of DNS requests per type, per zone.",
-	}, []string{"zone", "type"})
-
-	responseSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "response_size_bytes",
-		Help:      "Size of the returned response in bytes.",
-		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
-	}, []string{"zone", "proto"})
-
-	responseTransferSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "response_transfer_size_bytes",
-		Help:      "Size of the returned zone transfer in bytes.",
-		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
-	}, []string{"zone", "proto"})
-
-	responseRcode = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: middleware.Namespace,
-		Subsystem: subsystem,
-		Name:      "response_rcode_count_total",
-		Help:      "Counter of response status codes.",
-	}, []string{"zone", "rcode"})
+func keys(m map[string]bool) []string {
+	sx := []string{}
+	for k := range m {
+		sx = append(sx, k)
+	}
+	return sx
 }
-
-const (
-	// Dropped indicates we dropped the query before any handling. It has no closing dot, so it can not be a valid zone.
-	Dropped   = "dropped"
-	subsystem = "dns"
-	path      = "/metrics"
-)
--- a/middleware/metrics/metrics_test.go
+++ b/middleware/metrics/metrics_test.go
@@ -0,0 +1,83 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/miekg/coredns/middleware"
+	mtest "github.com/miekg/coredns/middleware/metrics/test"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
+	"github.com/miekg/coredns/middleware/test"
+
+	"github.com/miekg/dns"
+	"golang.org/x/net/context"
+)
+
+func TestMetrics(t *testing.T) {
+	met := &Metrics{Addr: Addr, zoneMap: make(map[string]bool)}
+	if err := met.OnStartup(); err != nil {
+		t.Fatalf("Failed to start metrics handler: %s", err)
+	}
+	defer met.OnShutdown()
+
+	met.AddZone("example.org.")
+
+	tests := []struct {
+		next          middleware.Handler
+		qname         string
+		qtype         uint16
+		metric        string
+		expectedValue string
+	}{
+		// This all works because 1 bucket (1 zone, 1 type)
+		{
+			next:          test.NextHandler(dns.RcodeSuccess, nil),
+			qname:         "example.org",
+			metric:        "coredns_dns_request_count_total",
+			expectedValue: "1",
+		},
+		{
+			next:          test.NextHandler(dns.RcodeSuccess, nil),
+			qname:         "example.org",
+			metric:        "coredns_dns_request_count_total",
+			expectedValue: "2",
+		},
+		{
+			next:          test.NextHandler(dns.RcodeSuccess, nil),
+			qname:         "example.org",
+			metric:        "coredns_dns_request_type_count_total",
+			expectedValue: "3",
+		},
+		{
+			next:          test.NextHandler(dns.RcodeSuccess, nil),
+			qname:         "example.org",
+			metric:        "coredns_dns_response_rcode_count_total",
+			expectedValue: "4",
+		},
+	}
+
+	ctx := context.TODO()
+
+	for i, tc := range tests {
+		req := new(dns.Msg)
+		if tc.qtype == 0 {
+			tc.qtype = dns.TypeA
+		}
+		req.SetQuestion(dns.Fqdn(tc.qname), tc.qtype)
+		met.Next = tc.next
+
+		rec := dnsrecorder.New(&test.ResponseWriter{})
+		_, err := met.ServeDNS(ctx, rec, req)
+		if err != nil {
+			t.Fatalf("Test %d: Expected no error, but got %s", i, err)
+		}
+
+		result := mtest.Scrape(t, "http://"+Addr+"/metrics")
+
+		if tc.expectedValue != "" {
+			got, _ := mtest.MetricValue(tc.metric, result)
+			if got != tc.expectedValue {
+				t.Errorf("Test %d: Expected value %s for metrics %s, but got %s", i, tc.expectedValue, tc.metric, got)
+			}
+		}
+	}
+}
--- a/middleware/metrics/setup.go
+++ b/middleware/metrics/setup.go
@@ -38,18 +38,17 @@ func setup(c *caddy.Controller) error {

 func prometheusParse(c *caddy.Controller) (*Metrics, error) {
 	var (
-		met = &Metrics{Addr: addr}
+		met = &Metrics{Addr: Addr, zoneMap: make(map[string]bool)}
 		err error
 	)

 	for c.Next() {
-		if len(met.ZoneNames) > 0 {
-			return met, c.Err("metrics: can only have one metrics module per server")
+		if len(met.ZoneNames()) > 0 {
+			return met, c.Err("can only have one metrics module per server")
 		}
-		met.ZoneNames = make([]string, len(c.ServerBlockKeys))
-		copy(met.ZoneNames, c.ServerBlockKeys)
-		for i := range met.ZoneNames {
-			met.ZoneNames[i] = middleware.Host(met.ZoneNames[i]).Normalize()
+
+		for _, z := range c.ServerBlockKeys {
+			met.AddZone(middleware.Host(z).Normalize())
 		}
 		args := c.RemainingArgs()

@@ -78,7 +77,7 @@ func prometheusParse(c *caddy.Controller) (*Metrics, error) {
 					return met, e
 				}
 			default:
-				return met, c.Errf("metrics: unknown item: %s", c.Val())
+				return met, c.Errf("unknown item: %s", c.Val())
 			}

 		}
@@ -88,4 +87,4 @@ func prometheusParse(c *caddy.Controller) (*Metrics, error) {

 var metricsOnce sync.Once

-const addr = "localhost:9153"
+const Addr = "localhost:9153"
--- a/middleware/metrics/test/scrape.go
+++ b/middleware/metrics/test/scrape.go
@@ -0,0 +1,225 @@
+// Adapted by Miek Gieben for CoreDNS testing.
+//
+// License from prom2json
+// Copyright 2014 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package test will scrape a target and you can inspect the variables.
+// Basic usage:
+//
+//	result := Scrape("http://localhost:9153/metrics")
+//	v := MetricValue("coredns_cache_capacity_gauge", result)
+//
+package test
+
+import (
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"testing"
+
+	"github.com/matttproud/golang_protobuf_extensions/pbutil"
+	"github.com/prometheus/common/expfmt"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+type (
+	// MetricFamily holds a prometheus metric.
+	MetricFamily struct {
+		Name    string        `json:"name"`
+		Help    string        `json:"help"`
+		Type    string        `json:"type"`
+		Metrics []interface{} `json:"metrics,omitempty"` // Either metric or summary.
+	}
+
+	// metric is for all "single value" metrics.
+	metric struct {
+		Labels map[string]string `json:"labels,omitempty"`
+		Value  string            `json:"value"`
+	}
+
+	summary struct {
+		Labels    map[string]string `json:"labels,omitempty"`
+		Quantiles map[string]string `json:"quantiles,omitempty"`
+		Count     string            `json:"count"`
+		Sum       string            `json:"sum"`
+	}
+
+	histogram struct {
+		Labels  map[string]string `json:"labels,omitempty"`
+		Buckets map[string]string `json:"buckets,omitempty"`
+		Count   string            `json:"count"`
+		Sum     string            `json:"sum"`
+	}
+)
+
+// Scrape returns the all the vars a []*metricFamily.
+func Scrape(t *testing.T, url string) []*MetricFamily {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	go fetchMetricFamilies(t, url, mfChan)
+
+	result := []*MetricFamily{}
+	for mf := range mfChan {
+		result = append(result, newMetricFamily(mf))
+	}
+	return result
+}
+
+// MetricValue returns the value associated with name as a string as well as the labels.
+// It only returns the first metrics of the slice.
+func MetricValue(name string, mfs []*MetricFamily) (string, map[string]string) {
+	for _, mf := range mfs {
+		if mf.Name == name {
+			// Only works with Gauge and Counter...
+			return mf.Metrics[0].(metric).Value, mf.Metrics[0].(metric).Labels
+		}
+	}
+	return "", nil
+}
+
+// MetricValueLabel returns the value for name *and* label *value*.
+func MetricValueLabel(name, label string, mfs []*MetricFamily) (string, map[string]string) {
+	// bit hacky is this really handy...?
+	for _, mf := range mfs {
+		if mf.Name == name {
+			for _, m := range mf.Metrics {
+				for _, v := range m.(metric).Labels {
+					if v == label {
+						return m.(metric).Value, m.(metric).Labels
+					}
+				}
+
+			}
+		}
+	}
+	return "", nil
+}
+
+func newMetricFamily(dtoMF *dto.MetricFamily) *MetricFamily {
+	mf := &MetricFamily{
+		Name:    dtoMF.GetName(),
+		Help:    dtoMF.GetHelp(),
+		Type:    dtoMF.GetType().String(),
+		Metrics: make([]interface{}, len(dtoMF.Metric)),
+	}
+	for i, m := range dtoMF.Metric {
+		if dtoMF.GetType() == dto.MetricType_SUMMARY {
+			mf.Metrics[i] = summary{
+				Labels:    makeLabels(m),
+				Quantiles: makeQuantiles(m),
+				Count:     fmt.Sprint(m.GetSummary().GetSampleCount()),
+				Sum:       fmt.Sprint(m.GetSummary().GetSampleSum()),
+			}
+		} else if dtoMF.GetType() == dto.MetricType_HISTOGRAM {
+			mf.Metrics[i] = histogram{
+				Labels:  makeLabels(m),
+				Buckets: makeBuckets(m),
+				Count:   fmt.Sprint(m.GetHistogram().GetSampleCount()),
+				Sum:     fmt.Sprint(m.GetSummary().GetSampleSum()),
+			}
+		} else {
+			mf.Metrics[i] = metric{
+				Labels: makeLabels(m),
+				Value:  fmt.Sprint(value(m)),
+			}
+		}
+	}
+	return mf
+}
+
+func value(m *dto.Metric) float64 {
+	if m.Gauge != nil {
+		return m.GetGauge().GetValue()
+	}
+	if m.Counter != nil {
+		return m.GetCounter().GetValue()
+	}
+	if m.Untyped != nil {
+		return m.GetUntyped().GetValue()
+	}
+	return 0.
+}
+
+func makeLabels(m *dto.Metric) map[string]string {
+	result := map[string]string{}
+	for _, lp := range m.Label {
+		result[lp.GetName()] = lp.GetValue()
+	}
+	return result
+}
+
+func makeQuantiles(m *dto.Metric) map[string]string {
+	result := map[string]string{}
+	for _, q := range m.GetSummary().Quantile {
+		result[fmt.Sprint(q.GetQuantile())] = fmt.Sprint(q.GetValue())
+	}
+	return result
+}
+
+func makeBuckets(m *dto.Metric) map[string]string {
+	result := map[string]string{}
+	for _, b := range m.GetHistogram().Bucket {
+		result[fmt.Sprint(b.GetUpperBound())] = fmt.Sprint(b.GetCumulativeCount())
+	}
+	return result
+}
+
+func fetchMetricFamilies(t *testing.T, url string, ch chan<- *dto.MetricFamily) {
+	defer close(ch)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		t.Fatalf("creating GET request for URL %q failed: %s", url, err)
+	}
+	req.Header.Add("Accept", acceptHeader)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("executing GET request for URL %q failed: %s", url, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
+	}
+
+	mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+	if err == nil && mediatype == "application/vnd.google.protobuf" &&
+		params["encoding"] == "delimited" &&
+		params["proto"] == "io.prometheus.client.MetricFamily" {
+		for {
+			mf := &dto.MetricFamily{}
+			if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
+				if err == io.EOF {
+					break
+				}
+				t.Fatalf("reading metric family protocol buffer failed: %s", err)
+			}
+			ch <- mf
+		}
+	} else {
+		// We could do further content-type checks here, but the
+		// fallback for now will anyway be the text format
+		// version 0.0.4, so just go for it and see if it works.
+		var parser expfmt.TextParser
+		metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
+		if err != nil {
+			t.Fatal("reading text format failed:", err)
+		}
+		for _, mf := range metricFamilies {
+			ch <- mf
+		}
+	}
+}
+
+const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`
--- a/middleware/metrics/vars/report.go
+++ b/middleware/metrics/vars/report.go
@@ -0,0 +1,62 @@
+package vars
+
+import (
+	"time"
+
+	"github.com/miekg/coredns/request"
+
+	"github.com/miekg/dns"
+)
+
+// Report reports the metrics data associcated with request.
+func Report(req request.Request, zone, rcode string, size int, start time.Time) {
+	// Proto and Family
+	net := req.Proto()
+	fam := "1"
+	if req.Family() == 2 {
+		fam = "2"
+	}
+
+	typ := req.QType()
+
+	RequestCount.WithLabelValues(zone, net, fam).Inc()
+	RequestDuration.WithLabelValues(zone).Observe(float64(time.Since(start) / time.Millisecond))
+
+	if req.Do() {
+		RequestDo.WithLabelValues(zone).Inc()
+	}
+
+	if _, known := monitorType[typ]; known {
+		RequestType.WithLabelValues(zone, dns.Type(typ).String()).Inc()
+	} else {
+		RequestType.WithLabelValues(zone, other).Inc()
+	}
+
+	ResponseSize.WithLabelValues(zone, net).Observe(float64(size))
+	RequestSize.WithLabelValues(zone, net).Observe(float64(req.Size()))
+
+	ResponseRcode.WithLabelValues(zone, rcode).Inc()
+}
+
+var monitorType = map[uint16]bool{
+	dns.TypeAAAA:   true,
+	dns.TypeA:      true,
+	dns.TypeCNAME:  true,
+	dns.TypeDNSKEY: true,
+	dns.TypeDS:     true,
+	dns.TypeMX:     true,
+	dns.TypeNSEC3:  true,
+	dns.TypeNSEC:   true,
+	dns.TypeNS:     true,
+	dns.TypePTR:    true,
+	dns.TypeRRSIG:  true,
+	dns.TypeSOA:    true,
+	dns.TypeSRV:    true,
+	dns.TypeTXT:    true,
+	// Meta Qtypes
+	dns.TypeIXFR: true,
+	dns.TypeAXFR: true,
+	dns.TypeANY:  true,
+}
+
+const other = "other"
--- a/middleware/metrics/vars/vars.go
+++ b/middleware/metrics/vars/vars.go
@@ -0,0 +1,68 @@
+package vars
+
+import (
+	"github.com/miekg/coredns/middleware"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	RequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "request_count_total",
+		Help:      "Counter of DNS requests made per zone, protocol and family.",
+	}, []string{"zone", "proto", "family"})
+
+	RequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "request_duration_milliseconds",
+		Buckets:   append(prometheus.DefBuckets, []float64{50, 100, 200, 500, 1000, 2000, 3000, 4000, 5000, 10000}...),
+		Help:      "Histogram of the time (in milliseconds) each request took.",
+	}, []string{"zone"})
+
+	RequestSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "request_size_bytes",
+		Help:      "Size of the EDNS0 UDP buffer in bytes (64K for TCP).",
+		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
+	}, []string{"zone", "proto"})
+
+	RequestDo = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "request_do_count_total",
+		Help:      "Counter of DNS requests with DO bit set per zone.",
+	}, []string{"zone"})
+
+	RequestType = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "request_type_count_total",
+		Help:      "Counter of DNS requests per type, per zone.",
+	}, []string{"zone", "type"})
+
+	ResponseSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "response_size_bytes",
+		Help:      "Size of the returned response in bytes.",
+		Buckets:   []float64{0, 100, 200, 300, 400, 511, 1023, 2047, 4095, 8291, 16e3, 32e3, 48e3, 64e3},
+	}, []string{"zone", "proto"})
+
+	ResponseRcode = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: middleware.Namespace,
+		Subsystem: subsystem,
+		Name:      "response_rcode_count_total",
+		Help:      "Counter of response status codes.",
+	}, []string{"zone", "rcode"})
+)
+
+const (
+	subsystem = "dns"
+
+	// Dropped indicates we dropped the query before any handling. It has no closing dot, so it can not be a valid zone.
+	Dropped = "dropped"
+)