plugin/dnssec: add per server metrics (#1743)

* plugin/dnssec: add per server metrics

final plugin.

Fixes #1696 #1492 #1189

* Move cache cap into handler so we can access the server label

* Remove cache-capacity from it entirely

plugin/dnssec/README.md

@@ -48,10 +48,11 @@ used (See [bugs](#bugs)).
 
 If monitoring is enabled (via the *prometheus* directive) then the following metrics are exported:
 
-* `coredns_dnssec_cache_size{type}` - total elements in the cache, type is "signature".
+* `coredns_dnssec_cache_size{server, type}` - total elements in the cache, type is "signature".
-* `coredns_dnssec_cache_capacity{type}` - total capacity of the cache, type is "signature".
+* `coredns_dnssec_cache_hits_total{server}` - Counter of cache hits.
-* `coredns_dnssec_cache_hits_total{}` - Counter of cache hits.
+* `coredns_dnssec_cache_misses_total{server}` - Counter of cache misses.
-* `coredns_dnssec_cache_misses_total{}` - Counter of cache misses.
+
+The label `server` indicated the server handling the request, see the *metrics* plugin for details.
 
 ## Examples
 

plugin/dnssec/black_lies.go

@@ -14,7 +14,7 @@ import (
 //	a.example.com. 3600 IN NSEC \000.a.example.com. ( RRSIG NSEC ... )
 // This inturn makes every NXDOMAIN answer a NODATA one, don't forget to flip
 // the header rcode to NOERROR.
-func (d Dnssec) nsec(state request.Request, mt response.Type, ttl, incep, expir uint32) ([]dns.RR, error) {
+func (d Dnssec) nsec(state request.Request, mt response.Type, ttl, incep, expir uint32, server string) ([]dns.RR, error) {
 	nsec := &dns.NSEC{}
 	nsec.Hdr = dns.RR_Header{Name: state.QName(), Ttl: ttl, Class: dns.ClassINET, Rrtype: dns.TypeNSEC}
 	nsec.NextDomain = "\\000." + state.QName()
@@ -24,7 +24,7 @@ func (d Dnssec) nsec(state request.Request, mt response.Type, ttl, incep, expir
 		nsec.TypeBitMap = filter14(state.QType(), zoneBitmap, mt)
 	}
 
-	sigs, err := d.sign([]dns.RR{nsec}, state.Zone, ttl, incep, expir)
+	sigs, err := d.sign([]dns.RR{nsec}, state.Zone, ttl, incep, expir, server)
 	if err != nil {
 		return nil, err
 	}

plugin/dnssec/black_lies_bitmap_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/miekg/dns"
 )
 
+const server = "dns//."
+
 func TestBlackLiesBitmapNoData(t *testing.T) {
 	d, rm1, rm2 := newDnssec(t, []string{"example.org."})
 	defer rm1()
@@ -17,7 +19,7 @@ func TestBlackLiesBitmapNoData(t *testing.T) {
 
 	m := testTLSAMsg()
 	state := request.Request{Req: m, Zone: "example.org."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 
 	var nsec *dns.NSEC
 	for _, r := range m.Ns {
@@ -39,7 +41,7 @@ func TestBlackLiesBitmapNameError(t *testing.T) {
 	m := testTLSAMsg()
 	m.Rcode = dns.RcodeNameError // change to name error
 	state := request.Request{Req: m, Zone: "example.org."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 
 	var nsec *dns.NSEC
 	for _, r := range m.Ns {

plugin/dnssec/black_lies_test.go

@@ -17,7 +17,7 @@ func TestZoneSigningBlackLies(t *testing.T) {
 
 	m := testNxdomainMsg()
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Ns, 2) {
 		t.Errorf("authority section should have 2 sig")
 	}
@@ -48,7 +48,7 @@ func TestBlackLiesNoError(t *testing.T) {
 
 	m := testSuccessMsg()
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 
 	if m.Rcode != dns.RcodeSuccess {
 		t.Errorf("expected rcode %d, got %d", dns.RcodeSuccess, m.Rcode)

plugin/dnssec/cache_test.go

@@ -25,9 +25,9 @@ func TestCacheSet(t *testing.T) {
 	state := request.Request{Req: m, Zone: "miek.nl."}
 	k := hash(m.Answer) // calculate *before* we add the sig
 	d := New([]string{"miek.nl."}, []*DNSKEY{dnskey}, nil, c)
-	d.Sign(state, time.Now().UTC())
+	d.Sign(state, time.Now().UTC(), server)
 
-	_, ok := d.get(k)
+	_, ok := d.get(k, server)
 	if !ok {
 		t.Errorf("signature was not added to the cache")
 	}
@@ -49,9 +49,9 @@ func TestCacheNotValidExpired(t *testing.T) {
 	state := request.Request{Req: m, Zone: "miek.nl."}
 	k := hash(m.Answer) // calculate *before* we add the sig
 	d := New([]string{"miek.nl."}, []*DNSKEY{dnskey}, nil, c)
-	d.Sign(state, time.Now().UTC().AddDate(0, 0, -9))
+	d.Sign(state, time.Now().UTC().AddDate(0, 0, -9), server)
 
-	_, ok := d.get(k)
+	_, ok := d.get(k, server)
 	if ok {
 		t.Errorf("signature was added to the cache even though not valid")
 	}
@@ -73,9 +73,9 @@ func TestCacheNotValidYet(t *testing.T) {
 	state := request.Request{Req: m, Zone: "miek.nl."}
 	k := hash(m.Answer) // calculate *before* we add the sig
 	d := New([]string{"miek.nl."}, []*DNSKEY{dnskey}, nil, c)
-	d.Sign(state, time.Now().UTC().AddDate(0, 0, +9))
+	d.Sign(state, time.Now().UTC().AddDate(0, 0, +9), server)
 
-	_, ok := d.get(k)
+	_, ok := d.get(k, server)
 	if ok {
 		t.Errorf("signature was added to the cache even though not valid yet")
 	}

plugin/dnssec/dnskey.go

@@ -57,7 +57,7 @@ func ParseKeyFile(pubFile, privFile string) (*DNSKEY, error) {
 }
 
 // getDNSKEY returns the correct DNSKEY to the client. Signatures are added when do is true.
-func (d Dnssec) getDNSKEY(state request.Request, zone string, do bool) *dns.Msg {
+func (d Dnssec) getDNSKEY(state request.Request, zone string, do bool, server string) *dns.Msg {
 	keys := make([]dns.RR, len(d.keys))
 	for i, k := range d.keys {
 		keys[i] = dns.Copy(k.K)
@@ -71,7 +71,7 @@ func (d Dnssec) getDNSKEY(state request.Request, zone string, do bool) *dns.Msg
 	}
 
 	incep, expir := incepExpir(time.Now().UTC())
-	if sigs, err := d.sign(keys, zone, 3600, incep, expir); err == nil {
+	if sigs, err := d.sign(keys, zone, 3600, incep, expir, server); err == nil {
 		m.Answer = append(m.Answer, sigs...)
 	}
 	return m

plugin/dnssec/dnssec.go

@@ -39,7 +39,7 @@ func New(zones []string, keys []*DNSKEY, next plugin.Handler, c *cache.Cache) Dn
 // will insert DS records and sign those.
 // Signatures will be cached for a short while. By default we sign for 8 days,
 // starting 3 hours ago.
-func (d Dnssec) Sign(state request.Request, now time.Time) *dns.Msg {
+func (d Dnssec) Sign(state request.Request, now time.Time, server string) *dns.Msg {
 	req := state.Req
 
 	incep, expir := incepExpir(now)
@@ -71,10 +71,10 @@ func (d Dnssec) Sign(state request.Request, now time.Time) *dns.Msg {
 
 		ttl := req.Ns[0].Header().Ttl
 
-		if sigs, err := d.sign(req.Ns, state.Zone, ttl, incep, expir); err == nil {
+		if sigs, err := d.sign(req.Ns, state.Zone, ttl, incep, expir, server); err == nil {
 			req.Ns = append(req.Ns, sigs...)
 		}
-		if sigs, err := d.nsec(state, mt, ttl, incep, expir); err == nil {
+		if sigs, err := d.nsec(state, mt, ttl, incep, expir, server); err == nil {
 			req.Ns = append(req.Ns, sigs...)
 		}
 		if len(req.Ns) > 1 { // actually added nsec and sigs, reset the rcode
@@ -85,28 +85,28 @@ func (d Dnssec) Sign(state request.Request, now time.Time) *dns.Msg {
 
 	for _, r := range rrSets(req.Answer) {
 		ttl := r[0].Header().Ttl
-		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir); err == nil {
+		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir, server); err == nil {
 			req.Answer = append(req.Answer, sigs...)
 		}
 	}
 	for _, r := range rrSets(req.Ns) {
 		ttl := r[0].Header().Ttl
-		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir); err == nil {
+		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir, server); err == nil {
 			req.Ns = append(req.Ns, sigs...)
 		}
 	}
 	for _, r := range rrSets(req.Extra) {
 		ttl := r[0].Header().Ttl
-		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir); err == nil {
+		if sigs, err := d.sign(r, state.Zone, ttl, incep, expir, server); err == nil {
 			req.Extra = append(sigs, req.Extra...) // prepend to leave OPT alone
 		}
 	}
 	return req
 }
 
-func (d Dnssec) sign(rrs []dns.RR, signerName string, ttl, incep, expir uint32) ([]dns.RR, error) {
+func (d Dnssec) sign(rrs []dns.RR, signerName string, ttl, incep, expir uint32, server string) ([]dns.RR, error) {
 	k := hash(rrs)
-	sgs, ok := d.get(k)
+	sgs, ok := d.get(k, server)
 	if ok {
 		return sgs, nil
 	}
@@ -129,21 +129,21 @@ func (d Dnssec) set(key uint32, sigs []dns.RR) {
 	d.cache.Add(key, sigs)
 }
 
-func (d Dnssec) get(key uint32) ([]dns.RR, bool) {
+func (d Dnssec) get(key uint32, server string) ([]dns.RR, bool) {
 	if s, ok := d.cache.Get(key); ok {
 		// we sign for 8 days, check if a signature in the cache reached 3/4 of that
 		is75 := time.Now().UTC().Add(sixDays)
 		for _, rr := range s.([]dns.RR) {
 			if !rr.(*dns.RRSIG).ValidityPeriod(is75) {
-				cacheMisses.Inc()
+				cacheMisses.WithLabelValues(server).Inc()
 				return nil, false
 			}
 		}
 
-		cacheHits.Inc()
+		cacheHits.WithLabelValues(server).Inc()
 		return s.([]dns.RR), true
 	}
-	cacheMisses.Inc()
+	cacheMisses.WithLabelValues(server).Inc()
 	return nil, false
 }
 

plugin/dnssec/dnssec_test.go

@@ -19,7 +19,7 @@ func TestZoneSigning(t *testing.T) {
 	m := testMsg()
 	state := request.Request{Req: m, Zone: "miek.nl."}
 
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Answer, 1) {
 		t.Errorf("Answer section should have 1 RRSIG")
 	}
@@ -46,7 +46,7 @@ func TestZoneSigningDouble(t *testing.T) {
 
 	m := testMsg()
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Answer, 2) {
 		t.Errorf("Answer section should have 1 RRSIG")
 	}
@@ -71,7 +71,7 @@ func TestSigningDifferentZone(t *testing.T) {
 	state := request.Request{Req: m, Zone: "example.org."}
 	c := cache.New(defaultCap)
 	d := New([]string{"example.org."}, []*DNSKEY{key}, nil, c)
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Answer, 1) {
 		t.Errorf("Answer section should have 1 RRSIG")
 		t.Logf("%+v\n", m)
@@ -89,7 +89,7 @@ func TestSigningCname(t *testing.T) {
 
 	m := testMsgCname()
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Answer, 1) {
 		t.Errorf("Answer section should have 1 RRSIG")
 	}
@@ -103,7 +103,7 @@ func testZoneSigningDelegation(t *testing.T) {
 
 	m := testDelegationMsg()
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Ns, 1) {
 		t.Errorf("Authority section should have 1 RRSIG")
 		t.Logf("%v\n", m)
@@ -134,7 +134,7 @@ func TestSigningDname(t *testing.T) {
 	m := testMsgDname()
 	state := request.Request{Req: m, Zone: "miek.nl."}
 	// We sign *everything* we see, also the synthesized CNAME.
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Answer, 3) {
 		t.Errorf("Answer section should have 3 RRSIGs")
 	}
@@ -148,7 +148,7 @@ func TestSigningEmpty(t *testing.T) {
 	m := testEmptyMsg()
 	m.SetQuestion("a.miek.nl.", dns.TypeA)
 	state := request.Request{Req: m, Zone: "miek.nl."}
-	m = d.Sign(state, time.Now().UTC())
+	m = d.Sign(state, time.Now().UTC(), server)
 	if !section(m.Ns, 2) {
 		t.Errorf("Authority section should have 2 RRSIGs")
 	}

plugin/dnssec/handler.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/metrics"
 	"github.com/coredns/coredns/request"
 
 	"github.com/miekg/dns"
@@ -24,13 +25,14 @@ func (d Dnssec) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
 	}
 
 	state.Zone = zone
+	server := metrics.WithServer(ctx)
 
 	// Intercept queries for DNSKEY, but only if one of the zones matches the qname, otherwise we let
 	// the query through.
 	if qtype == dns.TypeDNSKEY {
 		for _, z := range d.zones {
 			if qname == z {
-				resp := d.getDNSKEY(state, z, do)
+				resp := d.getDNSKEY(state, z, do, server)
 				resp.Authoritative = true
 				state.SizeAndDo(resp)
 				w.WriteMsg(resp)
@@ -39,7 +41,7 @@ func (d Dnssec) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
 		}
 	}
 
-	drr := &ResponseWriter{w, d}
+	drr := &ResponseWriter{w, d, server}
 	return plugin.NextOrFailure(d.Name(), d.Next, ctx, drr, r)
 }
 
@@ -49,28 +51,28 @@ var (
 		Subsystem: "dnssec",
 		Name:      "cache_size",
 		Help:      "The number of elements in the dnssec cache.",
-	}, []string{"type"})
+	}, []string{"server", "type"})
 
 	cacheCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "dnssec",
 		Name:      "cache_capacity",
 		Help:      "The dnssec cache's capacity.",
-	}, []string{"type"})
+	}, []string{"server", "type"})
 
-	cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
+	cacheHits = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "dnssec",
 		Name:      "cache_hits_total",
 		Help:      "The count of cache hits.",
-	})
+	}, []string{"server"})
 
-	cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
+	cacheMisses = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: plugin.Namespace,
 		Subsystem: "dnssec",
 		Name:      "cache_misses_total",
 		Help:      "The count of cache misses.",
-	})
+	}, []string{"server"})
 )
 
 // Name implements the Handler interface.

plugin/dnssec/responsewriter.go

@@ -13,6 +13,7 @@ import (
 type ResponseWriter struct {
 	dns.ResponseWriter
 	d      Dnssec
+	server string // server label for metrics.
 }
 
 // WriteMsg implements the dns.ResponseWriter interface.
@@ -28,9 +29,9 @@ func (d *ResponseWriter) WriteMsg(res *dns.Msg) error {
 	state.Zone = zone
 
 	if state.Do() {
-		res = d.d.Sign(state, time.Now().UTC())
+		res = d.d.Sign(state, time.Now().UTC(), d.server)
 
-		cacheSize.WithLabelValues("signature").Set(float64(d.d.cache.Len()))
+		cacheSize.WithLabelValues(d.server, "signature").Set(float64(d.d.cache.Len()))
 	}
 	state.SizeAndDo(res)
 

plugin/dnssec/setup.go

@@ -36,14 +36,11 @@ func setup(c *caddy.Controller) error {
 
 	c.OnStartup(func() error {
 		once.Do(func() {
-			metrics.MustRegister(c, cacheSize, cacheCapacity, cacheHits, cacheMisses)
+			metrics.MustRegister(c, cacheSize, cacheHits, cacheMisses)
 		})
 		return nil
 	})
 
-	// Export the capacity for the metrics. This only happens once, because this is a re-load change only.
-	cacheCapacity.WithLabelValues("signature").Set(float64(capacity))
-
 	return nil
 }