middleware/metrics: survive restart (#542)

* middleware/metrics: survive restart

Keep the handler running during restart. Stopping and starting the
handler results in "address in use" - sometimes, meaning the reload
will be flaky. In turn this behavior means any changes to the monitor
stanza are not picked up.

* remove resync

middleware/metrics/README.md

@@ -27,6 +27,7 @@ Extra labels used are:
 If monitoring is enabled, queries that do not enter the middleware chain are exported under the fake
 name "dropped" (without a closing dot - this is never a valid domain name).
 
+
 ## Syntax
 
 ~~~
@@ -45,3 +46,8 @@ Use an alternative address:
 ~~~
 prometheus localhost:9253
 ~~~
+
+# Bugs
+
+When reloading, we keep the handler running, meaning that any changes to the handler aren't picked
+up. You'll need to restart CoreDNS for that to happen.

middleware/metrics/metrics.go

@@ -13,13 +13,23 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+func init() {
+	prometheus.MustRegister(vars.RequestCount)
+	prometheus.MustRegister(vars.RequestDuration)
+	prometheus.MustRegister(vars.RequestSize)
+	prometheus.MustRegister(vars.RequestDo)
+	prometheus.MustRegister(vars.RequestType)
+
+	prometheus.MustRegister(vars.ResponseSize)
+	prometheus.MustRegister(vars.ResponseRcode)
+}
+
 // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics
 type Metrics struct {
 	Next middleware.Handler
 	Addr string
 	ln   net.Listener
 	mux  *http.ServeMux
-	Once sync.Once
 
 	zoneNames []string
 	zoneMap   map[string]bool
@@ -52,38 +62,25 @@ func (m *Metrics) ZoneNames() []string {
 
 // OnStartup sets up the metrics on startup.
 func (m *Metrics) OnStartup() error {
-	m.Once.Do(func() {
+	ln, err := net.Listen("tcp", m.Addr)
+	if err != nil {
+		log.Printf("[ERROR] Failed to start metrics handler: %s", err)
+		return err
+	}
 
-		ln, err := net.Listen("tcp", m.Addr)
+	m.ln = ln
-		if err != nil {
+	ListenAddr = m.ln.Addr().String()
-			log.Printf("[ERROR] Failed to start metrics handler: %s", err)
-			return
-		}
 
-		m.ln = ln
+	m.mux = http.NewServeMux()
-		ListenAddr = m.ln.Addr().String()
+	m.mux.Handle("/metrics", prometheus.Handler())
 
-		m.mux = http.NewServeMux()
+	go func() {
-
+		http.Serve(m.ln, m.mux)
-		prometheus.MustRegister(vars.RequestCount)
+	}()
-		prometheus.MustRegister(vars.RequestDuration)
-		prometheus.MustRegister(vars.RequestSize)
-		prometheus.MustRegister(vars.RequestDo)
-		prometheus.MustRegister(vars.RequestType)
-
-		prometheus.MustRegister(vars.ResponseSize)
-		prometheus.MustRegister(vars.ResponseRcode)
-
-		m.mux.Handle("/metrics", prometheus.Handler())
-
-		go func() {
-			http.Serve(m.ln, m.mux)
-		}()
-	})
 	return nil
 }
 
-// OnShutdown tears down the metrics on shutdown.
+// OnShutdown tears down the metrics on shutdown and restart.
 func (m *Metrics) OnShutdown() error {
 	if m.ln != nil {
 		return m.ln.Close()

middleware/metrics/setup.go

@@ -28,9 +28,10 @@ func setup(c *caddy.Controller) error {
 		return m
 	})
 
+	// During restarts we will keep this handler running.
 	metricsOnce.Do(func() {
-		c.OnStartup(m.OnStartup)
+		c.OncePerServerBlock(m.OnStartup)
-		c.OnShutdown(m.OnShutdown)
+		c.OnFinalShutdown(m.OnShutdown)
 	})
 
 	return nil