| 
									
										
										
										
											2017-09-14 09:36:06 +01:00
										 |  |  | // Package metrics implement a handler and plugin that provides Prometheus metrics. | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | package metrics | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | 	"context" | 
					
						
							| 
									
										
										
										
											2016-04-29 07:28:35 +01:00
										 |  |  | 	"net" | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	"net/http" | 
					
						
							|  |  |  | 	"sync" | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | 	"time" | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-09-14 09:36:06 +01:00
										 |  |  | 	"github.com/coredns/coredns/plugin" | 
					
						
							|  |  |  | 	"github.com/coredns/coredns/plugin/metrics/vars" | 
					
						
							| 
									
										
										
										
											2019-12-06 18:55:40 +08:00
										 |  |  | 	"github.com/coredns/coredns/plugin/pkg/reuseport" | 
					
						
							| 
									
										
										
											
												Metrics (#1579)
* plugin/metrics: set server address in context
Allow cross server block metrics to co-exist; for this we should label
each metric with the server label. Put this information in the context
and provide a helper function to get it out.
Abstracting with entirely away with difficult as the release client_go
(0.8.0) doesn't have the CurryWith functions yet. So current use is like
so:
define metric, with server label:
	RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: plugin.Namespace,
		Subsystem: "forward",
		Name:      "response_rcode_count_total",
		Help:      "Counter of requests made per upstream.",
	}, []string{"server", "rcode", "to"})
And report ith with the helper function metrics.WithServer:
	RcodeCount.WithLabelValues(metrics.WithServer(ctx), rc, p.addr).Add(1)
											
										 
											2018-04-01 13:57:03 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	"github.com/prometheus/client_golang/prometheus" | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	"github.com/prometheus/client_golang/prometheus/promhttp" | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics . | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | type Metrics struct { | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 	Next plugin.Handler | 
					
						
							|  |  |  | 	Addr string | 
					
						
							|  |  |  | 	Reg  *prometheus.Registry | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	ln      net.Listener | 
					
						
							|  |  |  | 	lnSetup bool | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	mux *http.ServeMux | 
					
						
							|  |  |  | 	srv *http.Server | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	zoneNames []string | 
					
						
							| 
									
										
										
										
											2018-12-10 10:17:15 +00:00
										 |  |  | 	zoneMap   map[string]struct{} | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | 	zoneMu    sync.RWMutex | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | // New returns a new instance of Metrics with the given address. | 
					
						
							| 
									
										
										
										
											2017-12-13 16:59:10 -05:00
										 |  |  | func New(addr string) *Metrics { | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	met := &Metrics{ | 
					
						
							|  |  |  | 		Addr:    addr, | 
					
						
							|  |  |  | 		Reg:     prometheus.NewRegistry(), | 
					
						
							| 
									
										
										
										
											2018-12-10 10:17:15 +00:00
										 |  |  | 		zoneMap: make(map[string]struct{}), | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	} | 
					
						
							|  |  |  | 	// Add the default collectors | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | 	met.MustRegister(prometheus.NewGoCollector()) | 
					
						
							| 
									
										
										
										
											2018-12-01 14:38:03 -08:00
										 |  |  | 	met.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// Add all of our collectors | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 	met.MustRegister(buildInfo) | 
					
						
							| 
									
										
										
										
											2018-05-05 19:47:41 +02:00
										 |  |  | 	met.MustRegister(vars.Panic) | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | 	met.MustRegister(vars.RequestCount) | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestDuration) | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestSize) | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestDo) | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestType) | 
					
						
							|  |  |  | 	met.MustRegister(vars.ResponseSize) | 
					
						
							|  |  |  | 	met.MustRegister(vars.ResponseRcode) | 
					
						
							| 
									
										
										
										
											2019-03-23 10:43:15 +01:00
										 |  |  | 	met.MustRegister(vars.PluginEnabled) | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	return met | 
					
						
							| 
									
										
										
										
											2017-12-13 16:59:10 -05:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | // MustRegister wraps m.Reg.MustRegister. | 
					
						
							| 
									
										
										
										
											2018-11-01 15:56:00 -04:00
										 |  |  | func (m *Metrics) MustRegister(c prometheus.Collector) { | 
					
						
							|  |  |  | 	err := m.Reg.Register(c) | 
					
						
							|  |  |  | 	if err != nil { | 
					
						
							|  |  |  | 		// ignore any duplicate error, but fatal on any other kind of error | 
					
						
							|  |  |  | 		if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { | 
					
						
							|  |  |  | 			log.Fatalf("Cannot register metrics collector: %s", err) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | // AddZone adds zone z to m. | 
					
						
							|  |  |  | func (m *Metrics) AddZone(z string) { | 
					
						
							|  |  |  | 	m.zoneMu.Lock() | 
					
						
							| 
									
										
										
										
											2018-12-10 10:17:15 +00:00
										 |  |  | 	m.zoneMap[z] = struct{}{} | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | 	m.zoneNames = keys(m.zoneMap) | 
					
						
							|  |  |  | 	m.zoneMu.Unlock() | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // RemoveZone remove zone z from m. | 
					
						
							|  |  |  | func (m *Metrics) RemoveZone(z string) { | 
					
						
							|  |  |  | 	m.zoneMu.Lock() | 
					
						
							|  |  |  | 	delete(m.zoneMap, z) | 
					
						
							|  |  |  | 	m.zoneNames = keys(m.zoneMap) | 
					
						
							|  |  |  | 	m.zoneMu.Unlock() | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // ZoneNames returns the zones of m. | 
					
						
							|  |  |  | func (m *Metrics) ZoneNames() []string { | 
					
						
							|  |  |  | 	m.zoneMu.RLock() | 
					
						
							|  |  |  | 	s := m.zoneNames | 
					
						
							|  |  |  | 	m.zoneMu.RUnlock() | 
					
						
							|  |  |  | 	return s | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-09-23 09:14:12 +01:00
										 |  |  | // OnStartup sets up the metrics on startup. | 
					
						
							|  |  |  | func (m *Metrics) OnStartup() error { | 
					
						
							| 
									
										
										
										
											2019-12-06 18:55:40 +08:00
										 |  |  | 	ln, err := reuseport.Listen("tcp", m.Addr) | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	if err != nil { | 
					
						
							| 
									
										
										
										
											2018-04-19 07:41:56 +01:00
										 |  |  | 		log.Errorf("Failed to start metrics handler: %s", err) | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 		return err | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2016-06-25 18:12:13 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	m.ln = ln | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	m.lnSetup = true | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	m.mux = http.NewServeMux() | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{})) | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// creating some helper variables to avoid data races on m.srv and m.ln | 
					
						
							|  |  |  | 	server := &http.Server{Handler: m.mux} | 
					
						
							|  |  |  | 	m.srv = server | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	go func() { | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 		server.Serve(ln) | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	}() | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	ListenAddr = ln.Addr().String() // For tests. | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	return nil | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | // OnRestart stops the listener on reload. | 
					
						
							|  |  |  | func (m *Metrics) OnRestart() error { | 
					
						
							|  |  |  | 	if !m.lnSetup { | 
					
						
							|  |  |  | 		return nil | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2019-05-18 18:34:46 +01:00
										 |  |  | 	u.Unset(m.Addr) | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | 	return m.stopServer() | 
					
						
							|  |  |  | } | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | func (m *Metrics) stopServer() error { | 
					
						
							|  |  |  | 	if !m.lnSetup { | 
					
						
							|  |  |  | 		return nil | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) | 
					
						
							|  |  |  | 	defer cancel() | 
					
						
							|  |  |  | 	if err := m.srv.Shutdown(ctx); err != nil { | 
					
						
							|  |  |  | 		log.Infof("Failed to stop prometheus http server: %s", err) | 
					
						
							|  |  |  | 		return err | 
					
						
							|  |  |  | 	} | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	m.lnSetup = false | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | 	m.ln.Close() | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	return nil | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // OnFinalShutdown tears down the metrics listener on shutdown and restart. | 
					
						
							| 
									
										
										
										
											2019-05-13 12:26:05 +01:00
										 |  |  | func (m *Metrics) OnFinalShutdown() error { return m.stopServer() } | 
					
						
							| 
									
										
										
										
											2016-04-29 07:28:35 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-12-10 10:17:15 +00:00
										 |  |  | func keys(m map[string]struct{}) []string { | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | 	sx := []string{} | 
					
						
							|  |  |  | 	for k := range m { | 
					
						
							|  |  |  | 		sx = append(sx, k) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 	return sx | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | } | 
					
						
							| 
									
										
										
										
											2016-10-28 12:57:10 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | // ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where | 
					
						
							|  |  |  | // we listen on "localhost:0" and need to retrieve the actual address. | 
					
						
							|  |  |  | var ListenAddr string | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-08-21 08:52:25 -07:00
										 |  |  | // shutdownTimeout is the maximum amount of time the metrics plugin will wait | 
					
						
							|  |  |  | // before erroring when it tries to close the metrics server | 
					
						
							|  |  |  | const shutdownTimeout time.Duration = time.Second * 5 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-05-05 19:47:41 +02:00
										 |  |  | var buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{ | 
					
						
							|  |  |  | 	Namespace: plugin.Namespace, | 
					
						
							|  |  |  | 	Name:      "build_info", | 
					
						
							|  |  |  | 	Help:      "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.", | 
					
						
							|  |  |  | }, []string{"version", "revision", "goversion"}) |