| 
									
										
										
										
											2017-09-14 09:36:06 +01:00
										 |  |  | // Package metrics implement a handler and plugin that provides Prometheus metrics.
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | package metrics
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import (
 | 
					
						
							| 
									
										
										
										
											2016-04-29 07:28:35 +01:00
										 |  |  | 	"net"
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	"net/http"
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	"os"
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	"sync"
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-09-14 09:36:06 +01:00
										 |  |  | 	"github.com/coredns/coredns/plugin"
 | 
					
						
							|  |  |  | 	"github.com/coredns/coredns/plugin/metrics/vars"
 | 
					
						
							| 
									
										
										
											
												Metrics (#1579)
* plugin/metrics: set server address in context
Allow cross server block metrics to co-exist; for this we should label
each metric with the server label. Put this information in the context
and provide a helper function to get it out.
Abstracting with entirely away with difficult as the release client_go
(0.8.0) doesn't have the CurryWith functions yet. So current use is like
so:
define metric, with server label:
	RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: plugin.Namespace,
		Subsystem: "forward",
		Name:      "response_rcode_count_total",
		Help:      "Counter of requests made per upstream.",
	}, []string{"server", "rcode", "to"})
And report ith with the helper function metrics.WithServer:
	RcodeCount.WithLabelValues(metrics.WithServer(ctx), rc, p.addr).Add(1)
											
										 
											2018-04-01 13:57:03 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	"github.com/prometheus/client_golang/prometheus"
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	"github.com/prometheus/client_golang/prometheus/promhttp"
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | )
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics
 | 
					
						
							|  |  |  | type Metrics struct {
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	Next    plugin.Handler
 | 
					
						
							|  |  |  | 	Addr    string
 | 
					
						
							|  |  |  | 	Reg     *prometheus.Registry
 | 
					
						
							|  |  |  | 	ln      net.Listener
 | 
					
						
							|  |  |  | 	lnSetup bool
 | 
					
						
							|  |  |  | 	mux     *http.ServeMux
 | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	zoneNames []string
 | 
					
						
							|  |  |  | 	zoneMap   map[string]bool
 | 
					
						
							|  |  |  | 	zoneMu    sync.RWMutex
 | 
					
						
							|  |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-12-13 16:59:10 -05:00
										 |  |  | // New returns a new instance of Metrics with the given address
 | 
					
						
							|  |  |  | func New(addr string) *Metrics {
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	met := &Metrics{
 | 
					
						
							|  |  |  | 		Addr:    addr,
 | 
					
						
							|  |  |  | 		Reg:     prometheus.NewRegistry(),
 | 
					
						
							|  |  |  | 		zoneMap: make(map[string]bool),
 | 
					
						
							|  |  |  | 	}
 | 
					
						
							|  |  |  | 	// Add the default collectors
 | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | 	met.MustRegister(prometheus.NewGoCollector())
 | 
					
						
							|  |  |  | 	met.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	// Add all of our collectors
 | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 	met.MustRegister(buildInfo)
 | 
					
						
							| 
									
										
										
										
											2018-05-05 19:47:41 +02:00
										 |  |  | 	met.MustRegister(vars.Panic)
 | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | 	met.MustRegister(vars.RequestCount)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestDuration)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestSize)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestDo)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.RequestType)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.ResponseSize)
 | 
					
						
							|  |  |  | 	met.MustRegister(vars.ResponseRcode)
 | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	return met
 | 
					
						
							| 
									
										
										
										
											2017-12-13 16:59:10 -05:00
										 |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-12-27 14:14:53 +00:00
										 |  |  | // MustRegister wraps m.Reg.MustRegister.
 | 
					
						
							|  |  |  | func (m *Metrics) MustRegister(c prometheus.Collector) { m.Reg.MustRegister(c) }
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | // AddZone adds zone z to m.
 | 
					
						
							|  |  |  | func (m *Metrics) AddZone(z string) {
 | 
					
						
							|  |  |  | 	m.zoneMu.Lock()
 | 
					
						
							|  |  |  | 	m.zoneMap[z] = true
 | 
					
						
							|  |  |  | 	m.zoneNames = keys(m.zoneMap)
 | 
					
						
							|  |  |  | 	m.zoneMu.Unlock()
 | 
					
						
							|  |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // RemoveZone remove zone z from m.
 | 
					
						
							|  |  |  | func (m *Metrics) RemoveZone(z string) {
 | 
					
						
							|  |  |  | 	m.zoneMu.Lock()
 | 
					
						
							|  |  |  | 	delete(m.zoneMap, z)
 | 
					
						
							|  |  |  | 	m.zoneNames = keys(m.zoneMap)
 | 
					
						
							|  |  |  | 	m.zoneMu.Unlock()
 | 
					
						
							|  |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // ZoneNames returns the zones of m.
 | 
					
						
							|  |  |  | func (m *Metrics) ZoneNames() []string {
 | 
					
						
							|  |  |  | 	m.zoneMu.RLock()
 | 
					
						
							|  |  |  | 	s := m.zoneNames
 | 
					
						
							|  |  |  | 	m.zoneMu.RUnlock()
 | 
					
						
							|  |  |  | 	return s
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-09-23 09:14:12 +01:00
										 |  |  | // OnStartup sets up the metrics on startup.
 | 
					
						
							|  |  |  | func (m *Metrics) OnStartup() error {
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	ln, err := net.Listen("tcp", m.Addr)
 | 
					
						
							|  |  |  | 	if err != nil {
 | 
					
						
							| 
									
										
										
										
											2018-04-19 07:41:56 +01:00
										 |  |  | 		log.Errorf("Failed to start metrics handler: %s", err)
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 		return err
 | 
					
						
							|  |  |  | 	}
 | 
					
						
							| 
									
										
										
										
											2016-06-25 18:12:13 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	m.ln = ln
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	m.lnSetup = true
 | 
					
						
							|  |  |  | 	ListenAddr = m.ln.Addr().String() // For tests
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	m.mux = http.NewServeMux()
 | 
					
						
							| 
									
										
										
										
											2017-12-14 13:19:03 -05:00
										 |  |  | 	m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{}))
 | 
					
						
							| 
									
										
										
										
											2016-04-29 07:28:35 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2017-02-21 19:34:40 +00:00
										 |  |  | 	go func() {
 | 
					
						
							|  |  |  | 		http.Serve(m.ln, m.mux)
 | 
					
						
							|  |  |  | 	}()
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | 	return nil
 | 
					
						
							|  |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | // OnRestart stops the listener on reload.
 | 
					
						
							|  |  |  | func (m *Metrics) OnRestart() error {
 | 
					
						
							|  |  |  | 	if !m.lnSetup {
 | 
					
						
							|  |  |  | 		return nil
 | 
					
						
							|  |  |  | 	}
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-04-25 11:45:09 +01:00
										 |  |  | 	uniqAddr.SetTodo(m.Addr)
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	m.ln.Close()
 | 
					
						
							|  |  |  | 	m.lnSetup = false
 | 
					
						
							|  |  |  | 	return nil
 | 
					
						
							|  |  |  | }
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // OnFinalShutdown tears down the metrics listener on shutdown and restart.
 | 
					
						
							|  |  |  | func (m *Metrics) OnFinalShutdown() error {
 | 
					
						
							| 
									
										
											  
											
												Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
	/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
	/home/miek
* comments on why
											
										 
											2018-03-02 18:17:05 -08:00
										 |  |  | 	// We allow prometheus statements in multiple Server Blocks, but only the first
 | 
					
						
							|  |  |  | 	// will open the listener, for the rest they are all nil; guard against that.
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 	if !m.lnSetup {
 | 
					
						
							|  |  |  | 		return nil
 | 
					
						
							| 
									
										
											  
											
												Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
	/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
	/home/miek
* comments on why
											
										 
											2018-03-02 18:17:05 -08:00
										 |  |  | 	}
 | 
					
						
							| 
									
										
										
										
											2018-04-21 17:43:02 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 	m.lnSetup = false
 | 
					
						
							|  |  |  | 	return m.ln.Close()
 | 
					
						
							| 
									
										
											  
											
												Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
	/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
	/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
	/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
	/home/miek
* comments on why
											
										 
											2018-03-02 18:17:05 -08:00
										 |  |  | }
 | 
					
						
							| 
									
										
										
										
											2016-04-29 07:28:35 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2016-10-26 10:01:52 +01:00
										 |  |  | func keys(m map[string]bool) []string {
 | 
					
						
							|  |  |  | 	sx := []string{}
 | 
					
						
							|  |  |  | 	for k := range m {
 | 
					
						
							|  |  |  | 		sx = append(sx, k)
 | 
					
						
							|  |  |  | 	}
 | 
					
						
							|  |  |  | 	return sx
 | 
					
						
							| 
									
										
										
										
											2016-03-18 20:57:35 +00:00
										 |  |  | }
 | 
					
						
							| 
									
										
										
										
											2016-10-28 12:57:10 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | // ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where
 | 
					
						
							|  |  |  | // we listen on "localhost:0" and need to retrieve the actual address.
 | 
					
						
							|  |  |  | var ListenAddr string
 | 
					
						
							| 
									
										
										
										
											2018-01-23 21:10:55 +01:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2018-05-05 19:47:41 +02:00
										 |  |  | var buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 | 
					
						
							|  |  |  | 	Namespace: plugin.Namespace,
 | 
					
						
							|  |  |  | 	Name:      "build_info",
 | 
					
						
							|  |  |  | 	Help:      "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
 | 
					
						
							|  |  |  | }, []string{"version", "revision", "goversion"})
 |