2017-09-14 09:36:06 +01:00
|
|
|
// Package metrics implement a handler and plugin that provides Prometheus metrics.
|
2016-03-18 20:57:35 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
|
|
import (
|
2016-04-29 07:28:35 +01:00
|
|
|
"net"
|
2016-03-18 20:57:35 +00:00
|
|
|
"net/http"
|
2017-12-14 13:19:03 -05:00
|
|
|
"os"
|
2016-03-18 20:57:35 +00:00
|
|
|
"sync"
|
|
|
|
|
|
2017-09-14 09:36:06 +01:00
|
|
|
"github.com/coredns/coredns/plugin"
|
|
|
|
|
"github.com/coredns/coredns/plugin/metrics/vars"
|
Metrics (#1579)
* plugin/metrics: set server address in context
Allow cross server block metrics to co-exist; for this we should label
each metric with the server label. Put this information in the context
and provide a helper function to get it out.
Abstracting with entirely away with difficult as the release client_go
(0.8.0) doesn't have the CurryWith functions yet. So current use is like
so:
define metric, with server label:
RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: plugin.Namespace,
Subsystem: "forward",
Name: "response_rcode_count_total",
Help: "Counter of requests made per upstream.",
}, []string{"server", "rcode", "to"})
And report ith with the helper function metrics.WithServer:
RcodeCount.WithLabelValues(metrics.WithServer(ctx), rc, p.addr).Add(1)
2018-04-01 13:57:03 +01:00
|
|
|
|
2016-03-18 20:57:35 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2017-12-14 13:19:03 -05:00
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
2016-03-18 20:57:35 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics
|
|
|
|
|
type Metrics struct {
|
2018-04-21 17:43:02 +01:00
|
|
|
Next plugin.Handler
|
|
|
|
|
Addr string
|
|
|
|
|
Reg *prometheus.Registry
|
|
|
|
|
ln net.Listener
|
|
|
|
|
lnSetup bool
|
|
|
|
|
mux *http.ServeMux
|
2016-10-26 10:01:52 +01:00
|
|
|
|
|
|
|
|
zoneNames []string
|
|
|
|
|
zoneMap map[string]bool
|
|
|
|
|
zoneMu sync.RWMutex
|
|
|
|
|
}
|
|
|
|
|
|
2017-12-13 16:59:10 -05:00
|
|
|
// New returns a new instance of Metrics with the given address
|
|
|
|
|
func New(addr string) *Metrics {
|
2017-12-14 13:19:03 -05:00
|
|
|
met := &Metrics{
|
|
|
|
|
Addr: addr,
|
|
|
|
|
Reg: prometheus.NewRegistry(),
|
|
|
|
|
zoneMap: make(map[string]bool),
|
|
|
|
|
}
|
|
|
|
|
// Add the default collectors
|
2017-12-27 14:14:53 +00:00
|
|
|
met.MustRegister(prometheus.NewGoCollector())
|
|
|
|
|
met.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
|
2017-12-14 13:19:03 -05:00
|
|
|
|
|
|
|
|
// Add all of our collectors
|
2018-01-23 21:10:55 +01:00
|
|
|
met.MustRegister(buildInfo)
|
2018-05-05 19:47:41 +02:00
|
|
|
met.MustRegister(vars.Panic)
|
2017-12-27 14:14:53 +00:00
|
|
|
met.MustRegister(vars.RequestCount)
|
|
|
|
|
met.MustRegister(vars.RequestDuration)
|
|
|
|
|
met.MustRegister(vars.RequestSize)
|
|
|
|
|
met.MustRegister(vars.RequestDo)
|
|
|
|
|
met.MustRegister(vars.RequestType)
|
|
|
|
|
met.MustRegister(vars.ResponseSize)
|
|
|
|
|
met.MustRegister(vars.ResponseRcode)
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2017-12-14 13:19:03 -05:00
|
|
|
return met
|
2017-12-13 16:59:10 -05:00
|
|
|
}
|
|
|
|
|
|
2017-12-27 14:14:53 +00:00
|
|
|
// MustRegister wraps m.Reg.MustRegister.
|
|
|
|
|
func (m *Metrics) MustRegister(c prometheus.Collector) { m.Reg.MustRegister(c) }
|
|
|
|
|
|
2016-10-26 10:01:52 +01:00
|
|
|
// AddZone adds zone z to m.
|
|
|
|
|
func (m *Metrics) AddZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
|
|
|
|
m.zoneMap[z] = true
|
|
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// RemoveZone remove zone z from m.
|
|
|
|
|
func (m *Metrics) RemoveZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
|
|
|
|
delete(m.zoneMap, z)
|
|
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ZoneNames returns the zones of m.
|
|
|
|
|
func (m *Metrics) ZoneNames() []string {
|
|
|
|
|
m.zoneMu.RLock()
|
|
|
|
|
s := m.zoneNames
|
|
|
|
|
m.zoneMu.RUnlock()
|
|
|
|
|
return s
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
|
|
|
|
|
2016-09-23 09:14:12 +01:00
|
|
|
// OnStartup sets up the metrics on startup.
|
|
|
|
|
func (m *Metrics) OnStartup() error {
|
2017-02-21 19:34:40 +00:00
|
|
|
ln, err := net.Listen("tcp", m.Addr)
|
|
|
|
|
if err != nil {
|
2018-04-19 07:41:56 +01:00
|
|
|
log.Errorf("Failed to start metrics handler: %s", err)
|
2017-02-21 19:34:40 +00:00
|
|
|
return err
|
|
|
|
|
}
|
2016-06-25 18:12:13 +01:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.ln = ln
|
2018-04-21 17:43:02 +01:00
|
|
|
m.lnSetup = true
|
|
|
|
|
ListenAddr = m.ln.Addr().String() // For tests
|
2016-03-18 20:57:35 +00:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.mux = http.NewServeMux()
|
2017-12-14 13:19:03 -05:00
|
|
|
m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{}))
|
2016-04-29 07:28:35 +01:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
go func() {
|
|
|
|
|
http.Serve(m.ln, m.mux)
|
|
|
|
|
}()
|
2016-03-18 20:57:35 +00:00
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-21 17:43:02 +01:00
|
|
|
// OnRestart stops the listener on reload.
|
|
|
|
|
func (m *Metrics) OnRestart() error {
|
|
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-25 11:45:09 +01:00
|
|
|
uniqAddr.SetTodo(m.Addr)
|
2018-04-21 17:43:02 +01:00
|
|
|
|
|
|
|
|
m.ln.Close()
|
|
|
|
|
m.lnSetup = false
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// OnFinalShutdown tears down the metrics listener on shutdown and restart.
|
|
|
|
|
func (m *Metrics) OnFinalShutdown() error {
|
Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
/home/miek
* comments on why
2018-03-02 18:17:05 -08:00
|
|
|
// We allow prometheus statements in multiple Server Blocks, but only the first
|
|
|
|
|
// will open the listener, for the rest they are all nil; guard against that.
|
2018-04-21 17:43:02 +01:00
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
/home/miek
* comments on why
2018-03-02 18:17:05 -08:00
|
|
|
}
|
2018-04-21 17:43:02 +01:00
|
|
|
|
|
|
|
|
m.lnSetup = false
|
|
|
|
|
return m.ln.Close()
|
Metrics2 (#1588)
* plugin/metrics: still need nil check for shutdown
the second prometheus statement will trigger: (on control-C)
[signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x94f45a]
goroutine 25 [running]:
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown(0xc420252000, 0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/metrics.go:107 +0x2a
github.com/coredns/coredns/plugin/metrics.(*Metrics).OnShutdown-fm(0x0, 0x0)
/home/miek/g/src/github.com/coredns/coredns/plugin/metrics/setup.go:39 +0x2a
github.com/mholt/caddy.(*Instance).ShutdownCallbacks(0xc4202c81e0, 0x0, 0x0, 0x0)
/home/miek/g/src/github.com/mholt/caddy/caddy.go:164 +0xb3
github.com/mholt/caddy.allShutdownCallbacks(0x1743935, 0x8, 0x14a1b40)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:95 +0x10d
github.com/mholt/caddy.executeShutdownCallbacks.func1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:75 +0x8f
sync.(*Once).Do(0x2256b80, 0xc42036df88)
/home/miek/upstream/go/src/sync/once.go:44 +0xbe
github.com/mholt/caddy.executeShutdownCallbacks(0x174033f, 0x6, 0x0)
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:71 +0x73
github.com/mholt/caddy.trapSignalsCrossPlatform.func1.1()
/home/miek/g/src/github.com/mholt/caddy/sigtrap.go:61 +0x36
created by github.com/mholt/caddy.trapSignalsCrossPlatform.func1
/home/miek
* comments on why
2018-03-02 18:17:05 -08:00
|
|
|
}
|
2016-04-29 07:28:35 +01:00
|
|
|
|
2016-10-26 10:01:52 +01:00
|
|
|
func keys(m map[string]bool) []string {
|
|
|
|
|
sx := []string{}
|
|
|
|
|
for k := range m {
|
|
|
|
|
sx = append(sx, k)
|
|
|
|
|
}
|
|
|
|
|
return sx
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
2016-10-28 12:57:10 +01:00
|
|
|
|
|
|
|
|
// ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where
|
|
|
|
|
// we listen on "localhost:0" and need to retrieve the actual address.
|
|
|
|
|
var ListenAddr string
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2018-05-05 19:47:41 +02:00
|
|
|
var buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
|
|
|
Namespace: plugin.Namespace,
|
|
|
|
|
Name: "build_info",
|
|
|
|
|
Help: "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
|
|
|
|
|
}, []string{"version", "revision", "goversion"})
|