2017-09-14 09:36:06 +01:00
|
|
|
// Package metrics implement a handler and plugin that provides Prometheus metrics.
|
2016-03-18 20:57:35 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
|
|
import (
|
2018-08-21 08:52:25 -07:00
|
|
|
"context"
|
2016-04-29 07:28:35 +01:00
|
|
|
"net"
|
2016-03-18 20:57:35 +00:00
|
|
|
"net/http"
|
|
|
|
|
"sync"
|
2018-08-21 08:52:25 -07:00
|
|
|
"time"
|
2016-03-18 20:57:35 +00:00
|
|
|
|
2017-09-14 09:36:06 +01:00
|
|
|
"github.com/coredns/coredns/plugin"
|
|
|
|
|
"github.com/coredns/coredns/plugin/metrics/vars"
|
Metrics (#1579)
* plugin/metrics: set server address in context
Allow cross server block metrics to co-exist; for this we should label
each metric with the server label. Put this information in the context
and provide a helper function to get it out.
Abstracting with entirely away with difficult as the release client_go
(0.8.0) doesn't have the CurryWith functions yet. So current use is like
so:
define metric, with server label:
RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: plugin.Namespace,
Subsystem: "forward",
Name: "response_rcode_count_total",
Help: "Counter of requests made per upstream.",
}, []string{"server", "rcode", "to"})
And report ith with the helper function metrics.WithServer:
RcodeCount.WithLabelValues(metrics.WithServer(ctx), rc, p.addr).Add(1)
2018-04-01 13:57:03 +01:00
|
|
|
|
2016-03-18 20:57:35 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2017-12-14 13:19:03 -05:00
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
2016-03-18 20:57:35 +00:00
|
|
|
)
|
|
|
|
|
|
2019-05-18 18:34:46 +01:00
|
|
|
// Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics .
|
2016-03-18 20:57:35 +00:00
|
|
|
type Metrics struct {
|
2019-05-18 18:34:46 +01:00
|
|
|
Next plugin.Handler
|
|
|
|
|
Addr string
|
|
|
|
|
Reg *prometheus.Registry
|
|
|
|
|
|
2018-04-21 17:43:02 +01:00
|
|
|
ln net.Listener
|
|
|
|
|
lnSetup bool
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
mux *http.ServeMux
|
|
|
|
|
srv *http.Server
|
2016-10-26 10:01:52 +01:00
|
|
|
|
|
|
|
|
zoneNames []string
|
2018-12-10 10:17:15 +00:00
|
|
|
zoneMap map[string]struct{}
|
2016-10-26 10:01:52 +01:00
|
|
|
zoneMu sync.RWMutex
|
|
|
|
|
}
|
|
|
|
|
|
2019-05-18 18:34:46 +01:00
|
|
|
// New returns a new instance of Metrics with the given address.
|
2017-12-13 16:59:10 -05:00
|
|
|
func New(addr string) *Metrics {
|
2017-12-14 13:19:03 -05:00
|
|
|
met := &Metrics{
|
|
|
|
|
Addr: addr,
|
|
|
|
|
Reg: prometheus.NewRegistry(),
|
2018-12-10 10:17:15 +00:00
|
|
|
zoneMap: make(map[string]struct{}),
|
2017-12-14 13:19:03 -05:00
|
|
|
}
|
|
|
|
|
// Add the default collectors
|
2017-12-27 14:14:53 +00:00
|
|
|
met.MustRegister(prometheus.NewGoCollector())
|
2018-12-01 14:38:03 -08:00
|
|
|
met.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
|
2017-12-14 13:19:03 -05:00
|
|
|
|
|
|
|
|
// Add all of our collectors
|
2018-01-23 21:10:55 +01:00
|
|
|
met.MustRegister(buildInfo)
|
2018-05-05 19:47:41 +02:00
|
|
|
met.MustRegister(vars.Panic)
|
2017-12-27 14:14:53 +00:00
|
|
|
met.MustRegister(vars.RequestCount)
|
|
|
|
|
met.MustRegister(vars.RequestDuration)
|
|
|
|
|
met.MustRegister(vars.RequestSize)
|
|
|
|
|
met.MustRegister(vars.RequestDo)
|
|
|
|
|
met.MustRegister(vars.RequestType)
|
|
|
|
|
met.MustRegister(vars.ResponseSize)
|
|
|
|
|
met.MustRegister(vars.ResponseRcode)
|
2019-03-23 10:43:15 +01:00
|
|
|
met.MustRegister(vars.PluginEnabled)
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2017-12-14 13:19:03 -05:00
|
|
|
return met
|
2017-12-13 16:59:10 -05:00
|
|
|
}
|
|
|
|
|
|
2017-12-27 14:14:53 +00:00
|
|
|
// MustRegister wraps m.Reg.MustRegister.
|
2018-11-01 15:56:00 -04:00
|
|
|
func (m *Metrics) MustRegister(c prometheus.Collector) {
|
|
|
|
|
err := m.Reg.Register(c)
|
|
|
|
|
if err != nil {
|
|
|
|
|
// ignore any duplicate error, but fatal on any other kind of error
|
|
|
|
|
if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
|
|
|
|
|
log.Fatalf("Cannot register metrics collector: %s", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2017-12-27 14:14:53 +00:00
|
|
|
|
2016-10-26 10:01:52 +01:00
|
|
|
// AddZone adds zone z to m.
|
|
|
|
|
func (m *Metrics) AddZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
2018-12-10 10:17:15 +00:00
|
|
|
m.zoneMap[z] = struct{}{}
|
2016-10-26 10:01:52 +01:00
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// RemoveZone remove zone z from m.
|
|
|
|
|
func (m *Metrics) RemoveZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
|
|
|
|
delete(m.zoneMap, z)
|
|
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ZoneNames returns the zones of m.
|
|
|
|
|
func (m *Metrics) ZoneNames() []string {
|
|
|
|
|
m.zoneMu.RLock()
|
|
|
|
|
s := m.zoneNames
|
|
|
|
|
m.zoneMu.RUnlock()
|
|
|
|
|
return s
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
|
|
|
|
|
2016-09-23 09:14:12 +01:00
|
|
|
// OnStartup sets up the metrics on startup.
|
|
|
|
|
func (m *Metrics) OnStartup() error {
|
2017-02-21 19:34:40 +00:00
|
|
|
ln, err := net.Listen("tcp", m.Addr)
|
|
|
|
|
if err != nil {
|
2018-04-19 07:41:56 +01:00
|
|
|
log.Errorf("Failed to start metrics handler: %s", err)
|
2017-02-21 19:34:40 +00:00
|
|
|
return err
|
|
|
|
|
}
|
2016-06-25 18:12:13 +01:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.ln = ln
|
2018-04-21 17:43:02 +01:00
|
|
|
m.lnSetup = true
|
2016-03-18 20:57:35 +00:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.mux = http.NewServeMux()
|
2017-12-14 13:19:03 -05:00
|
|
|
m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{}))
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
// creating some helper variables to avoid data races on m.srv and m.ln
|
|
|
|
|
server := &http.Server{Handler: m.mux}
|
|
|
|
|
m.srv = server
|
|
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
go func() {
|
2019-05-18 18:34:46 +01:00
|
|
|
server.Serve(ln)
|
2017-02-21 19:34:40 +00:00
|
|
|
}()
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
ListenAddr = ln.Addr().String() // For tests.
|
2016-03-18 20:57:35 +00:00
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-21 17:43:02 +01:00
|
|
|
// OnRestart stops the listener on reload.
|
|
|
|
|
func (m *Metrics) OnRestart() error {
|
|
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
2019-05-18 18:34:46 +01:00
|
|
|
u.Unset(m.Addr)
|
2018-08-21 08:52:25 -07:00
|
|
|
return m.stopServer()
|
|
|
|
|
}
|
2018-04-21 17:43:02 +01:00
|
|
|
|
2018-08-21 08:52:25 -07:00
|
|
|
func (m *Metrics) stopServer() error {
|
|
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
|
|
|
|
defer cancel()
|
|
|
|
|
if err := m.srv.Shutdown(ctx); err != nil {
|
|
|
|
|
log.Infof("Failed to stop prometheus http server: %s", err)
|
|
|
|
|
return err
|
|
|
|
|
}
|
2018-04-21 17:43:02 +01:00
|
|
|
m.lnSetup = false
|
2018-08-21 08:52:25 -07:00
|
|
|
m.ln.Close()
|
2018-04-21 17:43:02 +01:00
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// OnFinalShutdown tears down the metrics listener on shutdown and restart.
|
2019-05-13 12:26:05 +01:00
|
|
|
func (m *Metrics) OnFinalShutdown() error { return m.stopServer() }
|
2016-04-29 07:28:35 +01:00
|
|
|
|
2018-12-10 10:17:15 +00:00
|
|
|
func keys(m map[string]struct{}) []string {
|
2016-10-26 10:01:52 +01:00
|
|
|
sx := []string{}
|
|
|
|
|
for k := range m {
|
|
|
|
|
sx = append(sx, k)
|
|
|
|
|
}
|
|
|
|
|
return sx
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
2016-10-28 12:57:10 +01:00
|
|
|
|
|
|
|
|
// ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where
|
|
|
|
|
// we listen on "localhost:0" and need to retrieve the actual address.
|
|
|
|
|
var ListenAddr string
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2018-08-21 08:52:25 -07:00
|
|
|
// shutdownTimeout is the maximum amount of time the metrics plugin will wait
|
|
|
|
|
// before erroring when it tries to close the metrics server
|
|
|
|
|
const shutdownTimeout time.Duration = time.Second * 5
|
|
|
|
|
|
2018-05-05 19:47:41 +02:00
|
|
|
var buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
|
|
|
Namespace: plugin.Namespace,
|
|
|
|
|
Name: "build_info",
|
|
|
|
|
Help: "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
|
|
|
|
|
}, []string{"version", "revision", "goversion"})
|