2017-09-14 09:36:06 +01:00
|
|
|
// Package metrics implement a handler and plugin that provides Prometheus metrics.
|
2016-03-18 20:57:35 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
|
|
import (
|
2018-08-21 08:52:25 -07:00
|
|
|
"context"
|
2016-04-29 07:28:35 +01:00
|
|
|
"net"
|
2016-03-18 20:57:35 +00:00
|
|
|
"net/http"
|
|
|
|
|
"sync"
|
2018-08-21 08:52:25 -07:00
|
|
|
"time"
|
2016-03-18 20:57:35 +00:00
|
|
|
|
Metrics: expand coredns_dns_responses_total with plugin label (#4914)
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* better comment
Signed-off-by: Miek Gieben <miek@miek.nl>
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* Update core/dnsserver/server.go
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
* Use [3]string
Signed-off-by: Miek Gieben <miek@miek.nl>
* imports
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove dnstest changes
Signed-off-by: Miek Gieben <miek@miek.nl>
* revert
Signed-off-by: Miek Gieben <miek@miek.nl>
* Add some sleeps to make it less flaky
Signed-off-by: Miek Gieben <miek@miek.nl>
* Revert "Add some sleeps to make it less flaky"
This reverts commit b5c6655196e3ad570555f086832ceb1f48f6f2d5.
* Remove forward when not needed
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove newline
Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
2021-11-12 16:07:05 +00:00
|
|
|
"github.com/coredns/caddy"
|
2017-09-14 09:36:06 +01:00
|
|
|
"github.com/coredns/coredns/plugin"
|
2019-12-06 18:55:40 +08:00
|
|
|
"github.com/coredns/coredns/plugin/pkg/reuseport"
|
Metrics (#1579)
* plugin/metrics: set server address in context
Allow cross server block metrics to co-exist; for this we should label
each metric with the server label. Put this information in the context
and provide a helper function to get it out.
Abstracting with entirely away with difficult as the release client_go
(0.8.0) doesn't have the CurryWith functions yet. So current use is like
so:
define metric, with server label:
RcodeCount = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: plugin.Namespace,
Subsystem: "forward",
Name: "response_rcode_count_total",
Help: "Counter of requests made per upstream.",
}, []string{"server", "rcode", "to"})
And report ith with the helper function metrics.WithServer:
RcodeCount.WithLabelValues(metrics.WithServer(ctx), rc, p.addr).Add(1)
2018-04-01 13:57:03 +01:00
|
|
|
|
2016-03-18 20:57:35 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2020-07-25 23:06:28 +08:00
|
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
2017-12-14 13:19:03 -05:00
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
2016-03-18 20:57:35 +00:00
|
|
|
)
|
|
|
|
|
|
2019-05-18 18:34:46 +01:00
|
|
|
// Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics .
|
2016-03-18 20:57:35 +00:00
|
|
|
type Metrics struct {
|
2019-05-18 18:34:46 +01:00
|
|
|
Next plugin.Handler
|
|
|
|
|
Addr string
|
|
|
|
|
Reg *prometheus.Registry
|
|
|
|
|
|
2018-04-21 17:43:02 +01:00
|
|
|
ln net.Listener
|
|
|
|
|
lnSetup bool
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
mux *http.ServeMux
|
|
|
|
|
srv *http.Server
|
2016-10-26 10:01:52 +01:00
|
|
|
|
|
|
|
|
zoneNames []string
|
2018-12-10 10:17:15 +00:00
|
|
|
zoneMap map[string]struct{}
|
2016-10-26 10:01:52 +01:00
|
|
|
zoneMu sync.RWMutex
|
Metrics: expand coredns_dns_responses_total with plugin label (#4914)
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* better comment
Signed-off-by: Miek Gieben <miek@miek.nl>
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* Update core/dnsserver/server.go
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
* Use [3]string
Signed-off-by: Miek Gieben <miek@miek.nl>
* imports
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove dnstest changes
Signed-off-by: Miek Gieben <miek@miek.nl>
* revert
Signed-off-by: Miek Gieben <miek@miek.nl>
* Add some sleeps to make it less flaky
Signed-off-by: Miek Gieben <miek@miek.nl>
* Revert "Add some sleeps to make it less flaky"
This reverts commit b5c6655196e3ad570555f086832ceb1f48f6f2d5.
* Remove forward when not needed
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove newline
Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
2021-11-12 16:07:05 +00:00
|
|
|
|
|
|
|
|
plugins map[string]struct{} // all available plugins, used to determine which plugin made the client write
|
2016-10-26 10:01:52 +01:00
|
|
|
}
|
|
|
|
|
|
2019-05-18 18:34:46 +01:00
|
|
|
// New returns a new instance of Metrics with the given address.
|
2017-12-13 16:59:10 -05:00
|
|
|
func New(addr string) *Metrics {
|
2017-12-14 13:19:03 -05:00
|
|
|
met := &Metrics{
|
|
|
|
|
Addr: addr,
|
2020-07-25 23:06:28 +08:00
|
|
|
Reg: prometheus.DefaultRegisterer.(*prometheus.Registry),
|
2018-12-10 10:17:15 +00:00
|
|
|
zoneMap: make(map[string]struct{}),
|
Metrics: expand coredns_dns_responses_total with plugin label (#4914)
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* better comment
Signed-off-by: Miek Gieben <miek@miek.nl>
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* Update core/dnsserver/server.go
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
* Use [3]string
Signed-off-by: Miek Gieben <miek@miek.nl>
* imports
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove dnstest changes
Signed-off-by: Miek Gieben <miek@miek.nl>
* revert
Signed-off-by: Miek Gieben <miek@miek.nl>
* Add some sleeps to make it less flaky
Signed-off-by: Miek Gieben <miek@miek.nl>
* Revert "Add some sleeps to make it less flaky"
This reverts commit b5c6655196e3ad570555f086832ceb1f48f6f2d5.
* Remove forward when not needed
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove newline
Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
2021-11-12 16:07:05 +00:00
|
|
|
plugins: pluginList(caddy.ListPlugins()),
|
2017-12-14 13:19:03 -05:00
|
|
|
}
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2017-12-14 13:19:03 -05:00
|
|
|
return met
|
2017-12-13 16:59:10 -05:00
|
|
|
}
|
|
|
|
|
|
2017-12-27 14:14:53 +00:00
|
|
|
// MustRegister wraps m.Reg.MustRegister.
|
2018-11-01 15:56:00 -04:00
|
|
|
func (m *Metrics) MustRegister(c prometheus.Collector) {
|
|
|
|
|
err := m.Reg.Register(c)
|
|
|
|
|
if err != nil {
|
|
|
|
|
// ignore any duplicate error, but fatal on any other kind of error
|
|
|
|
|
if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
|
|
|
|
|
log.Fatalf("Cannot register metrics collector: %s", err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2017-12-27 14:14:53 +00:00
|
|
|
|
2016-10-26 10:01:52 +01:00
|
|
|
// AddZone adds zone z to m.
|
|
|
|
|
func (m *Metrics) AddZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
2018-12-10 10:17:15 +00:00
|
|
|
m.zoneMap[z] = struct{}{}
|
2016-10-26 10:01:52 +01:00
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// RemoveZone remove zone z from m.
|
|
|
|
|
func (m *Metrics) RemoveZone(z string) {
|
|
|
|
|
m.zoneMu.Lock()
|
|
|
|
|
delete(m.zoneMap, z)
|
|
|
|
|
m.zoneNames = keys(m.zoneMap)
|
|
|
|
|
m.zoneMu.Unlock()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ZoneNames returns the zones of m.
|
|
|
|
|
func (m *Metrics) ZoneNames() []string {
|
|
|
|
|
m.zoneMu.RLock()
|
|
|
|
|
s := m.zoneNames
|
|
|
|
|
m.zoneMu.RUnlock()
|
|
|
|
|
return s
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
|
|
|
|
|
2016-09-23 09:14:12 +01:00
|
|
|
// OnStartup sets up the metrics on startup.
|
|
|
|
|
func (m *Metrics) OnStartup() error {
|
2019-12-06 18:55:40 +08:00
|
|
|
ln, err := reuseport.Listen("tcp", m.Addr)
|
2017-02-21 19:34:40 +00:00
|
|
|
if err != nil {
|
2018-04-19 07:41:56 +01:00
|
|
|
log.Errorf("Failed to start metrics handler: %s", err)
|
2017-02-21 19:34:40 +00:00
|
|
|
return err
|
|
|
|
|
}
|
2016-06-25 18:12:13 +01:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.ln = ln
|
2018-04-21 17:43:02 +01:00
|
|
|
m.lnSetup = true
|
2016-03-18 20:57:35 +00:00
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
m.mux = http.NewServeMux()
|
2017-12-14 13:19:03 -05:00
|
|
|
m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{}))
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
// creating some helper variables to avoid data races on m.srv and m.ln
|
2025-08-29 06:03:55 +03:00
|
|
|
server := &http.Server{
|
|
|
|
|
Handler: m.mux,
|
|
|
|
|
ReadTimeout: 5 * time.Second,
|
|
|
|
|
WriteTimeout: 5 * time.Second,
|
|
|
|
|
IdleTimeout: 5 * time.Second,
|
|
|
|
|
}
|
2019-05-18 18:34:46 +01:00
|
|
|
m.srv = server
|
|
|
|
|
|
2017-02-21 19:34:40 +00:00
|
|
|
go func() {
|
2019-05-18 18:34:46 +01:00
|
|
|
server.Serve(ln)
|
2017-02-21 19:34:40 +00:00
|
|
|
}()
|
2019-05-18 18:34:46 +01:00
|
|
|
|
|
|
|
|
ListenAddr = ln.Addr().String() // For tests.
|
2016-03-18 20:57:35 +00:00
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
2018-04-21 17:43:02 +01:00
|
|
|
// OnRestart stops the listener on reload.
|
|
|
|
|
func (m *Metrics) OnRestart() error {
|
|
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
2019-05-18 18:34:46 +01:00
|
|
|
u.Unset(m.Addr)
|
2018-08-21 08:52:25 -07:00
|
|
|
return m.stopServer()
|
|
|
|
|
}
|
2018-04-21 17:43:02 +01:00
|
|
|
|
2018-08-21 08:52:25 -07:00
|
|
|
func (m *Metrics) stopServer() error {
|
|
|
|
|
if !m.lnSetup {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
|
|
|
|
defer cancel()
|
|
|
|
|
if err := m.srv.Shutdown(ctx); err != nil {
|
|
|
|
|
log.Infof("Failed to stop prometheus http server: %s", err)
|
|
|
|
|
return err
|
|
|
|
|
}
|
2018-04-21 17:43:02 +01:00
|
|
|
m.lnSetup = false
|
2018-08-21 08:52:25 -07:00
|
|
|
m.ln.Close()
|
2018-04-21 17:43:02 +01:00
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// OnFinalShutdown tears down the metrics listener on shutdown and restart.
|
2019-05-13 12:26:05 +01:00
|
|
|
func (m *Metrics) OnFinalShutdown() error { return m.stopServer() }
|
2016-04-29 07:28:35 +01:00
|
|
|
|
2018-12-10 10:17:15 +00:00
|
|
|
func keys(m map[string]struct{}) []string {
|
2016-10-26 10:01:52 +01:00
|
|
|
sx := []string{}
|
|
|
|
|
for k := range m {
|
|
|
|
|
sx = append(sx, k)
|
|
|
|
|
}
|
|
|
|
|
return sx
|
2016-03-18 20:57:35 +00:00
|
|
|
}
|
2016-10-28 12:57:10 +01:00
|
|
|
|
Metrics: expand coredns_dns_responses_total with plugin label (#4914)
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* better comment
Signed-off-by: Miek Gieben <miek@miek.nl>
* Metrics: expand coredns_dns_responses_total with plugin label
This adds (somewhat hacky?) code to add a plugin label to the
coredns_dns_responses_total metric. It's completely obvlious to the
plugin as we just check who called the *recorder.WriteMsg method. We use
runtime.Caller( 1 2 3) to get multiple levels of callers, this should be
deep enough, but it depends on the dns.ResponseWriter wrapping that's
occuring.
README.md of metrics updates and test added in test/metrics_test.go to
check for the label being set.
I went through the plugin to see what metrics could be removed, but
actually didn't find any, the plugin push out metrics that make sense.
Due to the path fiddling to figure out the plugin name I doubt this
works (out-of-the-box) for external plugins, but I haven't tested that.
Signed-off-by: Miek Gieben <miek@miek.nl>
* Update core/dnsserver/server.go
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
* Use [3]string
Signed-off-by: Miek Gieben <miek@miek.nl>
* imports
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove dnstest changes
Signed-off-by: Miek Gieben <miek@miek.nl>
* revert
Signed-off-by: Miek Gieben <miek@miek.nl>
* Add some sleeps to make it less flaky
Signed-off-by: Miek Gieben <miek@miek.nl>
* Revert "Add some sleeps to make it less flaky"
This reverts commit b5c6655196e3ad570555f086832ceb1f48f6f2d5.
* Remove forward when not needed
Signed-off-by: Miek Gieben <miek@miek.nl>
* remove newline
Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: dilyevsky <ilyevsky@gmail.com>
2021-11-12 16:07:05 +00:00
|
|
|
// pluginList iterates over the returned plugin map from caddy and removes the "dns." prefix from them.
|
|
|
|
|
func pluginList(m map[string][]string) map[string]struct{} {
|
|
|
|
|
pm := map[string]struct{}{}
|
|
|
|
|
for _, p := range m["others"] {
|
|
|
|
|
// only add 'dns.' plugins
|
|
|
|
|
if len(p) > 3 {
|
|
|
|
|
pm[p[4:]] = struct{}{}
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return pm
|
|
|
|
|
}
|
|
|
|
|
|
2016-10-28 12:57:10 +01:00
|
|
|
// ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where
|
|
|
|
|
// we listen on "localhost:0" and need to retrieve the actual address.
|
|
|
|
|
var ListenAddr string
|
2018-01-23 21:10:55 +01:00
|
|
|
|
2018-08-21 08:52:25 -07:00
|
|
|
// shutdownTimeout is the maximum amount of time the metrics plugin will wait
|
|
|
|
|
// before erroring when it tries to close the metrics server
|
|
|
|
|
const shutdownTimeout time.Duration = time.Second * 5
|
|
|
|
|
|
2020-07-25 23:06:28 +08:00
|
|
|
var buildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
2018-05-05 19:47:41 +02:00
|
|
|
Namespace: plugin.Namespace,
|
|
|
|
|
Name: "build_info",
|
|
|
|
|
Help: "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
|
|
|
|
|
}, []string{"version", "revision", "goversion"})
|