From 7ff001dca7ca6dcdef1cddb4cbba2c4431aa933b Mon Sep 17 00:00:00 2001 From: Peppi-Lotta <73192628+peppi-lotta@users.noreply.github.com> Date: Thu, 12 Mar 2026 22:49:00 +0200 Subject: [PATCH] Add optional TLS support to /metrics endpoint (#7255) * Use exporter-toolkit to enable optional TLS encryption on /metrics endpoint Signed-off-by: peppi-lotta * Implement startup listener to signal server readiness Signed-off-by: peppi-lotta --------- Signed-off-by: peppi-lotta --- go.mod | 8 +- go.sum | 17 +- plugin/metrics/metrics.go | 82 +++- plugin/metrics/metrics_test.go | 391 ++++++++++++++++++ plugin/metrics/setup.go | 23 +- .../test_data/configs/certPath_empty.yml | 3 + .../test_data/configs/certPath_invalid.yml | 3 + plugin/metrics/test_data/configs/empty.yml | 0 plugin/metrics/test_data/configs/junk.yml | 20 + .../test_data/configs/keyPath_empty.yml | 3 + .../test_data/configs/keyPath_invalid.yml | 3 + .../configs/valid_requireanyclientcert.yml | 4 + .../configs/valid_verifyclientcertifgiven.yml | 4 + 13 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 plugin/metrics/test_data/configs/certPath_empty.yml create mode 100644 plugin/metrics/test_data/configs/certPath_invalid.yml create mode 100644 plugin/metrics/test_data/configs/empty.yml create mode 100644 plugin/metrics/test_data/configs/junk.yml create mode 100644 plugin/metrics/test_data/configs/keyPath_empty.yml create mode 100644 plugin/metrics/test_data/configs/keyPath_invalid.yml create mode 100644 plugin/metrics/test_data/configs/valid_requireanyclientcert.yml create mode 100644 plugin/metrics/test_data/configs/valid_verifyclientcertifgiven.yml diff --git a/go.mod b/go.mod index 25e653961..1f687900d 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( require ( github.com/pires/go-proxyproto v0.11.0 + github.com/prometheus/exporter-toolkit v0.15.1 golang.org/x/net v0.51.0 ) @@ -97,7 +98,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -113,6 +114,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect @@ -129,6 +131,7 @@ require ( github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect @@ -136,12 +139,15 @@ require ( github.com/linkdata/deadlock v0.5.5 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mdlayher/socket v0.4.1 // indirect + github.com/mdlayher/vsock v1.2.1 // indirect github.com/minio/simdjson-go v0.4.5 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect github.com/oschwald/maxminddb-golang/v2 v2.1.1 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect diff --git a/go.sum b/go.sum index 7a953fb62..f7ccbf9cb 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/coredns/caddy v1.1.4-0.20250930002214-15135a999495 h1:JFeOmbjLnVRhvmL github.com/coredns/caddy v1.1.4-0.20250930002214-15135a999495/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -167,7 +167,6 @@ github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -175,6 +174,8 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= @@ -232,6 +233,8 @@ github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9 h1:w66aaP3c6 github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9/go.mod h1:BaIJzjD2ZnHmx2acPF6XfGLPzNCMiBbMRqJr+8/8uRI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -257,6 +260,10 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= +github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= @@ -274,6 +281,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= @@ -319,6 +328,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/exporter-toolkit v0.15.1 h1:XrGGr/qWl8Gd+pqJqTkNLww9eG8vR/CoRk0FubOKfLE= +github.com/prometheus/exporter-toolkit v0.15.1/go.mod h1:P/NR9qFRGbCFgpklyhix9F6v6fFr/VQB/CVsrMDGKo4= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= diff --git a/plugin/metrics/metrics.go b/plugin/metrics/metrics.go index 9ebb4a5aa..d8c16de26 100644 --- a/plugin/metrics/metrics.go +++ b/plugin/metrics/metrics.go @@ -3,18 +3,22 @@ package metrics import ( "context" + "log/slog" "net" "net/http" + "os" "sync" "time" "github.com/coredns/caddy" "github.com/coredns/coredns/plugin" + "github.com/coredns/coredns/plugin/pkg/log" "github.com/coredns/coredns/plugin/pkg/reuseport" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/exporter-toolkit/web" ) // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics . @@ -34,6 +38,8 @@ type Metrics struct { zoneMu sync.RWMutex plugins map[string]struct{} // all available plugins, used to determine which plugin made the client write + + tlsConfigPath string } // New returns a new instance of Metrics with the given address. @@ -83,6 +89,32 @@ func (m *Metrics) ZoneNames() []string { return s } +// startupListener wraps a net.Listener to detect when Accept() is first called +type startupListener struct { + net.Listener + readyOnce sync.Once + ready chan struct{} +} + +func newStartupListener(l net.Listener) *startupListener { + return &startupListener{ + Listener: l, + ready: make(chan struct{}), + } +} + +func (sl *startupListener) Accept() (net.Conn, error) { + // Signal ready on first Accept() call (server is running) + sl.readyOnce.Do(func() { + close(sl.ready) + }) + return sl.Listener.Accept() +} + +func (sl *startupListener) Ready() <-chan struct{} { + return sl.ready +} + // OnStartup sets up the metrics on startup. func (m *Metrics) OnStartup() error { ln, err := reuseport.Listen("tcp", m.Addr) @@ -91,7 +123,9 @@ func (m *Metrics) OnStartup() error { return err } - m.ln = ln + startupListener := newStartupListener(ln) + + m.ln = startupListener m.lnSetup = true m.mux = http.NewServeMux() @@ -99,6 +133,7 @@ func (m *Metrics) OnStartup() error { // creating some helper variables to avoid data races on m.srv and m.ln server := &http.Server{ + Addr: m.Addr, Handler: m.mux, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, @@ -106,10 +141,53 @@ func (m *Metrics) OnStartup() error { } m.srv = server + if m.tlsConfigPath == "" { + go func() { + if err := server.Serve(ln); err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start HTTP metrics server: %s", err) + } + }() + ListenAddr = ln.Addr().String() // For tests. + return nil + } + + // Check TLS config file existence + if _, err := os.Stat(m.tlsConfigPath); os.IsNotExist(err) { + log.Errorf("TLS config file does not exist: %s", m.tlsConfigPath) + return err + } + + // Create web config for ListenAndServe + webConfig := &web.FlagConfig{ + WebListenAddresses: &[]string{m.Addr}, + WebSystemdSocket: new(bool), // false by default + WebConfigFile: &m.tlsConfigPath, + } + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + // Create channels for synchronization + startUpErr := make(chan error, 1) + go func() { - server.Serve(ln) + // Try to start the server and report result if there an error. + // web.Serve() never returns nil, it always returns a non-nil error and + // it doesn't retun anything if server starts successfully. + // startupListener handles capturing succesful startup. + err := web.Serve(m.ln, server, webConfig, logger) + if err != nil && err != http.ErrServerClosed { + log.Errorf("Failed to start HTTPS metrics server: %v", err) + startUpErr <- err + } }() + // Wait for startup errors + select { + case err := <-startUpErr: + return err + case <-startupListener.Ready(): + log.Infof("Server is ready and accepting connections") + } + ListenAddr = ln.Addr().String() // For tests. return nil } diff --git a/plugin/metrics/metrics_test.go b/plugin/metrics/metrics_test.go index 6861e3606..8aaf55636 100644 --- a/plugin/metrics/metrics_test.go +++ b/plugin/metrics/metrics_test.go @@ -2,9 +2,18 @@ package metrics import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" "io" + "math/big" "net" "net/http" + "os" "testing" "time" @@ -17,6 +26,388 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) +const ( + serverCertFile = "test_data/server.crt" + serverKeyFile = "test_data/server.key" + clientCertFile = "test_data/client_selfsigned.crt" + clientKeyFile = "test_data/client_selfsigned.key" + tlsCaChainFile = "test_data/tls-ca-chain.pem" +) + +func createTestCertFiles(t *testing.T) error { + t.Helper() + // Generate CA certificate + caCert, caKey, err := generateCA() + if err != nil { + t.Fatalf("Failed to generate CA certificate: %v", err) + return err + } + + // Generate server certificate signed by CA + cert, key, err := generateCert(caCert, caKey) + if err != nil { + t.Fatalf("Failed to generate server certificate: %v", err) + return err + } + + // Generate client CA certificate + clientCaCert, clientCaKey, err := generateCA() + if err != nil { + t.Fatalf("Failed to generate client CA certificate: %v", err) + return err + } + + // Generate client certificate signed by CA + clientCert, clientKey, err := generateCert(clientCaCert, clientCaKey) + if err != nil { + t.Fatalf("Failed to generate client certificate: %v", err) + return err + } + + // Create ca chain file + caChain := append(caCert, clientCaCert...) + + // Write certificates to temporary files + err = writeFile(t, string(cert), serverCertFile) + if err != nil { + t.Fatalf("Failed to write server certificate: %v", err) + return err + } + err = writeFile(t, string(key), serverKeyFile) + if err != nil { + t.Fatalf("Failed to write server key: %v", err) + return err + } + err = writeFile(t, string(clientCert), clientCertFile) + if err != nil { + t.Fatalf("Failed to write client certificate: %v", err) + return err + } + err = writeFile(t, string(clientKey), clientKeyFile) + if err != nil { + t.Fatalf("Failed to write client key: %v", err) + return err + } + err = writeFile(t, string(caChain), tlsCaChainFile) + if err != nil { + t.Fatalf("Failed to write CA certificate: %v", err) + return err + } + + return nil +} + +func generateCA() ([]byte, []byte, error) { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2023), + Subject: pkix.Name{ + Organization: []string{"Test CA"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return nil, nil, err + } + + caPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + + caPrivKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey), + }) + + return caPEM, caPrivKeyPEM, nil +} + +func generateCert(caCertPEM, caKeyPEM []byte) ([]byte, []byte, error) { + caCertBlock, _ := pem.Decode(caCertPEM) + caCert, err := x509.ParseCertificate(caCertBlock.Bytes) + if err != nil { + return nil, nil, err + } + + caKeyBlock, _ := pem.Decode(caKeyPEM) + caKey, err := x509.ParsePKCS1PrivateKey(caKeyBlock.Bytes) + if err != nil { + return nil, nil, err + } + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(2023), + Subject: pkix.Name{ + Organization: []string{"Test Server"}, + }, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &certPrivKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + + return certPEM, certPrivKeyPEM, nil +} + +func cleanupTestCertFiles() { + os.Remove(serverCertFile) + os.Remove(serverKeyFile) + os.Remove(clientCertFile) + os.Remove(clientKeyFile) + os.Remove(tlsCaChainFile) +} + +func writeFile(t *testing.T, content, path string) error { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + return err + } + return nil +} + +func getTLSClient(clientCertName bool) *http.Client { + cert, err := os.ReadFile(tlsCaChainFile) + if err != nil { + panic("Unable to start TLS client. Check cert path") + } + + var clientCertficate tls.Certificate + if clientCertName { + clientCertficate, err = tls.LoadX509KeyPair( + clientCertFile, + clientKeyFile, + ) + if err != nil { + panic(fmt.Sprintf("failed to load client certificate: %v", err)) + } + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: func() *x509.CertPool { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(cert) + return caCertPool + }(), + GetClientCertificate: func(req *tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &clientCertficate, nil + }, + }, + }, + } + return client +} +func TestMetricsTLS(t *testing.T) { + err := createTestCertFiles(t) + if err != nil { + t.Fatalf("Failed to create test certificate files: %v", err) + } + defer cleanupTestCertFiles() + + tests := []struct { + name string + tlsConfigPath string + UseTLSClient bool + clientCertificate bool + caFile string + expectStartupError bool + expectRequestError bool + }{ + { + name: "No TLS config: starts a HTTP server, connect successfully with default client", + tlsConfigPath: "", + }, + { + name: "No TLS config: starts HTTP server, connection fails with TLS client", + tlsConfigPath: "", + UseTLSClient: true, + expectRequestError: true, + }, + { + name: "Empty TLS config: starts a HTTP server", + tlsConfigPath: "test_data/configs/empty.yml", + }, + { + name: "Valid TLS config, no client cert, successful connection with TLS client", + tlsConfigPath: "test_data/configs/valid_verifyclientcertifgiven.yml", + UseTLSClient: true, + }, + { + name: `Valid TLS config, connection fails with default client`, + tlsConfigPath: "test_data/configs/valid_verifyclientcertifgiven.yml", + expectRequestError: true, + }, + { + name: `Valid TLS config with RequireAnyClientCert, connection succeeds with TLS client presenting (valid) certificate`, + tlsConfigPath: "test_data/configs/valid_requireanyclientcert.yml", + UseTLSClient: true, + clientCertificate: true, + }, + { + name: "Wrong path to TLS config file fails to start server", + tlsConfigPath: "test_data/configs/this-does-not-exist.yml", + UseTLSClient: true, + expectStartupError: true, + }, + { + name: `TLS config hasinvalid structure, fails to start server`, + tlsConfigPath: "test_data/configs/junk.yml", + UseTLSClient: true, + expectStartupError: true, + }, + { + name: "Missing key file, fails to start server", + tlsConfigPath: "test_data/configs/keyPath_empty.yml", + UseTLSClient: true, + expectStartupError: true, + }, + { + name: "Missing cert file, fails to start server", + tlsConfigPath: "test_data/configs/certPath_empty.yml", + UseTLSClient: true, + expectStartupError: true, + }, + { + name: "Wrong key file path, fails to start server", + tlsConfigPath: "test_data/configs/keyPath_invalid.yml", + UseTLSClient: true, + expectStartupError: true, + }, + { + name: "Wrong cert file path, fails to start server", + tlsConfigPath: "test_data/configs/certPath_invalid.yml", + UseTLSClient: true, + expectStartupError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + met := New("localhost:0") + met.tlsConfigPath = tt.tlsConfigPath + + // Start server + err := met.OnStartup() + if tt.expectStartupError { + if err == nil { + t.Error("Expected error but got none") + } + return + } + if err != nil { + t.Fatalf("Failed to start metrics handler: %s", err) + } + defer met.OnFinalShutdown() + + // Wait for server to be ready + select { + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for server to start") + case <-func() chan struct{} { + ch := make(chan struct{}) + go func() { + for { + conn, err := net.DialTimeout("tcp", ListenAddr, 100*time.Millisecond) + if err == nil { + conn.Close() + close(ch) + return + } + time.Sleep(100 * time.Millisecond) + } + }() + return ch + }(): + } + + // Create appropriate client and protocol + var client *http.Client + var protocol string + if tt.UseTLSClient { + client = getTLSClient(tt.clientCertificate) + protocol = "https" + } else { + client = http.DefaultClient + protocol = "http" + } + + // Try multiple times to account for server startup time + var resp *http.Response + var err2 error + for i := range 10 { + url := fmt.Sprintf("%s://%s/metrics", protocol, ListenAddr) + t.Logf("Attempt %d: Connecting to %s", i+1, url) + resp, err2 = client.Get(url) + if err2 == nil { + t.Logf("Successfully connected to metrics server") + break + } + t.Logf("Connection attempt failed: %v", err2) + time.Sleep(200 * time.Millisecond) + } + if err2 != nil { + if tt.expectRequestError { + return + } + t.Fatalf("Failed to connect to metrics server: %v", err2) + } + if resp != nil { + defer resp.Body.Close() + } + + if tt.expectRequestError { + // If we expect a request error but got a response, check if it's a bad status code + // which indicates the connection succeeded but the request was invalid (e.g., HTTP to HTTPS server) + if resp.StatusCode == http.StatusBadRequest { + // Got expected error response + return + } + // Got unexpected response status + t.Fatalf("Expected request error with status %d but got response with status %d", http.StatusBadRequest, resp.StatusCode) + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + }) + } +} + func TestMetrics(t *testing.T) { met := New("localhost:0") if err := met.OnStartup(); err != nil { diff --git a/plugin/metrics/setup.go b/plugin/metrics/setup.go index bee7d1f49..a9d683061 100644 --- a/plugin/metrics/setup.go +++ b/plugin/metrics/setup.go @@ -9,12 +9,10 @@ import ( "github.com/coredns/coredns/coremain" "github.com/coredns/coredns/plugin" "github.com/coredns/coredns/plugin/metrics/vars" - clog "github.com/coredns/coredns/plugin/pkg/log" "github.com/coredns/coredns/plugin/pkg/uniq" ) var ( - log = clog.NewWithPlugin("prometheus") u = uniq.New() registry = newReg() ) @@ -97,6 +95,27 @@ func parse(c *caddy.Controller) (*Metrics, error) { default: return met, c.ArgErr() } + + // Parse TLS block if present + for c.NextBlock() { + switch c.Val() { + case "tls": + if met.tlsConfigPath != "" { + return nil, c.Err("tls block already specified") + } + + // Get cert and key files as positional arguments + args := c.RemainingArgs() + if len(args) != 1 { + return nil, c.ArgErr() + } + tlsCfgPath := args[0] + + met.tlsConfigPath = tlsCfgPath + default: + return nil, c.Errf("unknown option: %s", c.Val()) + } + } } return met, nil } diff --git a/plugin/metrics/test_data/configs/certPath_empty.yml b/plugin/metrics/test_data/configs/certPath_empty.yml new file mode 100644 index 000000000..5f4d45d95 --- /dev/null +++ b/plugin/metrics/test_data/configs/certPath_empty.yml @@ -0,0 +1,3 @@ +tls_server_config: + cert_file: "" + key_file: "../server.key" diff --git a/plugin/metrics/test_data/configs/certPath_invalid.yml b/plugin/metrics/test_data/configs/certPath_invalid.yml new file mode 100644 index 000000000..59705ded5 --- /dev/null +++ b/plugin/metrics/test_data/configs/certPath_invalid.yml @@ -0,0 +1,3 @@ +tls_server_config: + cert_file: "somefile" + key_file: "../server.key" diff --git a/plugin/metrics/test_data/configs/empty.yml b/plugin/metrics/test_data/configs/empty.yml new file mode 100644 index 000000000..e69de29bb diff --git a/plugin/metrics/test_data/configs/junk.yml b/plugin/metrics/test_data/configs/junk.yml new file mode 100644 index 000000000..568a7c404 --- /dev/null +++ b/plugin/metrics/test_data/configs/junk.yml @@ -0,0 +1,20 @@ +hWkNKCp3fvIx3jKnsaBI +TuEjdwNS8A2vYdFbiKqr +ay3RiOtykgt4m6m3KOol +ZreGpJRGmpDSVV9cioiF +r7kDOHhHU2frvv0nLcY2 +uQMQM4XgqFkCG6gFAIJZ +g99tTkrZhN9b6pkJ6J2y +rzdt729HrA2RblDGYfjs +MW7GxrBdlCnliYJGPhfr +g9kaXxMXcDwsw0C0rv0u +637ZmfRGElb6VBVOtgqn +RG0MRezjLYCJQBMUdRDE +RzO4VicAzj7asVZAT3oo +nPw267UONk7h7KBYRgch +Alj38foWqjV3heXXdahm +TrMzMgl6JIQ1x4OZB5i4 +qlrXFJoeV6Pr77nuiEh9 +3yE5vMnnKHm2nImEfzMG +bI01UDObHRSaoJLC0vTD +G9tlcKU883NkQ6nsxJ8Y diff --git a/plugin/metrics/test_data/configs/keyPath_empty.yml b/plugin/metrics/test_data/configs/keyPath_empty.yml new file mode 100644 index 000000000..a91d086ec --- /dev/null +++ b/plugin/metrics/test_data/configs/keyPath_empty.yml @@ -0,0 +1,3 @@ +tls_server_config: + cert_file: "../server.crt" + key_file: "" diff --git a/plugin/metrics/test_data/configs/keyPath_invalid.yml b/plugin/metrics/test_data/configs/keyPath_invalid.yml new file mode 100644 index 000000000..f2c28408d --- /dev/null +++ b/plugin/metrics/test_data/configs/keyPath_invalid.yml @@ -0,0 +1,3 @@ +tls_server_config: + cert_file: "../server.cert" + key_file: "somefile" diff --git a/plugin/metrics/test_data/configs/valid_requireanyclientcert.yml b/plugin/metrics/test_data/configs/valid_requireanyclientcert.yml new file mode 100644 index 000000000..88097b100 --- /dev/null +++ b/plugin/metrics/test_data/configs/valid_requireanyclientcert.yml @@ -0,0 +1,4 @@ +tls_server_config: + cert_file: "../server.crt" + key_file: "../server.key" + client_auth_type: "RequireAnyClientCert" diff --git a/plugin/metrics/test_data/configs/valid_verifyclientcertifgiven.yml b/plugin/metrics/test_data/configs/valid_verifyclientcertifgiven.yml new file mode 100644 index 000000000..35de280ef --- /dev/null +++ b/plugin/metrics/test_data/configs/valid_verifyclientcertifgiven.yml @@ -0,0 +1,4 @@ +tls_server_config: + cert_file: "../server.crt" + key_file: "../server.key" + client_auth_type: "VerifyClientCertIfGiven"