Files
coredns/plugin/shed/setup.go

69 lines
1.8 KiB
Go
Raw Normal View History

plugin/shed: add UDP overload protection plugin (#8312) * plugin/shed: add UDP overload protection plugin UDP responses written back through one listener socket serialize on the Go runtime's internal fdMutex, which allows at most 2^20-1 concurrent operations per file descriptor and panics the process when exceeded. CoreDNS serves UDP with one goroutine per query, all writing through the shared packet connection, so a sustained overload parks every excess in-flight query in that wait queue until the process dies with "too many concurrent operations on a single file or socket". Observed in production: ~2.8M goroutines and 60GiB RSS before the panic. The shed plugin makes the panic structurally unreachable. It installs, via Config.UDPDecorateWriterFunc, a per-socket bounded evict-oldest stack drained newest-first by a single writer goroutine, so the fd never sees more than one writer and residual capacity under overload always goes to the freshest response. While a socket's stack is full, arriving queries are dropped before any plugin runs. Drops are silent (the client's resolver retries elsewhere) and counted in coredns_shed_dropped_total{server, reason}. plugin/shed/fdmutex_test.go demonstrates the failure and the fix with one shared flood harness. Two subprocess tests reproduce the exact runtime panic without the plugin's write discipline - one deterministic (a held write plus >2^20 queued writers), one with nothing held or mocked; both exercise the Go runtime rather than the plugin, so they are gated behind SHED_FLOOD_TEST=1. The counterfactual - the same load through the plugin's stack, completing with every response accounted for as written or dropped - runs in every test invocation, including -race, at 50k responders, and at the full 1.5M with SHED_FLOOD_TEST=1: SHED_FLOOD_TEST=1 go test ./plugin/shed/ Signed-off-by: Ryan Brewster <rpb@anthropic.com> * test: add shed e2e test Query a shed-enabled server over UDP (the plugin's deferred single-writer path) and TCP (which shed passes through), and check that coredns_shed_dropped_total is exported with its reason label. No-Verification-Needed: test-only change Signed-off-by: Ryan Brewster <rpb@anthropic.com> --------- Signed-off-by: Ryan Brewster <rpb@anthropic.com>
2026-07-27 05:13:25 -04:00
package shed
import (
"github.com/coredns/caddy"
"github.com/coredns/coredns/core/dnsserver"
"github.com/coredns/coredns/plugin"
clog "github.com/coredns/coredns/plugin/pkg/log"
pkgparse "github.com/coredns/coredns/plugin/pkg/parse"
"github.com/coredns/coredns/plugin/pkg/transport"
)
const pluginName = "shed"
var log = clog.NewWithPlugin(pluginName)
// stackDepth is a burst budget, not a knob: ~12-16ms at a typical socket's
// serialized drain rate; a larger value would only hold staler responses.
const stackDepth = 1024
func init() { plugin.Register(pluginName, setup) }
func setup(c *caddy.Controller) error {
s, err := parse(c)
if err != nil {
return plugin.Error(pluginName, err)
}
// The hook only exists on the plain-DNS UDP path; on any other transport
// shed would silently protect nothing, so refuse at parse time. Every
// block key is checked: caddy propagates the plugin list to all keys of
// a server block, not just the one setup runs for.
for _, key := range c.ServerBlockKeys {
if tr, _ := pkgparse.Transport(key); tr != transport.DNS {
return plugin.Error(pluginName, c.Errf("only plain DNS server blocks are supported; %q uses transport %q", key, tr))
}
}
cfg := dnsserver.GetConfig(c)
cfg.UDPDecorateWriterFunc = s.decorateWriterFactory
// On a graceful reload the retiring instance must stop its writer
// goroutines; shutdown removes only this instance's sockets.
c.OnShutdown(s.shutdown)
cfg.AddPlugin(func(next plugin.Handler) plugin.Handler {
s.Next = next
return s
})
return nil
}
func parse(c *caddy.Controller) (*Shed, error) {
s := &Shed{}
i := 0
for c.Next() {
if i > 0 {
return nil, plugin.ErrOnce
}
i++
if len(c.RemainingArgs()) != 0 {
return nil, c.ArgErr()
}
if c.NextBlock() {
return nil, c.Errf("shed takes no options")
}
}
return s, nil
}