Make CoreDNS a server type plugin for Caddy (#220)

* Make CoreDNS a server type plugin for Caddy

Remove code we don't need and port all middleware over. Fix all tests
and rework the documentation.

Also make `go generate` build a caddy binary which we then copy into
our directory. This means `go build`-builds remain working as-is.

And new etc instances in each etcd test for better isolation.
Fix more tests and rework test.Server with the newer support Caddy offers.

Fix Makefile to support new mode of operation.

core/dnsserver/address.go

@@ -0,0 +1,44 @@
+package dnsserver
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+type zoneAddr struct {
+	Zone string
+	Port string
+}
+
+// String return z.Zone + ":" + z.Port as a string.
+func (z zoneAddr) String() string { return z.Zone + ":" + z.Port }
+
+// normalizeZone parses an zone string into a structured format with separate
+// host, and port portions, as well as the original input string.
+func normalizeZone(str string) (zoneAddr, error) {
+	var err error
+
+	// separate host and port
+	host, port, err := net.SplitHostPort(str)
+	if err != nil {
+		host, port, err = net.SplitHostPort(str + ":")
+		// no error check here; return err at end of function
+	}
+
+	if len(host) > 255 {
+		return zoneAddr{}, fmt.Errorf("specified zone is too long: %d > 255", len(host))
+	}
+	_, d := dns.IsDomainName(host)
+	if !d {
+		return zoneAddr{}, fmt.Errorf("zone is not a valid domain name: %s", host)
+	}
+
+	if port == "" {
+		port = "53"
+	}
+
+	return zoneAddr{Zone: strings.ToLower(dns.Fqdn(host)), Port: port}, err
+}

core/dnsserver/config.go

@@ -0,0 +1,38 @@
+package dnsserver
+
+import "github.com/mholt/caddy"
+
+// Config configuration for a single server.
+type Config struct {
+	// The zone of the site.
+	Zone string
+
+	// The hostname to bind listener to, defaults to the wildcard address
+	ListenHost string
+
+	// The port to listen on.
+	Port string
+
+	// The directory from which to parse db files, and store keys.
+	Root string
+
+	// Middleware stack.
+	Middleware []Middleware
+
+	// Compiled middleware stack.
+	middlewareChain Handler
+}
+
+// GetConfig gets the Config that corresponds to c.
+// If none exist nil is returned.
+func GetConfig(c *caddy.Controller) *Config {
+	ctx := c.Context().(*dnsContext)
+	if cfg, ok := ctx.keysToConfigs[c.Key]; ok {
+		return cfg
+	}
+	// we should only get here during tests because directive
+	// actions typically skip the server blocks where we make
+	// the configs.
+	ctx.saveConfig(c.Key, &Config{Root: Root})
+	return GetConfig(c)
+}

core/dnsserver/directives.go

@@ -0,0 +1,32 @@
+package dnsserver
+
+// Add here, and in core/coredns.go to use them.
+
+// Directives are registered in the order they should be
+// executed.
+//
+// Ordering is VERY important. Every middleware will
+// feel the effects of all other middleware below
+// (after) them during a request, but they must not
+// care what middleware above them are doing.
+var Directives = []string{
+	"bind",
+	"health",
+	"pprof",
+
+	"prometheus",
+	"errors",
+	"log",
+	"chaos",
+	"cache",
+
+	"rewrite",
+	"loadbalance",
+
+	"dnssec",
+	"file",
+	"secondary",
+	"etcd",
+	"kubernetes",
+	"proxy",
+}

core/dnsserver/middleware.go

@@ -0,0 +1,52 @@
+package dnsserver
+
+import (
+	"github.com/miekg/dns"
+	"golang.org/x/net/context"
+)
+
+type (
+	// Middleware is the middle layer which represents the traditional
+	// idea of middleware: it chains one Handler to the next by being
+	// passed the next Handler in the chain.
+	Middleware func(Handler) Handler
+
+	// Handler is like dns.Handler except ServeDNS may return an rcode
+	// and/or error.
+	//
+	// If ServeDNS writes to the response body, it should return a status
+	// code. If the status code is not one of the following:
+	// * SERVFAIL (dns.RcodeServerFailure)
+	// * REFUSED (dns.RecodeRefused)
+	// * FORMERR (dns.RcodeFormatError)
+	// * NOTIMP (dns.RcodeNotImplemented)
+	//
+	// CoreDNS assumes *no* reply has yet been written. All other response
+	// codes signal other handlers above it that the response message is
+	// already written, and that they should not write to it also.
+	//
+	// If ServeDNS encounters an error, it should return the error value
+	// so it can be logged by designated error-handling middleware.
+	//
+	// If writing a response after calling another ServeDNS method, the
+	// returned rcode SHOULD be used when writing the response.
+	//
+	// If handling errors after calling another ServeDNS method, the
+	// returned error value SHOULD be logged or handled accordingly.
+	//
+	// Otherwise, return values should be propagated down the middleware
+	// chain by returning them unchanged.
+	Handler interface {
+		ServeDNS(context.Context, dns.ResponseWriter, *dns.Msg) (int, error)
+	}
+
+	// HandlerFunc is a convenience type like dns.HandlerFunc, except
+	// ServeDNS returns an rcode and an error. See Handler
+	// documentation for more information.
+	HandlerFunc func(context.Context, dns.ResponseWriter, *dns.Msg) (int, error)
+)
+
+// ServeDNS implements the Handler interface.
+func (f HandlerFunc) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	return f(ctx, w, r)
+}

core/dnsserver/register.go

@@ -0,0 +1,156 @@
+package dnsserver
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/mholt/caddy"
+	"github.com/mholt/caddy/caddyfile"
+)
+
+const serverType = "dns"
+
+func init() {
+	caddy.RegisterServerType(serverType, caddy.ServerType{
+		Directives: Directives,
+		DefaultInput: func() caddy.Input {
+			if Port == DefaultPort && Zone != "" {
+				return caddy.CaddyfileInput{
+					Filepath:       "Corefile",
+					Contents:       nil,
+					ServerTypeName: serverType,
+				}
+			}
+			return caddy.CaddyfileInput{
+				Filepath:       "Corefile",
+				Contents:       nil,
+				ServerTypeName: serverType,
+			}
+		},
+		NewContext: newContext,
+	})
+}
+
+var TestNewContext = newContext
+
+func newContext() caddy.Context {
+	return &dnsContext{keysToConfigs: make(map[string]*Config)}
+}
+
+type dnsContext struct {
+	keysToConfigs map[string]*Config
+
+	// configs is the master list of all site configs.
+	configs []*Config
+}
+
+func (h *dnsContext) saveConfig(key string, cfg *Config) {
+	h.configs = append(h.configs, cfg)
+	h.keysToConfigs[key] = cfg
+}
+
+// InspectServerBlocks make sure that everything checks out before
+// executing directives and otherwise prepares the directives to
+// be parsed and executed.
+func (h *dnsContext) InspectServerBlocks(sourceFile string, serverBlocks []caddyfile.ServerBlock) ([]caddyfile.ServerBlock, error) {
+	// Normalize and check all the zone names and check for duplicates
+	dups := map[string]string{}
+	for _, s := range serverBlocks {
+		for i, k := range s.Keys {
+			za, err := normalizeZone(k)
+			if err != nil {
+				return nil, err
+			}
+			s.Keys[i] = za.String()
+			if v, ok := dups[za.Zone]; ok {
+				return nil, fmt.Errorf("cannot serve %s - zone already defined for %v", za, v)
+
+			}
+			dups[za.Zone] = za.String()
+
+			// Save the config to our master list, and key it for lookups
+			cfg := &Config{
+				Zone: za.Zone,
+				Port: za.Port,
+				// TODO(miek): more?
+			}
+			h.saveConfig(za.String(), cfg)
+		}
+	}
+	return serverBlocks, nil
+}
+
+// MakeServers uses the newly-created siteConfigs to create and return a list of server instances.
+func (h *dnsContext) MakeServers() ([]caddy.Server, error) {
+
+	// we must map (group) each config to a bind address
+	groups, err := groupConfigsByListenAddr(h.configs)
+	if err != nil {
+		return nil, err
+	}
+	// then we create a server for each group
+	var servers []caddy.Server
+	for addr, group := range groups {
+		s, err := NewServer(addr, group)
+		if err != nil {
+			return nil, err
+		}
+		servers = append(servers, s)
+	}
+
+	return servers, nil
+}
+
+// AddMiddleware adds a middleware to a site's middleware stack.
+func (sc *Config) AddMiddleware(m Middleware) {
+	sc.Middleware = append(sc.Middleware, m)
+}
+
+// groupSiteConfigsByListenAddr groups site configs by their listen
+// (bind) address, so sites that use the same listener can be served
+// on the same server instance. The return value maps the listen
+// address (what you pass into net.Listen) to the list of site configs.
+// This function does NOT vet the configs to ensure they are compatible.
+func groupConfigsByListenAddr(configs []*Config) (map[string][]*Config, error) {
+	groups := make(map[string][]*Config)
+
+	for _, conf := range configs {
+		if conf.Port == "" {
+			conf.Port = Port
+		}
+		addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(conf.ListenHost, conf.Port))
+		if err != nil {
+			return nil, err
+		}
+		addrstr := addr.String()
+		groups[addrstr] = append(groups[addrstr], conf)
+	}
+
+	return groups, nil
+}
+
+const (
+	// DefaultZone is the default zone.
+	DefaultZone = "."
+	// DefaultPort is the default port.
+	DefaultPort = "2053"
+	// DefaultRoot is the default root folder.
+	DefaultRoot = "."
+)
+
+// These "soft defaults" are configurable by
+// command line flags, etc.
+var (
+	// Root is the site root
+	Root = DefaultRoot
+
+	// Host is the site host
+	Zone = DefaultZone
+
+	// Port is the site port
+	Port = DefaultPort
+
+	// GracefulTimeout is the maximum duration of a graceful shutdown.
+	GracefulTimeout time.Duration
+)

core/dnsserver/server.go

@@ -0,0 +1,254 @@
+package dnsserver
+
+import (
+	"log"
+	"net"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/miekg/coredns/middleware"
+
+	"github.com/miekg/dns"
+	"golang.org/x/net/context"
+)
+
+// Server represents an instance of a server, which serves
+// DNS requests at a particular address (host and port). A
+// server is capable of serving numerous zones on
+// the same address and the listener may be stopped for
+// graceful termination (POSIX only).
+type Server struct {
+	Addr   string // Address we listen on
+	mux    *dns.ServeMux
+	server [2]*dns.Server // 0 is a net.Listener, 1 is a net.PacketConn (a *UDPConn) in our case.
+
+	l net.Listener
+	p net.PacketConn
+	m sync.Mutex // protects listener and packetconn
+
+	zones       map[string]*Config // zones keyed by their address
+	dnsWg       sync.WaitGroup     // used to wait on outstanding connections
+	connTimeout time.Duration      // the maximum duration of a graceful shutdown
+}
+
+func NewServer(addr string, group []*Config) (*Server, error) {
+
+	s := &Server{
+		Addr:        addr,
+		zones:       make(map[string]*Config),
+		connTimeout: 5 * time.Second, // TODO(miek): was configurable
+	}
+	mux := dns.NewServeMux()
+	mux.Handle(".", s) // wildcard handler, everything will go through here
+	s.mux = mux
+
+	// We have to bound our wg with one increment
+	// to prevent a "race condition" that is hard-coded
+	// into sync.WaitGroup.Wait() - basically, an add
+	// with a positive delta must be guaranteed to
+	// occur before Wait() is called on the wg.
+	// In a way, this kind of acts as a safety barrier.
+	s.dnsWg.Add(1)
+
+	for _, site := range group {
+		// set the config per zone
+		s.zones[site.Zone] = site
+		// compile custom middleware for everything
+		var stack Handler
+		for i := len(site.Middleware) - 1; i >= 0; i-- {
+			stack = site.Middleware[i](stack)
+		}
+		site.middlewareChain = stack
+	}
+
+	return s, nil
+}
+
+// LocalAddr return the addresses where the server is bound to.
+func (s *Server) LocalAddr() net.Addr {
+	s.m.Lock()
+	defer s.m.Unlock()
+	return s.l.Addr()
+}
+
+// LocalAddrPacket return the net.PacketConn address where the server is bound to.
+func (s *Server) LocalAddrPacket() net.Addr {
+	s.m.Lock()
+	defer s.m.Unlock()
+	return s.p.LocalAddr()
+}
+
+// Serve starts the server with an existing listener. It blocks until the server stops.
+func (s *Server) Serve(l net.Listener) error {
+	s.m.Lock()
+	s.server[tcp] = &dns.Server{Listener: l, Net: "tcp", Handler: s.mux}
+	s.m.Unlock()
+
+	return s.server[tcp].ActivateAndServe()
+}
+
+// ServePacket starts the server with an existing packetconn. It blocks until the server stops.
+func (s *Server) ServePacket(p net.PacketConn) error {
+	s.m.Lock()
+	s.server[udp] = &dns.Server{PacketConn: p, Net: "udp", Handler: s.mux}
+	s.m.Unlock()
+
+	return s.server[udp].ActivateAndServe()
+}
+
+func (s *Server) Listen() (net.Listener, error) {
+	l, err := net.Listen("tcp", s.Addr)
+	if err != nil {
+		return nil, err
+	}
+	s.m.Lock()
+	s.l = l
+	s.m.Unlock()
+	return l, nil
+}
+
+func (s *Server) ListenPacket() (net.PacketConn, error) {
+	p, err := net.ListenPacket("udp", s.Addr)
+	if err != nil {
+		return nil, err
+	}
+
+	s.m.Lock()
+	s.p = p
+	s.m.Unlock()
+	return p, nil
+}
+
+// Stop stops the server. It blocks until the server is
+// totally stopped. On POSIX systems, it will wait for
+// connections to close (up to a max timeout of a few
+// seconds); on Windows it will close the listener
+// immediately.
+func (s *Server) Stop() (err error) {
+
+	if runtime.GOOS != "windows" {
+		// force connections to close after timeout
+		done := make(chan struct{})
+		go func() {
+			s.dnsWg.Done() // decrement our initial increment used as a barrier
+			s.dnsWg.Wait()
+			close(done)
+		}()
+
+		// Wait for remaining connections to finish or
+		// force them all to close after timeout
+		select {
+		case <-time.After(s.connTimeout):
+		case <-done:
+		}
+	}
+
+	// Close the listener now; this stops the server without delay
+	s.m.Lock()
+	if s.l != nil {
+		err = s.l.Close()
+	}
+	if s.p != nil {
+		err = s.p.Close()
+	}
+
+	for _, s1 := range s.server {
+		err = s1.Shutdown()
+	}
+	s.m.Unlock()
+	return
+}
+
+// ServeDNS is the entry point for every request to the address that s
+// is bound to. It acts as a multiplexer for the requests zonename as
+// defined in the request so that the correct zone
+// (configuration and middleware stack) will handle the request.
+func (s *Server) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+	// TODO(miek): expensive to use defer
+	defer func() {
+		// In case the user doesn't enable error middleware, we still
+		// need to make sure that we stay alive up here
+		if rec := recover(); rec != nil {
+			DefaultErrorFunc(w, r, dns.RcodeServerFailure)
+		}
+	}()
+
+	if m, err := middleware.Edns0Version(r); err != nil { // Wrong EDNS version, return at once.
+		w.WriteMsg(m)
+		return
+	}
+
+	q := r.Question[0].Name
+	b := make([]byte, len(q))
+	off, end := 0, false
+	ctx := context.Background()
+
+	for {
+		l := len(q[off:])
+		for i := 0; i < l; i++ {
+			b[i] = q[off+i]
+			// normalize the name for the lookup
+			if b[i] >= 'A' && b[i] <= 'Z' {
+				b[i] |= ('a' - 'A')
+			}
+		}
+
+		if h, ok := s.zones[string(b[:l])]; ok {
+			if r.Question[0].Qtype != dns.TypeDS {
+				rcode, _ := h.middlewareChain.ServeDNS(ctx, w, r)
+				if RcodeNoClientWrite(rcode) {
+					DefaultErrorFunc(w, r, rcode)
+				}
+				return
+			}
+		}
+		off, end = dns.NextLabel(q, off)
+		if end {
+			break
+		}
+	}
+	// Wildcard match, if we have found nothing try the root zone as a last resort.
+	if h, ok := s.zones["."]; ok {
+		rcode, _ := h.middlewareChain.ServeDNS(ctx, w, r)
+		if RcodeNoClientWrite(rcode) {
+			DefaultErrorFunc(w, r, rcode)
+		}
+		return
+	}
+
+	// Still here? Error out with REFUSED and some logging
+	remoteHost := w.RemoteAddr().String()
+	DefaultErrorFunc(w, r, dns.RcodeRefused)
+	log.Printf("[INFO] \"%s %s %s\" - No such zone at %s (Remote: %s)", dns.Type(r.Question[0].Qtype), dns.Class(r.Question[0].Qclass), q, s.Addr, remoteHost)
+}
+
+// DefaultErrorFunc responds to an DNS request with an error.
+func DefaultErrorFunc(w dns.ResponseWriter, r *dns.Msg, rcode int) {
+	state := middleware.State{W: w, Req: r}
+
+	answer := new(dns.Msg)
+	answer.SetRcode(r, rcode)
+	state.SizeAndDo(answer)
+
+	w.WriteMsg(answer)
+}
+
+func RcodeNoClientWrite(rcode int) bool {
+	switch rcode {
+	case dns.RcodeServerFailure:
+		fallthrough
+	case dns.RcodeRefused:
+		fallthrough
+	case dns.RcodeFormatError:
+		fallthrough
+	case dns.RcodeNotImplemented:
+		return true
+	}
+	return false
+}
+
+const (
+	tcp = 0
+	udp = 1
+)