Add etcd middleware

This middleware acts in the same way as SkyDNS. We might add options
to allow it to be behave different, but for now it will suffice.

A Corefile like:
.:1053 {
    etcd miek.nl
    proxy . 8.8.8.8:53
}
will perform lookup in etcd and proxy everything not miek.nl to Google
for further resolution.

The internal etcd forwarding *also* uses the proxy infrastructure,
meaning you get health check and such for (almost) free

core/directives.go

@@ -53,10 +53,13 @@ var directiveOrder = []directive{
 	// Directives that inject handlers (middleware)
 	{"prometheus", setup.Prometheus},
 	{"rewrite", setup.Rewrite},
-	{"file", setup.File},
-	{"reflect", setup.Reflect},
 	{"log", setup.Log},
 	{"errors", setup.Errors},
+
+	{"etcd", setup.Etcd},
+	{"file", setup.File},
+	{"reflect", setup.Reflect},
+
 	{"proxy", setup.Proxy},
 }
 

core/setup/etcd.go

@@ -0,0 +1,106 @@
+package setup
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/etcd"
+	"github.com/miekg/coredns/middleware/etcd/singleflight"
+	"github.com/miekg/coredns/middleware/proxy"
+
+	etcdc "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+)
+
+const defaultEndpoint = "http://127.0.0.1:2379"
+
+// Etcd sets up the etcd middleware.
+func Etcd(c *Controller) (middleware.Middleware, error) {
+	etcd, err := etcdParse(c)
+	if err != nil {
+		return nil, err
+	}
+
+	return func(next middleware.Handler) middleware.Handler {
+		etcd.Next = next
+		return etcd
+	}, nil
+}
+
+func etcdParse(c *Controller) (etcd.Etcd, error) {
+	etc := etcd.Etcd{
+		// make stuff configurable
+		Proxy:      proxy.New([]string{"8.8.8.8:53"}),
+		PathPrefix: "skydns",
+		Ctx:        context.Background(),
+		Inflight:   &singleflight.Group{},
+	}
+	for c.Next() {
+		if c.Val() == "etcd" {
+			// etcd [origin...]
+			client, err := newEtcdClient([]string{defaultEndpoint}, "", "", "")
+			if err != nil {
+				return etcd.Etcd{}, err
+			}
+			etc.Client = client
+			etc.Zones = c.RemainingArgs()
+			if len(etc.Zones) == 0 {
+				etc.Zones = c.ServerBlockHosts
+			}
+			middleware.Zones(etc.Zones).FullyQualify()
+			return etc, nil
+		}
+	}
+	return etcd.Etcd{}, nil
+}
+
+func newEtcdClient(endpoints []string, tlsCert, tlsKey, tlsCACert string) (etcdc.KeysAPI, error) {
+	etcdCfg := etcdc.Config{
+		Endpoints: endpoints,
+		Transport: newHTTPSTransport(tlsCert, tlsKey, tlsCACert),
+	}
+	cli, err := etcdc.New(etcdCfg)
+	if err != nil {
+		return nil, err
+	}
+	return etcdc.NewKeysAPI(cli), nil
+}
+
+func newHTTPSTransport(tlsCertFile, tlsKeyFile, tlsCACertFile string) etcdc.CancelableTransport {
+	var cc *tls.Config = nil
+
+	if tlsCertFile != "" && tlsKeyFile != "" {
+		var rpool *x509.CertPool
+		if tlsCACertFile != "" {
+			if pemBytes, err := ioutil.ReadFile(tlsCACertFile); err == nil {
+				rpool = x509.NewCertPool()
+				rpool.AppendCertsFromPEM(pemBytes)
+			}
+		}
+
+		if tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile); err == nil {
+			cc = &tls.Config{
+				RootCAs:            rpool,
+				Certificates:       []tls.Certificate{tlsCert},
+				InsecureSkipVerify: true,
+			}
+		}
+	}
+
+	tr := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig:     cc,
+	}
+
+	return tr
+}

core/setup/file.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/miekg/coredns/middleware"
 	"github.com/miekg/coredns/middleware/file"
+
 	"github.com/miekg/dns"
 )
 

middleware/etcd/etcd.go

@@ -0,0 +1,149 @@
+// Package etcd provides the etcd backend.
+package etcd
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/etcd/msg"
+	"github.com/miekg/coredns/middleware/etcd/singleflight"
+	"github.com/miekg/coredns/middleware/proxy"
+
+	etcdc "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+)
+
+type Etcd struct {
+	Next       middleware.Handler
+	Zones      []string
+	Proxy      proxy.Proxy
+	Client     etcdc.KeysAPI
+	Ctx        context.Context
+	Inflight   *singleflight.Group
+	PathPrefix string
+}
+
+func (g Etcd) Records(name string, exact bool) ([]msg.Service, error) {
+	path, star := g.PathWithWildcard(name)
+	r, err := g.Get(path, true)
+	if err != nil {
+		return nil, err
+	}
+	segments := strings.Split(g.Path(name), "/")
+	switch {
+	case exact && r.Node.Dir:
+		return nil, nil
+	case r.Node.Dir:
+		return g.loopNodes(r.Node.Nodes, segments, star, nil)
+	default:
+		return g.loopNodes([]*etcdc.Node{r.Node}, segments, false, nil)
+	}
+}
+
+// Get is a wrapper for client.Get that uses SingleInflight to suppress multiple outstanding queries.
+func (g Etcd) Get(path string, recursive bool) (*etcdc.Response, error) {
+	resp, err := g.Inflight.Do(path, func() (interface{}, error) {
+		r, e := g.Client.Get(g.Ctx, path, &etcdc.GetOptions{Sort: false, Recursive: recursive})
+		if e != nil {
+			return nil, e
+		}
+		return r, e
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.(*etcdc.Response), err
+}
+
+// skydns/local/skydns/east/staging/web
+// skydns/local/skydns/west/production/web
+//
+// skydns/local/skydns/*/*/web
+// skydns/local/skydns/*/web
+
+// loopNodes recursively loops through the nodes and returns all the values. The nodes' keyname
+// will be match against any wildcards when star is true.
+func (g Etcd) loopNodes(ns []*etcdc.Node, nameParts []string, star bool, bx map[msg.Service]bool) (sx []msg.Service, err error) {
+	if bx == nil {
+		bx = make(map[msg.Service]bool)
+	}
+Nodes:
+	for _, n := range ns {
+		if n.Dir {
+			nodes, err := g.loopNodes(n.Nodes, nameParts, star, bx)
+			if err != nil {
+				return nil, err
+			}
+			sx = append(sx, nodes...)
+			continue
+		}
+		if star {
+			keyParts := strings.Split(n.Key, "/")
+			for i, n := range nameParts {
+				if i > len(keyParts)-1 {
+					// name is longer than key
+					continue Nodes
+				}
+				if n == "*" || n == "any" {
+					continue
+				}
+				if keyParts[i] != n {
+					continue Nodes
+				}
+			}
+		}
+		serv := new(msg.Service)
+		if err := json.Unmarshal([]byte(n.Value), serv); err != nil {
+			return nil, err
+		}
+		b := msg.Service{Host: serv.Host, Port: serv.Port, Priority: serv.Priority, Weight: serv.Weight, Text: serv.Text}
+		if _, ok := bx[b]; ok {
+			continue
+		}
+		bx[b] = true
+
+		serv.Key = n.Key
+		serv.Ttl = g.Ttl(n, serv)
+		if serv.Priority == 0 {
+			serv.Priority = priority
+		}
+		sx = append(sx, *serv)
+	}
+	return sx, nil
+}
+
+// Ttl returns the smaller of the etcd TTL and the service's
+// TTL. If neither of these are set (have a zero value), a default is used.
+func (g Etcd) Ttl(node *etcdc.Node, serv *msg.Service) uint32 {
+	etcdTtl := uint32(node.TTL)
+
+	if etcdTtl == 0 && serv.Ttl == 0 {
+		return ttl
+	}
+	if etcdTtl == 0 {
+		return serv.Ttl
+	}
+	if serv.Ttl == 0 {
+		return etcdTtl
+	}
+	if etcdTtl < serv.Ttl {
+		return etcdTtl
+	}
+	return serv.Ttl
+}
+
+// etcNameError checks if the error is ErrorCodeKeyNotFound from etcd.
+func isEtcdNameError(err error) bool {
+	if e, ok := err.(etcdc.Error); ok && e.Code == etcdc.ErrorCodeKeyNotFound {
+		return true
+	}
+	return false
+}
+
+const (
+	priority   = 10  // default priority when nothing is set
+	ttl        = 300 // default ttl when nothing is set
+	minTtl     = 60
+	hostmaster = "hostmaster"
+)

middleware/etcd/etcd.md

@@ -0,0 +1,33 @@
+# etcd
+
+`etcd` enabled reading zone data from an etcd instance. The data in etcd has to be encoded as
+a [message](https://github.com/skynetservices/skydns/blob/2fcff74cdc9f9a7dd64189a447ef27ac354b725f/msg/service.go#L26)
+like [SkyDNS](https//github.com/skynetservices/skydns).
+
+## Syntax
+
+~~~
+etcd [zones...]
+~~~
+
+* `zones` zones it should be authoritative for.
+
+The will default to `/skydns` as the path and the local etcd proxy (http://127.0.0.1:2379).
+If no zones are specified the block's zone will be used as the zone.
+
+If you want to `round robin` A and AAAA responses look at the `round_robin` middleware. optimize
+middleware?
+
+~~~
+etcd {
+    path /skydns
+    endpoint endpoint...
+    stubzones
+}
+~~~
+
+* `path` /skydns
+* `endpoint` endpoints...
+* `stubzones`
+
+## Examples

middleware/etcd/handler.go

@@ -0,0 +1,74 @@
+package etcd
+
+import (
+	"github.com/miekg/coredns/middleware"
+
+	"github.com/miekg/dns"
+	"golang.org/x/net/context"
+)
+
+func (e Etcd) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	state := middleware.State{W: w, Req: r}
+	zone := middleware.Zones(e.Zones).Matches(state.Name())
+	if zone == "" {
+		return e.Next.ServeDNS(ctx, w, r)
+	}
+
+	m := state.AnswerMessage()
+	m.Authoritative, m.RecursionAvailable, m.Compress = true, true, true
+
+	var (
+		records, extra []dns.RR
+		err            error
+	)
+	switch state.Type() {
+	case "A":
+		records, err = e.A(zone, state, nil)
+	case "AAAA":
+		records, err = e.AAAA(zone, state, nil)
+	case "TXT":
+		records, err = e.TXT(zone, state)
+	case "CNAME":
+		records, err = e.CNAME(zone, state)
+	case "MX":
+		records, extra, err = e.MX(zone, state)
+	case "SRV":
+		records, extra, err = e.SRV(zone, state)
+	default:
+		// For SOA and NS we might still want this
+		// and use dns.<zones> as the name to put these
+		// also for stub
+		// rwrite and return
+		// Nodata response
+		// also catch other types, so that they return NODATA
+		return 0, nil
+	}
+	if isEtcdNameError(err) {
+		NameError(zone, state)
+		return dns.RcodeNameError, nil
+	}
+	if err != nil {
+		return dns.RcodeServerFailure, err
+	}
+	if len(records) > 0 {
+		m.Answer = append(m.Answer, records...)
+	}
+	if len(extra) > 0 {
+		m.Extra = append(m.Extra, extra...)
+	}
+	state.W.WriteMsg(m)
+	return 0, nil
+}
+
+// NameError writes a name error to the client.
+func NameError(zone string, state middleware.State) {
+	m := new(dns.Msg)
+	m.SetRcode(state.Req, dns.RcodeNameError)
+	m.Ns = []dns.RR{SOA(zone)}
+	state.W.WriteMsg(m)
+}
+
+// NoData write a nodata response to the client.
+func NoData(zone string, state middleware.State) {
+	// TODO(miek): write it
+}

middleware/etcd/lookup.go

@@ -0,0 +1,344 @@
+package etcd
+
+import (
+	"math"
+	"net"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/etcd/msg"
+
+	"github.com/miekg/dns"
+)
+
+// TODO(miek): factor out common code a bit
+
+func (e Etcd) A(zone string, state middleware.State, previousRecords []dns.RR) (records []dns.RR, err error) {
+	services, err := e.Records(state.Name(), false)
+	if err != nil {
+		return nil, err
+	}
+
+	services = msg.Group(services)
+
+	for _, serv := range services {
+		ip := net.ParseIP(serv.Host)
+		switch {
+		case ip == nil:
+			// Try to resolve as CNAME if it's not an IP, but only if we don't create loops.
+			// TODO(miek): lowercasing, use Match in middleware?
+			if state.Name() == dns.Fqdn(serv.Host) {
+				// x CNAME x is a direct loop, don't add those
+				continue
+			}
+
+			newRecord := serv.NewCNAME(state.QName(), serv.Host)
+			if len(previousRecords) > 7 {
+				// don't add it, and just continue
+				continue
+			}
+			if isDuplicateCNAME(newRecord, previousRecords) {
+				continue
+			}
+
+			state1 := copyState(state, serv.Host, state.QType())
+			nextRecords, err := e.A(zone, state1, append(previousRecords, newRecord))
+
+			if err == nil {
+				// Not only have we found something we should add the CNAME and the IP addresses.
+				if len(nextRecords) > 0 {
+					// TODO(miek): sorting here?
+					records = append(records, newRecord)
+					records = append(records, nextRecords...)
+				}
+				continue
+			}
+			// This means we can not complete the CNAME, try to look else where.
+			target := newRecord.Target
+			if dns.IsSubDomain(zone, target) {
+				// We should already have found it
+				continue
+			}
+			m1, e1 := e.Proxy.Lookup(state, target, state.QType())
+			if e1 != nil {
+				continue
+			}
+			// Len(m1.Answer) > 0 here is well?
+			records = append(records, newRecord)
+			records = append(records, m1.Answer...)
+			continue
+		case ip.To4() != nil:
+			records = append(records, serv.NewA(state.QName(), ip.To4()))
+		case ip.To4() == nil:
+			// noda?
+		}
+	}
+	return records, nil
+}
+
+func (e Etcd) AAAA(zone string, state middleware.State, previousRecords []dns.RR) (records []dns.RR, err error) {
+	services, err := e.Records(state.Name(), false)
+	if err != nil {
+		return nil, err
+	}
+
+	services = msg.Group(services)
+
+	for _, serv := range services {
+		ip := net.ParseIP(serv.Host)
+		switch {
+		case ip == nil:
+			// Try to resolve as CNAME if it's not an IP, but only if we don't create loops.
+			// TODO(miek): lowercasing, use Match in middleware/
+			if state.Name() == dns.Fqdn(serv.Host) {
+				// x CNAME x is a direct loop, don't add those
+				continue
+			}
+
+			newRecord := serv.NewCNAME(state.QName(), serv.Host)
+			if len(previousRecords) > 7 {
+				// don't add it, and just continue
+				continue
+			}
+			if isDuplicateCNAME(newRecord, previousRecords) {
+				continue
+			}
+
+			state1 := copyState(state, serv.Host, state.QType())
+			nextRecords, err := e.AAAA(zone, state1, append(previousRecords, newRecord))
+
+			if err == nil {
+				// Not only have we found something we should add the CNAME and the IP addresses.
+				if len(nextRecords) > 0 {
+					// TODO(miek): sorting here?
+					records = append(records, newRecord)
+					records = append(records, nextRecords...)
+				}
+				continue
+			}
+			// This means we can not complete the CNAME, try to look else where.
+			target := newRecord.Target
+			if dns.IsSubDomain(zone, target) {
+				// We should already have found it
+				continue
+			}
+			m1, e1 := e.Proxy.Lookup(state, target, state.QType())
+			if e1 != nil {
+				continue
+			}
+			// Len(m1.Answer) > 0 here is well?
+			records = append(records, newRecord)
+			records = append(records, m1.Answer...)
+			continue
+			// both here again
+		case ip.To4() != nil:
+			// nada?
+		case ip.To4() == nil:
+			records = append(records, serv.NewAAAA(state.QName(), ip.To16()))
+		}
+	}
+	return records, nil
+}
+
+// SRV returns SRV records from etcd.
+// If the Target is not a name but an IP address, a name is created on the fly.
+func (e Etcd) SRV(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) {
+	services, err := e.Records(state.Name(), false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	services = msg.Group(services)
+
+	// Looping twice to get the right weight vs priority
+	w := make(map[int]int)
+	for _, serv := range services {
+		weight := 100
+		if serv.Weight != 0 {
+			weight = serv.Weight
+		}
+		if _, ok := w[serv.Priority]; !ok {
+			w[serv.Priority] = weight
+			continue
+		}
+		w[serv.Priority] += weight
+	}
+	lookup := make(map[string]bool)
+	for _, serv := range services {
+		w1 := 100.0 / float64(w[serv.Priority])
+		if serv.Weight == 0 {
+			w1 *= 100
+		} else {
+			w1 *= float64(serv.Weight)
+		}
+		weight := uint16(math.Floor(w1))
+		ip := net.ParseIP(serv.Host)
+		switch {
+		case ip == nil:
+			srv := serv.NewSRV(state.QName(), weight)
+			records = append(records, srv)
+
+			if _, ok := lookup[srv.Target]; ok {
+				break
+			}
+
+			lookup[srv.Target] = true
+
+			if !dns.IsSubDomain(zone, srv.Target) {
+				m1, e1 := e.Proxy.Lookup(state, srv.Target, dns.TypeA)
+				if e1 == nil {
+					extra = append(extra, m1.Answer...)
+				}
+				m1, e1 = e.Proxy.Lookup(state, srv.Target, dns.TypeAAAA)
+				if e1 == nil {
+					// If we have seen CNAME's we *assume* that they are already added.
+					for _, a := range m1.Answer {
+						if _, ok := a.(*dns.CNAME); !ok {
+							extra = append(extra, a)
+						}
+					}
+				}
+				break
+			}
+			// Internal name, we should have some info on them, either v4 or v6
+			// Clients expect a complete answer, because we are a recursor in their
+			// view.
+			state1 := copyState(state, srv.Target, dns.TypeA)
+			// TODO(both is true here!
+			addr, e1 := e.A(zone, state1, nil)
+			if e1 == nil {
+				extra = append(extra, addr...)
+			}
+			// e.AAA(zone, state1, nil) as well...
+		case ip.To4() != nil:
+			serv.Host = e.Domain(serv.Key)
+			srv := serv.NewSRV(state.QName(), weight)
+
+			records = append(records, srv)
+			extra = append(extra, serv.NewA(srv.Target, ip.To4()))
+		case ip.To4() == nil:
+			serv.Host = e.Domain(serv.Key)
+			srv := serv.NewSRV(state.QName(), weight)
+
+			records = append(records, srv)
+			extra = append(extra, serv.NewAAAA(srv.Target, ip.To16()))
+		}
+	}
+	return records, extra, nil
+}
+
+// MX returns MX records from etcd.
+// If the Target is not a name but an IP address, a name is created on the fly.
+func (e Etcd) MX(zone string, state middleware.State) (records []dns.RR, extra []dns.RR, err error) {
+	services, err := e.Records(state.Name(), false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	lookup := make(map[string]bool)
+	for _, serv := range services {
+		if !serv.Mail {
+			continue
+		}
+		ip := net.ParseIP(serv.Host)
+		switch {
+		case ip == nil:
+			mx := serv.NewMX(state.QName())
+			records = append(records, mx)
+			if _, ok := lookup[mx.Mx]; ok {
+				break
+			}
+
+			lookup[mx.Mx] = true
+
+			if !dns.IsSubDomain(zone, mx.Mx) {
+				m1, e1 := e.Proxy.Lookup(state, mx.Mx, dns.TypeA)
+				if e1 == nil {
+					extra = append(extra, m1.Answer...)
+				}
+				m1, e1 = e.Proxy.Lookup(state, mx.Mx, dns.TypeAAAA)
+				if e1 == nil {
+					// If we have seen CNAME's we *assume* that they are already added.
+					for _, a := range m1.Answer {
+						if _, ok := a.(*dns.CNAME); !ok {
+							extra = append(extra, a)
+						}
+					}
+				}
+				break
+			}
+			// Internal name
+			// both is true here as well
+			state1 := copyState(state, mx.Mx, dns.TypeA)
+			addr, e1 := e.A(zone, state1, nil)
+			if e1 == nil {
+				extra = append(extra, addr...)
+			}
+			// e.AAAA as well
+		case ip.To4() != nil:
+			serv.Host = e.Domain(serv.Key)
+			records = append(records, serv.NewMX(state.QName()))
+			extra = append(extra, serv.NewA(serv.Host, ip.To4()))
+		case ip.To4() == nil:
+			serv.Host = e.Domain(serv.Key)
+			records = append(records, serv.NewMX(state.QName()))
+			extra = append(extra, serv.NewAAAA(serv.Host, ip.To16()))
+		}
+	}
+	return records, extra, nil
+}
+
+func (e Etcd) CNAME(zone string, state middleware.State) (records []dns.RR, err error) {
+	services, err := e.Records(state.Name(), true)
+	if err != nil {
+		return nil, err
+	}
+
+	services = msg.Group(services)
+
+	if len(services) > 0 {
+		serv := services[0]
+		if ip := net.ParseIP(serv.Host); ip == nil {
+			records = append(records, serv.NewCNAME(state.QName(), serv.Host))
+		}
+	}
+	return records, nil
+}
+
+func (e Etcd) TXT(zone string, state middleware.State) (records []dns.RR, err error) {
+	services, err := e.Records(state.Name(), false)
+	if err != nil {
+		return nil, err
+	}
+
+	services = msg.Group(services)
+
+	for _, serv := range services {
+		if serv.Text == "" {
+			continue
+		}
+		records = append(records, serv.NewTXT(state.QName()))
+	}
+	return records, nil
+}
+
+// synthesis a SOA Record.
+func SOA(zone string) *dns.SOA {
+	return &dns.SOA{}
+}
+
+func isDuplicateCNAME(r *dns.CNAME, records []dns.RR) bool {
+	for _, rec := range records {
+		if v, ok := rec.(*dns.CNAME); ok {
+			if v.Target == r.Target {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func copyState(state middleware.State, target string, typ uint16) middleware.State {
+	state1 := state
+	state1.Req.Question[0] = dns.Question{dns.Fqdn(target), dns.ClassINET, typ}
+	return state1
+}

middleware/etcd/lookup_test.go

@@ -0,0 +1,393 @@
+package etcd
+
+// etcd needs to be running on http://127.0.0.1:2379
+// *and* needs connectivity to the internet for remotely resolving
+// names.
+
+import (
+	"encoding/json"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/etcd/msg"
+	"github.com/miekg/coredns/middleware/etcd/singleflight"
+	"github.com/miekg/coredns/middleware/proxy"
+	"github.com/miekg/dns"
+
+	etcdc "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+)
+
+var (
+	etc    Etcd
+	client etcdc.KeysAPI
+	ctx    context.Context
+)
+
+type Section int
+
+const (
+	Answer Section = iota
+	Ns
+	Extra
+)
+
+func init() {
+	ctx = context.TODO()
+
+	etcdCfg := etcdc.Config{
+		Endpoints: []string{"http://localhost:2379"},
+	}
+	cli, _ := etcdc.New(etcdCfg)
+	etc = Etcd{
+		Proxy:      proxy.New([]string{"8.8.8.8:53"}),
+		PathPrefix: "skydns",
+		Ctx:        context.Background(),
+		Inflight:   &singleflight.Group{},
+		Zones:      []string{"skydns.test."},
+		Client:     etcdc.NewKeysAPI(cli),
+	}
+}
+
+func set(t *testing.T, e Etcd, k string, ttl time.Duration, m *msg.Service) {
+	b, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	path, _ := e.PathWithWildcard(k)
+	e.Client.Set(ctx, path, string(b), &etcdc.SetOptions{TTL: ttl})
+}
+
+func delete(t *testing.T, e Etcd, k string) {
+	path, _ := e.PathWithWildcard(k)
+	e.Client.Delete(ctx, path, &etcdc.DeleteOptions{Recursive: false})
+}
+
+type rrSet []dns.RR
+
+func (p rrSet) Len() int           { return len(p) }
+func (p rrSet) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p rrSet) Less(i, j int) bool { return p[i].String() < p[j].String() }
+
+func TestLookup(t *testing.T) {
+	for _, serv := range services {
+		set(t, etc, serv.Key, 0, serv)
+		defer delete(t, etc, serv.Key)
+	}
+	for _, tc := range dnsTestCases {
+		m := new(dns.Msg)
+		m.SetQuestion(dns.Fqdn(tc.Qname), tc.Qtype)
+
+		rec := middleware.NewResponseRecorder(&middleware.TestResponseWriter{})
+		code, err := etc.ServeDNS(ctx, rec, m)
+		if err != nil {
+			t.Errorf("expected no error, got %v\n", err)
+			return
+		}
+		resp := rec.Reply()
+		code = code // TODO(miek): test
+		// if nil then?
+
+		sort.Sort(rrSet(resp.Answer))
+		sort.Sort(rrSet(resp.Ns))
+		sort.Sort(rrSet(resp.Extra))
+
+		if resp.Rcode != tc.Rcode {
+			t.Errorf("rcode is %q, expected %q", dns.RcodeToString[resp.Rcode], dns.RcodeToString[tc.Rcode])
+			continue
+		}
+
+		if len(resp.Answer) != len(tc.Answer) {
+			t.Errorf("answer for %q contained %d results, %d expected", tc.Qname, len(resp.Answer), len(tc.Answer))
+			continue
+		}
+		if len(resp.Ns) != len(tc.Ns) {
+			t.Errorf("authority for %q contained %d results, %d expected", tc.Qname, len(resp.Ns), len(tc.Ns))
+			continue
+		}
+		if len(resp.Extra) != len(tc.Extra) {
+			t.Errorf("additional for %q contained %d results, %d expected", tc.Qname, len(resp.Extra), len(tc.Extra))
+			continue
+		}
+
+		checkSection(t, tc, Answer, resp.Answer)
+		checkSection(t, tc, Ns, resp.Ns)
+		checkSection(t, tc, Extra, resp.Extra)
+	}
+}
+
+type dnsTestCase struct {
+	Qname  string
+	Qtype  uint16
+	Rcode  int
+	Answer []dns.RR
+	Ns     []dns.RR
+	Extra  []dns.RR
+}
+
+// Note the key is encoded as DNS name, while in "reality" it is a etcd path.
+var services = []*msg.Service{
+	{Host: "server1", Port: 8080, Key: "a.server1.dev.region1.skydns.test."},
+	{Host: "10.0.0.1", Port: 8080, Key: "a.server1.prod.region1.skydns.test."},
+	{Host: "10.0.0.2", Port: 8080, Key: "b.server1.prod.region1.skydns.test."},
+	{Host: "::1", Port: 8080, Key: "b.server6.prod.region1.skydns.test."},
+
+	// CNAME dedup Test
+	{Host: "www.miek.nl", Key: "a.miek.nl.skydns.test."},
+	{Host: "www.miek.nl", Key: "b.miek.nl.skydns.test."},
+}
+
+var dnsTestCases = []dnsTestCase{
+	// SRV Test
+	{
+		Qname: "a.server1.dev.region1.skydns.test.", Qtype: dns.TypeSRV,
+		Answer: []dns.RR{newSRV("a.server1.dev.region1.skydns.test. 300 SRV 10 100 8080 server1.")},
+	},
+	// A Test
+	{
+		Qname: "a.server1.prod.region1.skydns.test.", Qtype: dns.TypeA,
+		Answer: []dns.RR{newA("a.server1.prod.region1.skydns.test. 300 A 10.0.0.1")},
+	},
+	// SRV Test where target is IP address
+	{
+		Qname: "a.server1.prod.region1.skydns.test.", Qtype: dns.TypeSRV,
+		Answer: []dns.RR{newSRV("a.server1.prod.region1.skydns.test. 300 SRV 10 100 8080 a.server1.prod.region1.skydns.test.")},
+		Extra:  []dns.RR{newA("a.server1.prod.region1.skydns.test. 300 A 10.0.0.1")},
+	},
+	// AAAA Test
+	{
+		Qname: "b.server6.prod.region1.skydns.test.", Qtype: dns.TypeAAAA,
+		Answer: []dns.RR{newAAAA("b.server6.prod.region1.skydns.test. 300 AAAA ::1")},
+	},
+	// Multiple A Record Test
+	{
+		Qname: "server1.prod.region1.skydns.test.", Qtype: dns.TypeA,
+		Answer: []dns.RR{
+			newA("server1.prod.region1.skydns.test. 300 A 10.0.0.1"),
+			newA("server1.prod.region1.skydns.test. 300 A 10.0.0.2"),
+		},
+	},
+	// Multi SRV with the same target, should be dedupped.
+	{
+		Qname: "*.miek.nl.skydns.test.", Qtype: dns.TypeSRV,
+		Answer: []dns.RR{
+			newSRV("*.miek.nl.skydns.test. 300 IN SRV 10 100 0 www.miek.nl."),
+		},
+		// TODO(miek): bit stupid to rely on my home DNS setup for this...
+		Extra: []dns.RR{
+			// 303 ttl: don't care for the ttl on these RRs.
+			newA("a.miek.nl. 303 IN A 139.162.196.78"),
+			newAAAA("a.miek.nl. 303 IN AAAA 2a01:7e00::f03c:91ff:fef1:6735"),
+			newCNAME("www.miek.nl. 303 IN CNAME a.miek.nl."),
+		},
+	},
+	/*
+		// CNAME (unresolvable internal name)
+		{
+			Qname: "2.cname.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{},
+			Ns:     []dns.RR{newSOA("skydns.test. 60 SOA ns.dns.skydns.test. hostmaster.skydns.test. 1407441600 28800 7200 604800 60")},
+		},
+		// CNAME loop detection
+		{
+			Qname: "3.cname.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{},
+			Ns:     []dns.RR{newSOA("skydns.test. 60 SOA ns.dns.skydns.test. hostmaster.skydns.test. 1407441600 28800 7200 604800 60")},
+		},
+		// CNAME (resolvable external name)
+		{
+			Qname: "external1.cname.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{
+				newA("a.miek.nl. 60 IN A 139.162.196.78"),
+				newCNAME("external1.cname.skydns.test. 60 IN CNAME www.miek.nl."),
+				newCNAME("www.miek.nl. 60 IN CNAME a.miek.nl."),
+			},
+		},
+		// CNAME (unresolvable external name)
+		{
+			Qname: "external2.cname.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{},
+			Ns:     []dns.RR{newSOA("skydns.test. 60 SOA ns.dns.skydns.test. hostmaster.skydns.test. 1407441600 28800 7200 604800 60")},
+		},
+		// Priority Test
+		{
+			Qname: "region6.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{newSRV("region6.skydns.test. 300 SRV 333 100 80 server4.")},
+		},
+		// Subdomain Test
+		{
+			Qname: "region1.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{
+				newSRV("region1.skydns.test. 300 SRV 10 33 0 104.server1.dev.region1.skydns.test."),
+				newSRV("region1.skydns.test. 300 SRV 10 33 80 server2"),
+				newSRV("region1.skydns.test. 300 SRV 10 33 8080 server1.")},
+			Extra: []dns.RR{newA("104.server1.dev.region1.skydns.test. 300 A 10.0.0.1")},
+		},
+		// Subdomain Weight Test
+		{
+			Qname: "region5.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{
+				newSRV("region5.skydns.test. 300 SRV 10 22 0 server2."),
+				newSRV("region5.skydns.test. 300 SRV 10 36 0 server1."),
+				newSRV("region5.skydns.test. 300 SRV 10 41 0 server3."),
+				newSRV("region5.skydns.test. 300 SRV 30 100 0 server4.")},
+		},
+		// Wildcard Test
+		{
+			Qname: "*.region1.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{
+				newSRV("*.region1.skydns.test. 300 SRV 10 33 0 104.server1.dev.region1.skydns.test."),
+				newSRV("*.region1.skydns.test. 300 SRV 10 33 80 server2"),
+				newSRV("*.region1.skydns.test. 300 SRV 10 33 8080 server1.")},
+			Extra: []dns.RR{newA("104.server1.dev.region1.skydns.test. 300 A 10.0.0.1")},
+		},
+		// Wildcard Test
+		{
+			Qname: "prod.*.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{
+				newSRV("prod.*.skydns.test. 300 IN SRV 10 50 0 105.server3.prod.region2.skydns.test."),
+				newSRV("prod.*.skydns.test. 300 IN SRV 10 50 80 server2.")},
+			Extra: []dns.RR{newAAAA("105.server3.prod.region2.skydns.test. 300 IN AAAA 2001::8:8:8:8")},
+		},
+		// Wildcard Test
+		{
+			Qname: "prod.any.skydns.test.", Qtype: dns.TypeSRV,
+			Answer: []dns.RR{
+				newSRV("prod.any.skydns.test. 300 IN SRV 10 50 0 105.server3.prod.region2.skydns.test."),
+				newSRV("prod.any.skydns.test. 300 IN SRV 10 50 80 server2.")},
+			Extra: []dns.RR{newAAAA("105.server3.prod.region2.skydns.test. 300 IN AAAA 2001::8:8:8:8")},
+		},
+		// NXDOMAIN Test
+		{
+			Qname: "doesnotexist.skydns.test.", Qtype: dns.TypeA,
+			Rcode: dns.RcodeNameError,
+			Ns: []dns.RR{
+				newSOA("skydns.test. 300 SOA ns.dns.skydns.test. hostmaster.skydns.test. 0 0 0 0 0"),
+			},
+		},
+		// NODATA Test
+		{
+			Qname: "104.server1.dev.region1.skydns.test.", Qtype: dns.TypeTXT,
+			Ns: []dns.RR{newSOA("skydns.test. 300 SOA ns.dns.skydns.test. hostmaster.skydns.test. 0 0 0 0 0")},
+		},
+		// NODATA Test 2
+		{
+			Qname: "100.server1.dev.region1.skydns.test.", Qtype: dns.TypeA,
+			Rcode: dns.RcodeSuccess,
+			Ns:    []dns.RR{newSOA("skydns.test. 300 SOA ns.dns.skydns.test. hostmaster.skydns.test. 0 0 0 0 0")},
+		},
+		{
+			// One has group, the other has not...  Include the non-group always.
+			Qname: "dom2.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{
+				newA("dom2.skydns.test. IN A 127.0.0.1"),
+				newA("dom2.skydns.test. IN A 127.0.0.2"),
+			},
+		},
+		{
+			// The groups differ.
+			Qname: "dom1.skydns.test.", Qtype: dns.TypeA,
+			Answer: []dns.RR{
+				newA("dom1.skydns.test. IN A 127.0.0.1"),
+			},
+		},
+	*/
+}
+
+func newA(rr string) *dns.A         { r, _ := dns.NewRR(rr); return r.(*dns.A) }
+func newAAAA(rr string) *dns.AAAA   { r, _ := dns.NewRR(rr); return r.(*dns.AAAA) }
+func newCNAME(rr string) *dns.CNAME { r, _ := dns.NewRR(rr); return r.(*dns.CNAME) }
+func newSRV(rr string) *dns.SRV     { r, _ := dns.NewRR(rr); return r.(*dns.SRV) }
+func newSOA(rr string) *dns.SOA     { r, _ := dns.NewRR(rr); return r.(*dns.SOA) }
+func newNS(rr string) *dns.NS       { r, _ := dns.NewRR(rr); return r.(*dns.NS) }
+func newPTR(rr string) *dns.PTR     { r, _ := dns.NewRR(rr); return r.(*dns.PTR) }
+func newTXT(rr string) *dns.TXT     { r, _ := dns.NewRR(rr); return r.(*dns.TXT) }
+func newMX(rr string) *dns.MX       { r, _ := dns.NewRR(rr); return r.(*dns.MX) }
+
+func checkSection(t *testing.T, tc dnsTestCase, sect Section, rr []dns.RR) {
+	section := []dns.RR{}
+	switch sect {
+	case 0:
+		section = tc.Answer
+	case 1:
+		section = tc.Ns
+	case 2:
+		section = tc.Extra
+	}
+
+	for i, a := range rr {
+		if a.Header().Name != section[i].Header().Name {
+			t.Errorf("answer %d should have a Header Name of %q, but has %q", i, section[i].Header().Name, a.Header().Name)
+			continue
+		}
+		// 303 signals: don't care what the ttl is.
+		if section[i].Header().Ttl != 303 && a.Header().Ttl != section[i].Header().Ttl {
+			t.Errorf("Answer %d should have a Header TTL of %d, but has %d", i, section[i].Header().Ttl, a.Header().Ttl)
+			continue
+		}
+		if a.Header().Rrtype != section[i].Header().Rrtype {
+			t.Errorf("answer %d should have a header rr type of %d, but has %dn", i, section[i].Header().Rrtype, a.Header().Rrtype)
+			continue
+		}
+
+		switch x := a.(type) {
+		case *dns.SRV:
+			if x.Priority != section[i].(*dns.SRV).Priority {
+				t.Errorf("answer %d should have a Priority of %d, but has %d", i, section[i].(*dns.SRV).Priority, x.Priority)
+			}
+			if x.Weight != section[i].(*dns.SRV).Weight {
+				t.Errorf("answer %d should have a Weight of %d, but has %d", i, section[i].(*dns.SRV).Weight, x.Weight)
+			}
+			if x.Port != section[i].(*dns.SRV).Port {
+				t.Errorf("answer %d should have a Port of %d, but has %d", i, section[i].(*dns.SRV).Port, x.Port)
+			}
+			if x.Target != section[i].(*dns.SRV).Target {
+				t.Errorf("answer %d should have a Target of %q, but has %q", i, section[i].(*dns.SRV).Target, x.Target)
+			}
+		case *dns.A:
+			if x.A.String() != section[i].(*dns.A).A.String() {
+				t.Errorf("answer %d should have a Address of %q, but has %q", i, section[i].(*dns.A).A.String(), x.A.String())
+			}
+		case *dns.AAAA:
+			if x.AAAA.String() != section[i].(*dns.AAAA).AAAA.String() {
+				t.Errorf("answer %d should have a Address of %q, but has %q", i, section[i].(*dns.AAAA).AAAA.String(), x.AAAA.String())
+			}
+		case *dns.TXT:
+			for j, txt := range x.Txt {
+				if txt != section[i].(*dns.TXT).Txt[j] {
+					t.Errorf("answer %d should have a Txt of %q, but has %q", i, section[i].(*dns.TXT).Txt[j], txt)
+				}
+			}
+		case *dns.SOA:
+			tt := section[i].(*dns.SOA)
+			if x.Ns != tt.Ns {
+				t.Errorf("SOA nameserver should be %q, but is %q", x.Ns, tt.Ns)
+			}
+		case *dns.PTR:
+			tt := section[i].(*dns.PTR)
+			if x.Ptr != tt.Ptr {
+				t.Errorf("PTR ptr should be %q, but is %q", x.Ptr, tt.Ptr)
+			}
+		case *dns.CNAME:
+			tt := section[i].(*dns.CNAME)
+			if x.Target != tt.Target {
+				t.Errorf("CNAME target should be %q, but is %q", x.Target, tt.Target)
+			}
+		case *dns.MX:
+			tt := section[i].(*dns.MX)
+			if x.Mx != tt.Mx {
+				t.Errorf("MX Mx should be %q, but is %q", x.Mx, tt.Mx)
+			}
+			if x.Preference != tt.Preference {
+				t.Errorf("MX Preference should be %q, but is %q", x.Preference, tt.Preference)
+			}
+		case *dns.NS:
+			tt := section[i].(*dns.NS)
+			if x.Ns != tt.Ns {
+				t.Errorf("NS nameserver should be %q, but is %q", x.Ns, tt.Ns)
+			}
+		}
+	}
+}

middleware/etcd/msg/service.go

@@ -0,0 +1,160 @@
+package msg
+
+import (
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+// This *is* the rdata from a SRV record, but with a twist.
+// Host (Target in SRV) must be a domain name, but if it looks like an IP
+// address (4/6), we will treat it like an IP address.
+type Service struct {
+	Host     string `json:"host,omitempty"`
+	Port     int    `json:"port,omitempty"`
+	Priority int    `json:"priority,omitempty"`
+	Weight   int    `json:"weight,omitempty"`
+	Text     string `json:"text,omitempty"`
+	Mail     bool   `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
+	Ttl      uint32 `json:"ttl,omitempty"`
+
+	// When a SRV record with a "Host: IP-address" is added, we synthesize
+	// a srv.Target domain name.  Normally we convert the full Key where
+	// the record lives to a DNS name and use this as the srv.Target.  When
+	// TargetStrip > 0 we strip the left most TargetStrip labels from the
+	// DNS name.
+	TargetStrip int `json:"targetstrip,omitempty"`
+
+	// Group is used to group (or *not* to group) different services
+	// together. Services with an identical Group are returned in the same
+	// answer.
+	Group string `json:"group,omitempty"`
+
+	// Etcd key where we found this service and ignored from json un-/marshalling
+	Key string `json:"-"`
+}
+
+// NewSRV returns a new SRV record based on the Service.
+func (s *Service) NewSRV(name string, weight uint16) *dns.SRV {
+	host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip)
+
+	return &dns.SRV{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeSRV, Class: dns.ClassINET, Ttl: s.Ttl},
+		Priority: uint16(s.Priority), Weight: weight, Port: uint16(s.Port), Target: dns.Fqdn(host)}
+}
+
+// NewMX returns a new MX record based on the Service.
+func (s *Service) NewMX(name string) *dns.MX {
+	host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip)
+
+	return &dns.MX{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: s.Ttl},
+		Preference: uint16(s.Priority), Mx: host}
+}
+
+// NewA returns a new A record based on the Service.
+func (s *Service) NewA(name string, ip net.IP) *dns.A {
+	return &dns.A{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.Ttl}, A: ip}
+}
+
+// NewAAAA returns a new AAAA record based on the Service.
+func (s *Service) NewAAAA(name string, ip net.IP) *dns.AAAA {
+	return &dns.AAAA{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.Ttl}, AAAA: ip}
+}
+
+// NewCNAME returns a new CNAME record based on the Service.
+func (s *Service) NewCNAME(name string, target string) *dns.CNAME {
+	return &dns.CNAME{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: s.Ttl}, Target: dns.Fqdn(target)}
+}
+
+// NewTXT returns a new TXT record based on the Service.
+func (s *Service) NewTXT(name string) *dns.TXT {
+	return &dns.TXT{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: s.Ttl}, Txt: split255(s.Text)}
+}
+
+// Group checks the services in sx, it looks for a Group attribute on the shortest
+// keys. If there are multiple shortest keys *and* the group attribute disagrees (and
+// is not empty), we don't consider it a group.
+// If a group is found, only services with *that* group (or no group) will be returned.
+func Group(sx []Service) []Service {
+	if len(sx) == 0 {
+		return sx
+	}
+
+	// Shortest key with group attribute sets the group for this set.
+	group := sx[0].Group
+	slashes := strings.Count(sx[0].Key, "/")
+	length := make([]int, len(sx))
+	for i, s := range sx {
+		x := strings.Count(s.Key, "/")
+		length[i] = x
+		if x < slashes {
+			if s.Group == "" {
+				break
+			}
+			slashes = x
+			group = s.Group
+		}
+	}
+
+	if group == "" {
+		return sx
+	}
+
+	ret := []Service{} // with slice-tricks in sx we can prolly save this allocation (TODO)
+
+	for i, s := range sx {
+		if s.Group == "" {
+			ret = append(ret, s)
+			continue
+		}
+
+		// Disagreement on the same level
+		if length[i] == slashes && s.Group != group {
+			return sx
+		}
+
+		if s.Group == group {
+			ret = append(ret, s)
+		}
+	}
+	return ret
+}
+
+// Split255 splits a string into 255 byte chunks.
+func split255(s string) []string {
+	if len(s) < 255 {
+		return []string{s}
+	}
+	sx := []string{}
+	p, i := 0, 255
+	for {
+		if i <= len(s) {
+			sx = append(sx, s[p:i])
+		} else {
+			sx = append(sx, s[p:])
+			break
+
+		}
+		p, i = p+255, i+255
+	}
+
+	return sx
+}
+
+// targetStrip strips "targetstrip" labels from the left side of the fully qualified name.
+func targetStrip(name string, targetStrip int) string {
+	if targetStrip == 0 {
+		return name
+	}
+
+	offset, end := 0, false
+	for i := 0; i < targetStrip; i++ {
+		offset, end = dns.NextLabel(name, offset)
+	}
+	if end {
+		// We overshot the name, use the orignal one.
+		offset = 0
+	}
+	name = name[offset:]
+	return name
+}

middleware/etcd/path.go

@@ -0,0 +1,46 @@
+package etcd
+
+import (
+	"path"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+// Path converts a domainname to an etcd path. If s looks like service.staging.skydns.local.,
+// the resulting key will be /skydns/local/skydns/staging/service .
+func (e Etcd) Path(s string) string {
+	l := dns.SplitDomainName(s)
+	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	return path.Join(append([]string{"/" + e.PathPrefix + "/"}, l...)...)
+}
+
+// Domain is the opposite of Path.
+func (e Etcd) Domain(s string) string {
+	l := strings.Split(s, "/")
+	// start with 1, to strip /skydns
+	for i, j := 1, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	return dns.Fqdn(strings.Join(l[1:len(l)-1], "."))
+}
+
+// As Path, but if a name contains wildcards (* or any), the name will be
+// chopped of before the (first) wildcard, and we do a highler evel search and
+// later find the matching names.  So service.*.skydns.local, will look for all
+// services under skydns.local and will later check for names that match
+// service.*.skydns.local.  If a wildcard is found the returned bool is true.
+func (e Etcd) PathWithWildcard(s string) (string, bool) {
+	l := dns.SplitDomainName(s)
+	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	for i, k := range l {
+		if k == "*" || k == "any" {
+			return path.Join(append([]string{"/" + e.PathPrefix + "/"}, l[:i]...)...), true
+		}
+	}
+	return path.Join(append([]string{"/" + e.PathPrefix + "/"}, l...)...), false
+}

middleware/etcd/path_test.go

@@ -0,0 +1,13 @@
+package etcd
+
+import "testing"
+
+func TestPath(t *testing.T) {
+	for _, path := range []string{"mydns", "skydns"} {
+		e := Etcd{PathPrefix: path}
+		result := e.Path("service.staging.skydns.local.")
+		if result != "/"+path+"/local/skydns/staging/service" {
+			t.Errorf("Failure to get domain's path with prefix: %s", result)
+		}
+	}
+}

middleware/etcd/singleflight/singleflight.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2012 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package singleflight provides a duplicate function call suppression
+// mechanism.
+package singleflight
+
+import "sync"
+
+// call is an in-flight or completed Do call
+type call struct {
+	wg  sync.WaitGroup
+	val interface{}
+	err error
+}
+
+// Group represents a class of work and forms a namespace in which
+// units of work can be executed with duplicate suppression.
+type Group struct {
+	mu sync.Mutex       // protects m
+	m  map[string]*call // lazily initialized
+}
+
+// Do executes and returns the results of the given function, making
+// sure that only one execution is in-flight for a given key at a
+// time. If a duplicate comes in, the duplicate caller waits for the
+// original to complete and receives the same results.
+func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) {
+	g.mu.Lock()
+	if g.m == nil {
+		g.m = make(map[string]*call)
+	}
+	if c, ok := g.m[key]; ok {
+		g.mu.Unlock()
+		c.wg.Wait()
+		return c.val, c.err
+	}
+	c := new(call)
+	c.wg.Add(1)
+	g.m[key] = c
+	g.mu.Unlock()
+
+	c.val, c.err = fn()
+	c.wg.Done()
+
+	g.mu.Lock()
+	delete(g.m, key)
+	g.mu.Unlock()
+
+	return c.val, c.err
+}

middleware/exchange.go

@@ -2,8 +2,6 @@ package middleware
 
 import "github.com/miekg/dns"
 
-// Exchang sends message m to the server.
-// TODO(miek): optionally it can do retries of other silly stuff.
 func Exchange(c *dns.Client, m *dns.Msg, server string) (*dns.Msg, error) {
 	r, _, err := c.Exchange(m, server)
 	return r, err

middleware/proxy/lookup.go

@@ -0,0 +1,105 @@
+package proxy
+
+// function OTHER middleware might want to use to do lookup in the same
+// style as the proxy.
+
+import (
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/dns"
+)
+
+func New(hosts []string) Proxy {
+	p := Proxy{Next: nil, Client: Clients()}
+
+	upstream := &staticUpstream{
+		from:         "",
+		proxyHeaders: make(http.Header),
+		Hosts:        make([]*UpstreamHost, len(hosts)),
+		Policy:       &Random{},
+		FailTimeout:  10 * time.Second,
+		MaxFails:     1,
+	}
+
+	for i, host := range hosts {
+		uh := &UpstreamHost{
+			Name:         host,
+			Conns:        0,
+			Fails:        0,
+			FailTimeout:  upstream.FailTimeout,
+			Unhealthy:    false,
+			ExtraHeaders: upstream.proxyHeaders,
+			CheckDown: func(upstream *staticUpstream) UpstreamHostDownFunc {
+				return func(uh *UpstreamHost) bool {
+					if uh.Unhealthy {
+						return true
+					}
+					if uh.Fails >= upstream.MaxFails &&
+						upstream.MaxFails != 0 {
+						return true
+					}
+					return false
+				}
+			}(upstream),
+			WithoutPathPrefix: upstream.WithoutPathPrefix,
+		}
+		upstream.Hosts[i] = uh
+	}
+	p.Upstreams = []Upstream{upstream}
+	return p
+}
+
+func (p Proxy) Lookup(state middleware.State, name string, tpe uint16) (*dns.Msg, error) {
+	req := new(dns.Msg)
+	req.SetQuestion(name, tpe)
+	// TODO(miek):
+	// USE STATE FOR DNSSEC ETCD BUFSIZE BLA BLA
+	return p.lookup(state, req)
+}
+
+func (p Proxy) lookup(state middleware.State, r *dns.Msg) (*dns.Msg, error) {
+	var (
+		reply *dns.Msg
+		err   error
+	)
+	for _, upstream := range p.Upstreams {
+		// allowed bla bla bla TODO(miek): fix full proxy spec from caddy
+		start := time.Now()
+
+		// Since Select() should give us "up" hosts, keep retrying
+		// hosts until timeout (or until we get a nil host).
+		for time.Now().Sub(start) < tryDuration {
+			host := upstream.Select()
+			if host == nil {
+				return nil, errUnreachable
+			}
+
+			atomic.AddInt64(&host.Conns, 1)
+			// tls+tcp ?
+			if state.Proto() == "tcp" {
+				reply, err = middleware.Exchange(p.Client.TCP, r, host.Name)
+			} else {
+				reply, err = middleware.Exchange(p.Client.UDP, r, host.Name)
+			}
+			atomic.AddInt64(&host.Conns, -1)
+
+			if err == nil {
+				return reply, nil
+			}
+			timeout := host.FailTimeout
+			if timeout == 0 {
+				timeout = 10 * time.Second
+			}
+			atomic.AddInt32(&host.Fails, 1)
+			go func(host *UpstreamHost, timeout time.Duration) {
+				time.Sleep(timeout)
+				atomic.AddInt32(&host.Fails, -1)
+			}(host, timeout)
+		}
+		return nil, errUnreachable
+	}
+	return nil, errUnreachable
+}

middleware/proxy/lookup_test.go

@@ -0,0 +1,34 @@
+package proxy
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/dns"
+)
+
+func TestLookupProxy(t *testing.T) {
+	// TODO(miek): make this fakeDNS backend and ask the question locally
+	log.SetOutput(ioutil.Discard)
+	defer log.SetOutput(os.Stderr)
+
+	p := New([]string{"8.8.8.8:53"})
+	resp, err := p.Lookup(fakeState(), "example.org.", dns.TypeA)
+	if err != nil {
+		t.Error("Expected to receive reply, but didn't")
+	}
+	// expect answer section with A record in it
+	if len(resp.Answer) == 0 {
+		t.Error("Expected to at least one RR in the answer section, got none")
+	}
+	if resp.Answer[0].Header().Rrtype != dns.TypeA {
+		t.Error("Expected RR to A, got: %d", resp.Answer[0].Header().Rrtype)
+	}
+}
+
+func fakeState() middleware.State {
+	return middleware.State{W: &middleware.TestResponseWriter{}, Req: new(dns.Msg)}
+}

middleware/recorder.go

@@ -17,6 +17,7 @@ type ResponseRecorder struct {
 	dns.ResponseWriter
 	rcode int
 	size  int
+	msg   *dns.Msg
 	start time.Time
 }
 
@@ -27,6 +28,7 @@ func NewResponseRecorder(w dns.ResponseWriter) *ResponseRecorder {
 	return &ResponseRecorder{
 		ResponseWriter: w,
 		rcode:          0,
+		msg:            nil,
 		start:          time.Now(),
 	}
 }
@@ -36,6 +38,7 @@ func NewResponseRecorder(w dns.ResponseWriter) *ResponseRecorder {
 func (r *ResponseRecorder) WriteMsg(res *dns.Msg) error {
 	r.rcode = res.Rcode
 	r.size = res.Len()
+	r.msg = res
 	return r.ResponseWriter.WriteMsg(res)
 }
 
@@ -63,6 +66,11 @@ func (r *ResponseRecorder) Start() time.Time {
 	return r.start
 }
 
+// Reply returns the written message from the ResponseRecorder.
+func (r *ResponseRecorder) Reply() *dns.Msg {
+	return r.msg
+}
+
 // Hijack implements dns.Hijacker. It simply wraps the underlying
 // ResponseWriter's Hijack method if there is one, or returns an error.
 func (r *ResponseRecorder) Hijack() {

middleware/state.go

@@ -83,6 +83,30 @@ func (s State) Family() int {
 	return 2
 }
 
+// Do returns if the request has the DO (DNSSEC OK) bit set.
+func (s State) Do() bool {
+	if o := s.Req.IsEdns0(); o != nil {
+		return o.Do()
+	}
+	return false
+}
+
+// UDPSize returns if UDP buffer size advertised in the requests OPT record.
+// Or when the request was over TCP, we return the maximum allowed size of 64K.
+func (s State) Size() int {
+	if s.Proto() == "tcp" {
+		return dns.MaxMsgSize
+	}
+	if o := s.Req.IsEdns0(); o != nil {
+		s := o.UDPSize()
+		if s < dns.MinMsgSize {
+			s = dns.MinMsgSize
+		}
+		return int(s)
+	}
+	return dns.MinMsgSize
+}
+
 // Type returns the type of the question as a string.
 func (s State) Type() string {
 	return dns.Type(s.Req.Question[0].Qtype).String()

middleware/zone.go

@@ -1,15 +1,19 @@
 package middleware
 
-import "strings"
+import (
+	"strings"
+
+	"github.com/miekg/dns"
+)
 
 type Zones []string
 
-// Matches checks to see if other matches p.
-// The match will return the most specific zones
-// that matches other. The empty string signals a not found
+// Matches checks to see if other matches p.  The match will return the most
+// specific zones that matches other. The empty string signals a not found
 // condition.
 func (z Zones) Matches(qname string) string {
 	zone := ""
+	// TODO(miek): use IsSubDomain here?
 	for _, zname := range z {
 		if strings.HasSuffix(qname, zname) {
 			if len(zname) > len(zone) {
@@ -19,3 +23,11 @@ func (z Zones) Matches(qname string) string {
 	}
 	return zone
 }
+
+// Fully qualify all zones in z
+func (z Zones) FullyQualify() {
+	for i, _ := range z {
+		z[i] = dns.Fqdn(z[i])
+	}
+
+}