Start working on a etcd backend

middleware/etcd/backend.go

@@ -0,0 +1,159 @@
+// Package etcd provides the etcd server Backend implementation,
+package etcd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/miekg/coredns/middleware/etcd/msg"
+
+	etcdc "github.com/coreos/etcd/client"
+)
+
+func (g *Backend) Records(name string, exact bool) ([]msg.Service, error) {
+	path, star := msg.PathWithWildcard(name)
+	r, err := g.get(path, true)
+	if err != nil {
+		return nil, err
+	}
+	segments := strings.Split(msg.Path(name), "/")
+	switch {
+	case exact && r.Node.Dir:
+		return nil, nil
+	case r.Node.Dir:
+		return g.loopNodes(r.Node.Nodes, segments, star, nil)
+	default:
+		return g.loopNodes([]*etcdc.Node{r.Node}, segments, false, nil)
+	}
+}
+
+func (g *Backend) ReverseRecord(name string) (*msg.Service, error) {
+	path, star := msg.PathWithWildcard(name)
+	if star {
+		return nil, fmt.Errorf("reverse can not contain wildcards")
+	}
+	r, err := g.get(path, true)
+	if err != nil {
+		return nil, err
+	}
+	if r.Node.Dir {
+		return nil, fmt.Errorf("reverse must not be a directory")
+	}
+	segments := strings.Split(msg.Path(name), "/")
+	records, err := g.loopNodes([]*etcdc.Node{r.Node}, segments, false, nil)
+	if err != nil {
+		return nil, err
+	}
+	if len(records) != 1 {
+		return nil, fmt.Errorf("must be only one service record")
+	}
+	return &records[0], nil
+}
+
+// get is a wrapper for client.Get that uses SingleInflight to suppress multiple
+// outstanding queries.
+func (g *Backend) get(path string, recursive bool) (*etcdc.Response, error) {
+	resp, err := g.inflight.Do(path, func() (interface{}, error) {
+		r, e := g.client.Get(g.ctx, path, &etcdc.GetOptions{Sort: false, Recursive: recursive})
+		if e != nil {
+			return nil, e
+		}
+		return r, e
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.(*etcdc.Response), err
+}
+
+type bareService struct {
+	Host     string
+	Port     int
+	Priority int
+	Weight   int
+	Text     string
+}
+
+// skydns/local/skydns/east/staging/web
+// skydns/local/skydns/west/production/web
+//
+// skydns/local/skydns/*/*/web
+// skydns/local/skydns/*/web
+
+// loopNodes recursively loops through the nodes and returns all the values. The nodes' keyname
+// will be match against any wildcards when star is true.
+func (g *Backend) loopNodes(ns []*etcdc.Node, nameParts []string, star bool, bx map[bareService]bool) (sx []msg.Service, err error) {
+	if bx == nil {
+		bx = make(map[bareService]bool)
+	}
+Nodes:
+	for _, n := range ns {
+		if n.Dir {
+			nodes, err := g.loopNodes(n.Nodes, nameParts, star, bx)
+			if err != nil {
+				return nil, err
+			}
+			sx = append(sx, nodes...)
+			continue
+		}
+		if star {
+			keyParts := strings.Split(n.Key, "/")
+			for i, n := range nameParts {
+				if i > len(keyParts)-1 {
+					// name is longer than key
+					continue Nodes
+				}
+				if n == "*" || n == "any" {
+					continue
+				}
+				if keyParts[i] != n {
+					continue Nodes
+				}
+			}
+		}
+		serv := new(msg.Service)
+		if err := json.Unmarshal([]byte(n.Value), serv); err != nil {
+			return nil, err
+		}
+		b := bareService{serv.Host, serv.Port, serv.Priority, serv.Weight, serv.Text}
+		if _, ok := bx[b]; ok {
+			continue
+		}
+		bx[b] = true
+
+		serv.Key = n.Key
+		serv.Ttl = g.calculateTtl(n, serv)
+		if serv.Priority == 0 {
+			serv.Priority = int(g.config.Priority)
+		}
+		sx = append(sx, *serv)
+	}
+	return sx, nil
+}
+
+// calculateTtl returns the smaller of the etcd TTL and the service's
+// TTL. If neither of these are set (have a zero value), the server
+// default is used.
+func (g *Backend) calculateTtl(node *etcdc.Node, serv *msg.Service) uint32 {
+	etcdTtl := uint32(node.TTL)
+
+	if etcdTtl == 0 && serv.Ttl == 0 {
+		return g.config.Ttl
+	}
+	if etcdTtl == 0 {
+		return serv.Ttl
+	}
+	if serv.Ttl == 0 {
+		return etcdTtl
+	}
+	if etcdTtl < serv.Ttl {
+		return etcdTtl
+	}
+	return serv.Ttl
+}
+
+// Client exposes the underlying Etcd client (used in tests).
+func (g *Backend) Client() etcdc.KeysAPI {
+	return g.client
+}

middleware/etcd/etcd.go

@@ -0,0 +1,34 @@
+// Package etcd provides the etcd backend.
+package etcd
+
+import (
+	"github.com/skynetservices/skydns/singleflight"
+
+	etcd "github.com/coreos/etcd/client"
+	"golang.org/x/net/context"
+)
+
+type (
+	Etcd struct {
+		Ttl      uint32
+		Priority uint16
+		Backend  *Backend
+	}
+)
+
+type Backend struct {
+	client   etcd.KeysAPI
+	ctx      context.Context
+	config   *Config
+	inflight *singleflight.Group
+}
+
+// NewBackend returns a new Backend.
+func NewBackend(client etcd.KeysAPI, ctx context.Context, config *Config) *Backend {
+	return &Backend{
+		client:   client,
+		ctx:      ctx,
+		config:   config,
+		inflight: &singleflight.Group{},
+	}
+}

middleware/etcd/etcd.md

@@ -0,0 +1,26 @@
+# etcd
+
+`etcd` enabled reading zone data from an etcd instance. The data in etcd has to be encoded as
+a [message](https://github.com/skynetservices/skydns/blob/2fcff74cdc9f9a7dd64189a447ef27ac354b725f/msg/service.go#L26)
+like SkyDNS.
+
+## Syntax
+
+~~~
+etcd [address...]
+~~~
+
+* `address` is the endpoint of etcd.
+
+The will default to `/skydns` as the path and the local etcd proxy (http://127.0.0.1:2379).
+
+~~~
+etcd {
+    round_robin
+    path /skydns
+    address address...
+    stubzones
+}
+~~~
+
+## Examples

middleware/etcd/msg/service.go

@@ -0,0 +1,212 @@
+package msg
+
+import (
+	"net"
+	"path"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+// PathPrefix is the prefix used to store CoreDNS data in etcd.
+var PathPrefix string = "skydns"
+
+// This *is* the rdata from a SRV record, but with a twist.
+// Host (Target in SRV) must be a domain name, but if it looks like an IP
+// address (4/6), we will treat it like an IP address.
+type Service struct {
+	Host     string `json:"host,omitempty"`
+	Port     int    `json:"port,omitempty"`
+	Priority int    `json:"priority,omitempty"`
+	Weight   int    `json:"weight,omitempty"`
+	Text     string `json:"text,omitempty"`
+	Mail     bool   `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
+	Ttl      uint32 `json:"ttl,omitempty"`
+
+	// When a SRV record with a "Host: IP-address" is added, we synthesize
+	// a srv.Target domain name.  Normally we convert the full Key where
+	// the record lives to a DNS name and use this as the srv.Target.  When
+	// TargetStrip > 0 we strip the left most TargetStrip labels from the
+	// DNS name.
+	TargetStrip int `json:"targetstrip,omitempty"`
+
+	// Group is used to group (or *not* to group) different services
+	// together. Services with an identical Group are returned in the same
+	// answer.
+	Group string `json:"group,omitempty"`
+
+	// Etcd key where we found this service and ignored from json un-/marshalling
+	Key string `json:"-"`
+}
+
+// NewSRV returns a new SRV record based on the Service.
+func (s *Service) NewSRV(name string, weight uint16) *dns.SRV {
+	host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip)
+
+	return &dns.SRV{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeSRV, Class: dns.ClassINET, Ttl: s.Ttl},
+		Priority: uint16(s.Priority), Weight: weight, Port: uint16(s.Port), Target: host}
+}
+
+// NewMX returns a new MX record based on the Service.
+func (s *Service) NewMX(name string) *dns.MX {
+	host := targetStrip(dns.Fqdn(s.Host), s.TargetStrip)
+
+	return &dns.MX{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: s.Ttl},
+		Preference: uint16(s.Priority), Mx: host}
+}
+
+// NewA returns a new A record based on the Service.
+func (s *Service) NewA(name string, ip net.IP) *dns.A {
+	return &dns.A{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.Ttl}, A: ip}
+}
+
+// NewAAAA returns a new AAAA record based on the Service.
+func (s *Service) NewAAAA(name string, ip net.IP) *dns.AAAA {
+	return &dns.AAAA{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.Ttl}, AAAA: ip}
+}
+
+// NewCNAME returns a new CNAME record based on the Service.
+func (s *Service) NewCNAME(name string, target string) *dns.CNAME {
+	return &dns.CNAME{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: s.Ttl}, Target: target}
+}
+
+// NewNS returns a new NS record based on the Service.
+func (s *Service) NewNS(name string, target string) *dns.NS {
+	return &dns.NS{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeNS, Class: dns.ClassINET, Ttl: s.Ttl}, Ns: target}
+}
+
+// NewTXT returns a new TXT record based on the Service.
+func (s *Service) NewTXT(name string) *dns.TXT {
+	return &dns.TXT{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: s.Ttl}, Txt: split255(s.Text)}
+}
+
+// NewPTR returns a new PTR record based on the Service.
+func (s *Service) NewPTR(name string, ttl uint32) *dns.PTR {
+	return &dns.PTR{Hdr: dns.RR_Header{Name: name, Rrtype: dns.TypePTR, Class: dns.ClassINET, Ttl: ttl}, Ptr: dns.Fqdn(s.Host)}
+}
+
+// As Path, but if a name contains wildcards (* or any), the name will be
+// chopped of before the (first) wildcard, and we do a highler evel search and
+// later find the matching names.  So service.*.skydns.local, will look for all
+// services under skydns.local and will later check for names that match
+// service.*.skydns.local.  If a wildcard is found the returned bool is true.
+func PathWithWildcard(s string) (string, bool) {
+	l := dns.SplitDomainName(s)
+	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	for i, k := range l {
+		if k == "*" || k == "any" {
+			return path.Join(append([]string{"/" + PathPrefix + "/"}, l[:i]...)...), true
+		}
+	}
+	return path.Join(append([]string{"/" + PathPrefix + "/"}, l...)...), false
+}
+
+// Path converts a domainname to an etcd path. If s looks like service.staging.skydns.local.,
+// the resulting key will be /skydns/local/skydns/staging/service .
+func Path(s string) string {
+	l := dns.SplitDomainName(s)
+	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	return path.Join(append([]string{"/" + PathPrefix + "/"}, l...)...)
+}
+
+// Domain is the opposite of Path.
+func Domain(s string) string {
+	l := strings.Split(s, "/")
+	// start with 1, to strip /skydns
+	for i, j := 1, len(l)-1; i < j; i, j = i+1, j-1 {
+		l[i], l[j] = l[j], l[i]
+	}
+	return dns.Fqdn(strings.Join(l[1:len(l)-1], "."))
+}
+
+// Group checks the services in sx, it looks for a Group attribute on the shortest
+// keys. If there are multiple shortest keys *and* the group attribute disagrees (and
+// is not empty), we don't consider it a group.
+// If a group is found, only services with *that* group (or no group) will be returned.
+func Group(sx []Service) []Service {
+	if len(sx) == 0 {
+		return sx
+	}
+
+	// Shortest key with group attribute sets the group for this set.
+	group := sx[0].Group
+	slashes := strings.Count(sx[0].Key, "/")
+	length := make([]int, len(sx))
+	for i, s := range sx {
+		x := strings.Count(s.Key, "/")
+		length[i] = x
+		if x < slashes {
+			if s.Group == "" {
+				break
+			}
+			slashes = x
+			group = s.Group
+		}
+	}
+
+	if group == "" {
+		return sx
+	}
+
+	ret := []Service{} // with slice-tricks in sx we can prolly save this allocation (TODO)
+
+	for i, s := range sx {
+		if s.Group == "" {
+			ret = append(ret, s)
+			continue
+		}
+
+		// Disagreement on the same level
+		if length[i] == slashes && s.Group != group {
+			return sx
+		}
+
+		if s.Group == group {
+			ret = append(ret, s)
+		}
+	}
+	return ret
+}
+
+// Split255 splits a string into 255 byte chunks.
+func split255(s string) []string {
+	if len(s) < 255 {
+		return []string{s}
+	}
+	sx := []string{}
+	p, i := 0, 255
+	for {
+		if i <= len(s) {
+			sx = append(sx, s[p:i])
+		} else {
+			sx = append(sx, s[p:])
+			break
+
+		}
+		p, i = p+255, i+255
+	}
+
+	return sx
+}
+
+// targetStrip strips "targetstrip" labels from the left side of the fully qualified name.
+func targetStrip(name string, targetStrip int) string {
+	if targetStrip == 0 {
+		return name
+	}
+
+	offset, end := 0, false
+	for i := 0; i < targetStrip; i++ {
+		offset, end = dns.NextLabel(name, offset)
+	}
+	if end {
+		// We overshot the name, use the orignal one.
+		offset = 0
+	}
+	name = name[offset:]
+	return name
+}

middleware/etcd/singleflight/singleflight.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2012 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package singleflight provides a duplicate function call suppression
+// mechanism.
+package singleflight
+
+import "sync"
+
+// call is an in-flight or completed Do call
+type call struct {
+	wg  sync.WaitGroup
+	val interface{}
+	err error
+}
+
+// Group represents a class of work and forms a namespace in which
+// units of work can be executed with duplicate suppression.
+type Group struct {
+	mu sync.Mutex       // protects m
+	m  map[string]*call // lazily initialized
+}
+
+// Do executes and returns the results of the given function, making
+// sure that only one execution is in-flight for a given key at a
+// time. If a duplicate comes in, the duplicate caller waits for the
+// original to complete and receives the same results.
+func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) {
+	g.mu.Lock()
+	if g.m == nil {
+		g.m = make(map[string]*call)
+	}
+	if c, ok := g.m[key]; ok {
+		g.mu.Unlock()
+		c.wg.Wait()
+		return c.val, c.err
+	}
+	c := new(call)
+	c.wg.Add(1)
+	g.m[key] = c
+	g.mu.Unlock()
+
+	c.val, c.err = fn()
+	c.wg.Done()
+
+	g.mu.Lock()
+	delete(g.m, key)
+	g.mu.Unlock()
+
+	return c.val, c.err
+}