Cleanup: put middleware helper functions in pkgs (#245)

Move all (almost all) Go files in middleware into their
own packages. This makes for better naming and discoverability.

Lot of changes elsewhere to make this change.

The middleware.State was renamed to request.Request which is better,
but still does not cover all use-cases. It was also moved out middleware
because it is used by `dnsserver` as well.

A pkg/dnsutil packages was added for shared, handy, dns util functions.

All normalize functions are now put in normalize.go

middleware/file/delegation_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/test"
 
 	"github.com/miekg/dns"
@@ -57,14 +57,14 @@ func TestLookupDelegation(t *testing.T) {
 	for _, tc := range delegationTestCases {
 		m := tc.Msg()
 
-		rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+		rec := dnsrecorder.New(&test.ResponseWriter{})
 		_, err := fm.ServeDNS(ctx, rec, m)
 		if err != nil {
 			t.Errorf("expected no error, got %v\n", err)
 			return
 		}
-		resp := rec.Msg()
 
+		resp := rec.Msg
 		sort.Sort(test.RRSet(resp.Answer))
 		sort.Sort(test.RRSet(resp.Ns))
 		sort.Sort(test.RRSet(resp.Extra))

middleware/file/dnssec_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/test"
 
 	"github.com/miekg/dns"
@@ -116,14 +116,14 @@ func TestLookupDNSSEC(t *testing.T) {
 	for _, tc := range dnssecTestCases {
 		m := tc.Msg()
 
-		rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+		rec := dnsrecorder.New(&test.ResponseWriter{})
 		_, err := fm.ServeDNS(ctx, rec, m)
 		if err != nil {
 			t.Errorf("expected no error, got %v\n", err)
 			return
 		}
-		resp := rec.Msg()
 
+		resp := rec.Msg
 		sort.Sort(test.RRSet(resp.Answer))
 		sort.Sort(test.RRSet(resp.Ns))
 		sort.Sort(test.RRSet(resp.Extra))
@@ -154,7 +154,7 @@ func BenchmarkLookupDNSSEC(b *testing.B) {
 
 	fm := File{Next: test.ErrorHandler(), Zones: Zones{Z: map[string]*Zone{testzone: zone}, Names: []string{testzone}}}
 	ctx := context.TODO()
-	rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+	rec := dnsrecorder.New(&test.ResponseWriter{})
 
 	tc := test.Case{
 		Qname: "b.miek.nl.", Qtype: dns.TypeA, Do: true,

middleware/file/ent_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/test"
 
 	"github.com/miekg/dns"
@@ -43,14 +43,14 @@ func TestLookupENT(t *testing.T) {
 	for _, tc := range entTestCases {
 		m := tc.Msg()
 
-		rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+		rec := dnsrecorder.New(&test.ResponseWriter{})
 		_, err := fm.ServeDNS(ctx, rec, m)
 		if err != nil {
 			t.Errorf("expected no error, got %v\n", err)
 			return
 		}
-		resp := rec.Msg()
 
+		resp := rec.Msg
 		sort.Sort(test.RRSet(resp.Answer))
 		sort.Sort(test.RRSet(resp.Ns))
 		sort.Sort(test.RRSet(resp.Extra))

middleware/file/file.go

@@ -6,6 +6,7 @@ import (
 	"log"
 
 	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/request"
 
 	"github.com/miekg/dns"
 	"golang.org/x/net/context"
@@ -24,7 +25,7 @@ type (
 )
 
 func (f File) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
-	state := middleware.State{W: w, Req: r}
+	state := request.Request{W: w, Req: r}
 
 	if state.QClass() != dns.ClassINET {
 		return dns.RcodeServerFailure, errors.New("can only deal with ClassINET")

middleware/file/lookup_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/test"
 
 	"github.com/miekg/dns"
@@ -87,14 +87,14 @@ func TestLookup(t *testing.T) {
 	for _, tc := range dnsTestCases {
 		m := tc.Msg()
 
-		rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+		rec := dnsrecorder.New(&test.ResponseWriter{})
 		_, err := fm.ServeDNS(ctx, rec, m)
 		if err != nil {
 			t.Errorf("expected no error, got %v\n", err)
 			return
 		}
-		resp := rec.Msg()
 
+		resp := rec.Msg
 		sort.Sort(test.RRSet(resp.Answer))
 		sort.Sort(test.RRSet(resp.Ns))
 		sort.Sort(test.RRSet(resp.Extra))
@@ -122,7 +122,7 @@ func TestLookupNil(t *testing.T) {
 	ctx := context.TODO()
 
 	m := dnsTestCases[0].Msg()
-	rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+	rec := dnsrecorder.New(&test.ResponseWriter{})
 	fm.ServeDNS(ctx, rec, m)
 }
 
@@ -134,7 +134,7 @@ func BenchmarkLookup(b *testing.B) {
 
 	fm := File{Next: test.ErrorHandler(), Zones: Zones{Z: map[string]*Zone{testzone: zone}, Names: []string{testzone}}}
 	ctx := context.TODO()
-	rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+	rec := dnsrecorder.New(&test.ResponseWriter{})
 
 	tc := test.Case{
 		Qname: "www.miek.nl.", Qtype: dns.TypeA,

middleware/file/notify.go

@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/request"
 
 	"github.com/miekg/dns"
 )
@@ -12,7 +13,7 @@ import (
 // isNotify checks if state is a notify message and if so, will *also* check if it
 // is from one of the configured masters. If not it will not be a valid notify
 // message. If the zone z is not a secondary zone the message will also be ignored.
-func (z *Zone) isNotify(state middleware.State) bool {
+func (z *Zone) isNotify(state request.Request) bool {
 	if state.Req.Opcode != dns.OpcodeNotify {
 		return false
 	}
@@ -56,7 +57,7 @@ func notify(zone string, to []string) error {
 
 func notifyAddr(c *dns.Client, m *dns.Msg, s string) error {
 	for i := 0; i < 3; i++ {
-		ret, err := middleware.Exchange(c, m, s)
+		ret, _, err := c.Exchange(m, s)
 		if err != nil {
 			continue
 		}

middleware/file/secondary.go

@@ -4,8 +4,6 @@ import (
 	"log"
 	"time"
 
-	"github.com/miekg/coredns/middleware"
-
 	"github.com/miekg/dns"
 )
 
@@ -75,7 +73,7 @@ func (z *Zone) shouldTransfer() (bool, error) {
 Transfer:
 	for _, tr := range z.TransferFrom {
 		Err = nil
-		ret, err := middleware.Exchange(c, m, tr)
+		ret, _, err := c.Exchange(m, tr)
 		if err != nil || ret.Rcode != dns.RcodeSuccess {
 			Err = err
 			continue

middleware/file/secondary_test.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
 	"github.com/miekg/coredns/middleware/test"
+	"github.com/miekg/coredns/request"
 
 	"github.com/miekg/dns"
 )
@@ -134,7 +134,7 @@ func TestIsNotify(t *testing.T) {
 	z := new(Zone)
 	z.Expired = new(bool)
 	z.origin = testZone
-	state := NewState(testZone, dns.TypeSOA)
+	state := newRequest(testZone, dns.TypeSOA)
 	// need to set opcode
 	state.Req.Opcode = dns.OpcodeNotify
 
@@ -148,9 +148,9 @@ func TestIsNotify(t *testing.T) {
 	}
 }
 
-func NewState(zone string, qtype uint16) middleware.State {
+func newRequest(zone string, qtype uint16) request.Request {
 	m := new(dns.Msg)
 	m.SetQuestion("example.com.", dns.TypeA)
 	m.SetEdns0(4097, true)
-	return middleware.State{W: &test.ResponseWriter{}, Req: m}
+	return request.Request{W: &test.ResponseWriter{}, Req: m}
 }

middleware/file/tree/elem.go

@@ -1,9 +1,6 @@
 package tree
 
-import (
-	"github.com/miekg/coredns/middleware"
-	"github.com/miekg/dns"
-)
+import "github.com/miekg/dns"
 
 type Elem struct {
 	m map[uint16][]dns.RR
@@ -91,8 +88,8 @@ func (e *Elem) Delete(rr dns.RR) (empty bool) {
 	return
 }
 
-// Less is a tree helper function that calls middleware.Less.
-func Less(a *Elem, name string) int { return middleware.Less(name, a.Name()) }
+// Less is a tree helper function that calls less.
+func Less(a *Elem, name string) int { return less(name, a.Name()) }
 
 // Assuming the same type and name this will check if the rdata is equal as well.
 func equalRdata(a, b dns.RR) bool {

middleware/file/tree/less.go

@@ -0,0 +1,59 @@
+package tree
+
+import (
+	"bytes"
+
+	"github.com/miekg/dns"
+)
+
+// less returns <0 when a is less than b, 0 when they are equal and
+// >0 when a is larger than b.
+// The function orders names in DNSSEC canonical order: RFC 4034s section-6.1
+//
+// See http://bert-hubert.blogspot.co.uk/2015/10/how-to-do-fast-canonical-ordering-of.html
+// for a blog article on this implementation.
+//
+// The values of a and b are *not* lowercased before the comparison!
+func less(a, b string) int {
+	i := 1
+	aj := len(a)
+	bj := len(b)
+	for {
+		ai, oka := dns.PrevLabel(a, i)
+		bi, okb := dns.PrevLabel(b, i)
+		if oka && okb {
+			return 0
+		}
+		// sadly this []byte will allocate... TODO(miek): check if this is needed
+		// for a name, otherwise compare the strings.
+		ab := []byte(a[ai:aj])
+		bb := []byte(b[bi:bj])
+		doDDD(ab)
+		doDDD(bb)
+
+		res := bytes.Compare(ab, bb)
+		if res != 0 {
+			return res
+		}
+
+		i++
+		aj, bj = ai, bi
+	}
+	return 0
+}
+
+func doDDD(b []byte) {
+	lb := len(b)
+	for i := 0; i < lb; i++ {
+		if i+3 < lb && b[i] == '\\' && isDigit(b[i+1]) && isDigit(b[i+2]) && isDigit(b[i+3]) {
+			b[i] = dddToByte(b[i:])
+			for j := i + 1; j < lb-3; j++ {
+				b[j] = b[j+3]
+			}
+			lb -= 3
+		}
+	}
+}
+
+func isDigit(b byte) bool     { return b >= '0' && b <= '9' }
+func dddToByte(s []byte) byte { return (s[1]-'0')*100 + (s[2]-'0')*10 + (s[3] - '0') }

middleware/file/tree/less_test.go

@@ -0,0 +1,81 @@
+package tree
+
+import (
+	"sort"
+	"strings"
+	"testing"
+)
+
+type set []string
+
+func (p set) Len() int           { return len(p) }
+func (p set) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p set) Less(i, j int) bool { d := less(p[i], p[j]); return d <= 0 }
+
+func TestLess(t *testing.T) {
+	tests := []struct {
+		in  []string
+		out []string
+	}{
+		{
+			[]string{"aaa.powerdns.de", "bbb.powerdns.net.", "xxx.powerdns.com."},
+			[]string{"xxx.powerdns.com.", "aaa.powerdns.de", "bbb.powerdns.net."},
+		},
+		{
+			[]string{"aaa.POWERDNS.de", "bbb.PoweRdnS.net.", "xxx.powerdns.com."},
+			[]string{"xxx.powerdns.com.", "aaa.POWERDNS.de", "bbb.PoweRdnS.net."},
+		},
+		{
+			[]string{"aaa.aaaa.aa.", "aa.aaa.a.", "bbb.bbbb.bb."},
+			[]string{"aa.aaa.a.", "aaa.aaaa.aa.", "bbb.bbbb.bb."},
+		},
+		{
+			[]string{"aaaaa.", "aaa.", "bbb."},
+			[]string{"aaa.", "aaaaa.", "bbb."},
+		},
+		{
+			[]string{"a.a.a.a.", "a.a.", "a.a.a."},
+			[]string{"a.a.", "a.a.a.", "a.a.a.a."},
+		},
+		{
+			[]string{"example.", "z.example.", "a.example."},
+			[]string{"example.", "a.example.", "z.example."},
+		},
+		{
+			[]string{"a.example.", "Z.a.example.", "z.example.", "yljkjljk.a.example.", "\\001.z.example.", "example.", "*.z.example.", "\\200.z.example.", "zABC.a.EXAMPLE."},
+			[]string{"example.", "a.example.", "yljkjljk.a.example.", "Z.a.example.", "zABC.a.EXAMPLE.", "z.example.", "\\001.z.example.", "*.z.example.", "\\200.z.example."},
+		},
+		{
+			// RFC3034 example.
+			[]string{"a.example.", "Z.a.example.", "z.example.", "yljkjljk.a.example.", "example.", "*.z.example.", "zABC.a.EXAMPLE."},
+			[]string{"example.", "a.example.", "yljkjljk.a.example.", "Z.a.example.", "zABC.a.EXAMPLE.", "z.example.", "*.z.example."},
+		},
+	}
+
+Tests:
+	for j, test := range tests {
+		// Need to lowercase these example as the Less function does lowercase for us anymore.
+		for i, b := range test.in {
+			test.in[i] = strings.ToLower(b)
+		}
+		for i, b := range test.out {
+			test.out[i] = strings.ToLower(b)
+		}
+
+		sort.Sort(set(test.in))
+		for i := 0; i < len(test.in); i++ {
+			if test.in[i] != test.out[i] {
+				t.Errorf("test %d: expected %s, got %s\n", j, test.out[i], test.in[i])
+				n := ""
+				for k, in := range test.in {
+					if k+1 == len(test.in) {
+						n = "\n"
+					}
+					t.Logf("%s <-> %s\n%s", in, test.out[k], n)
+				}
+				continue Tests
+			}
+
+		}
+	}
+}

middleware/file/wildcard_test.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/middleware/pkg/dnsrecorder"
 	"github.com/miekg/coredns/middleware/test"
 
 	"github.com/miekg/dns"
@@ -57,14 +57,14 @@ func TestLookupWildcard(t *testing.T) {
 	for _, tc := range wildcardTestCases {
 		m := tc.Msg()
 
-		rec := middleware.NewResponseRecorder(&test.ResponseWriter{})
+		rec := dnsrecorder.New(&test.ResponseWriter{})
 		_, err := fm.ServeDNS(ctx, rec, m)
 		if err != nil {
 			t.Errorf("expected no error, got %v\n", err)
 			return
 		}
-		resp := rec.Msg()
 
+		resp := rec.Msg
 		sort.Sort(test.RRSet(resp.Answer))
 		sort.Sort(test.RRSet(resp.Ns))
 		sort.Sort(test.RRSet(resp.Extra))

middleware/file/xfr.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/miekg/coredns/middleware"
+	"github.com/miekg/coredns/request"
 
 	"github.com/miekg/dns"
 	"golang.org/x/net/context"
@@ -18,7 +18,7 @@ type (
 
 // Serve an AXFR (and fallback of IXFR) as well.
 func (x Xfr) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
-	state := middleware.State{W: w, Req: r}
+	state := request.Request{W: w, Req: r}
 	if !x.TransferAllowed(state) {
 		return dns.RcodeServerFailure, nil
 	}

middleware/file/zone.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/miekg/coredns/middleware"
 	"github.com/miekg/coredns/middleware/file/tree"
+	"github.com/miekg/coredns/request"
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/miekg/dns"
@@ -102,7 +102,7 @@ func (z *Zone) Insert(r dns.RR) error {
 func (z *Zone) Delete(r dns.RR) { z.Tree.Delete(r) }
 
 // TransferAllowed checks if incoming request for transferring the zone is allowed according to the ACLs.
-func (z *Zone) TransferAllowed(state middleware.State) bool {
+func (z *Zone) TransferAllowed(req request.Request) bool {
 	for _, t := range z.TransferTo {
 		if t == "*" {
 			return true