plugin/metadata: add metadata plugin (#1894)

* plugin/metadata: add metadata plugin

* plugin/metadata: Add MD struct, refactor code, fix doc

* plugin/metadata: simplify metadata key

* plugin/metadata: improve setup_test

* Support of metadata by rewrite plugin. Move calculated variables to metadata.

* Move variables from metadata to pkg, add UTs, READMEs change, metadata small fixes

* Add client port validation to variables_test

* plugin/metadata: improve README

* plugin/metadata: rename methods

* plugin/metadata: Update Metadataer interface, update doc, cosmetic code changes

* plugin/metadata: move colllisions check to OnStartup(). Fix default variables metadataer.

* plugin/metadata: Fix comment for method setValue

* plugin/metadata: change variables order to fix linter warning

* plugin/metadata: rename Metadataer to Provider

plugin/metadata/README.md

@@ -0,0 +1,47 @@
+# metadata
+
+## Name
+
+*metadata* - enable a metadata collector.
+
+## Description
+
+By enabling *metadata* any plugin that implements [metadata.Provider interface](https://godoc.org/github.com/coredns/coredns/plugin/metadata#Provider) will be called for each DNS query, at being of the process for that query, in order to add it's own Metadata to context. The metadata collected will be available for all plugins handler, via the Context parameter provided in the ServeDNS function.
+Metadata plugin is automatically adding the so-called default medatada (extracted from the query) to the context. Those default metadata are: {qname}, {qtype}, {client_ip}, {client_port}, {protocol}, {server_ip}, {server_port}
+
+
+## Syntax
+
+~~~
+metadata [ZONES... ]
+~~~
+
+## Plugins
+
+metadata.Provider interface needs to be implemented by each plugin willing to provide metadata information for other plugins. It will be called by metadata and gather the information from all plugins in context.
+Note: this method should work quickly, because it is called for every request
+from the metadata plugin.
+If **ZONES** is specified then metadata add is limited by zones. Metadata is added to every context going through metadata.Provider if **ZONES** are not specified.
+
+
+## Examples
+
+Enable metadata for all requests. Rewrite uses one of the provided by default metadata variables.
+
+~~~ corefile
+. {
+    metadata
+    rewrite edns0 local set 0xffee {client_ip}
+    forward . 8.8.8.8:53
+}
+~~~
+
+Add metadata for all requests within `example.org.`. Rewrite uses one of provided by default metadata variables. Any other requests won't have metadata.
+
+~~~ corefile
+. {
+    metadata example.org
+    rewrite edns0 local set 0xffee {client_ip}
+    forward . 8.8.8.8:53
+}
+~~~

plugin/metadata/metadata.go

@@ -0,0 +1,55 @@
+package metadata
+
+import (
+	"context"
+
+	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/pkg/variables"
+	"github.com/coredns/coredns/request"
+
+	"github.com/miekg/dns"
+)
+
+// Metadata implements collecting metadata information from all plugins that
+// implement the Provider interface.
+type Metadata struct {
+	Zones     []string
+	Providers []Provider
+	Next      plugin.Handler
+}
+
+// Name implements the Handler interface.
+func (m *Metadata) Name() string { return "metadata" }
+
+// ServeDNS implements the plugin.Handler interface.
+func (m *Metadata) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+
+	md, ctx := newMD(ctx)
+
+	state := request.Request{W: w, Req: r}
+	if plugin.Zones(m.Zones).Matches(state.Name()) != "" {
+		// Go through all Providers and collect metadata
+		for _, provider := range m.Providers {
+			for _, varName := range provider.MetadataVarNames() {
+				if val, ok := provider.Metadata(ctx, w, r, varName); ok {
+					md.setValue(varName, val)
+				}
+			}
+		}
+	}
+
+	rcode, err := plugin.NextOrFailure(m.Name(), m.Next, ctx, w, r)
+
+	return rcode, err
+}
+
+// MetadataVarNames implements the plugin.Provider interface.
+func (m *Metadata) MetadataVarNames() []string { return variables.All }
+
+// Metadata implements the plugin.Provider interface.
+func (m *Metadata) Metadata(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, varName string) (interface{}, bool) {
+	if val, err := variables.GetValue(varName, w, r); err == nil {
+		return val, true
+	}
+	return nil, false
+}

plugin/metadata/metadata_test.go

@@ -0,0 +1,79 @@
+package metadata
+
+import (
+	"context"
+	"testing"
+
+	"github.com/coredns/coredns/plugin/test"
+
+	"github.com/miekg/dns"
+)
+
+// testProvider implements fake Providers. Plugins which inmplement Provider interface
+type testProvider map[string]interface{}
+
+func (m testProvider) MetadataVarNames() []string {
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func (m testProvider) Metadata(ctx context.Context, w dns.ResponseWriter, r *dns.Msg, key string) (val interface{}, ok bool) {
+	value, ok := m[key]
+	return value, ok
+}
+
+// testHandler implements plugin.Handler
+type testHandler struct{ ctx context.Context }
+
+func (m *testHandler) Name() string { return "testHandler" }
+
+func (m *testHandler) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	m.ctx = ctx
+	return 0, nil
+}
+
+func TestMetadataServDns(t *testing.T) {
+	expectedMetadata := []testProvider{
+		testProvider{"testkey1": "testvalue1"},
+		testProvider{"testkey2": 2, "testkey3": "testvalue3"},
+	}
+	// Create fake Providers based on expectedMetadata
+	providers := []Provider{}
+	for _, e := range expectedMetadata {
+		providers = append(providers, e)
+	}
+	// Fake handler which stores the resulting context
+	next := &testHandler{}
+
+	metadata := Metadata{
+		Zones:     []string{"."},
+		Providers: providers,
+		Next:      next,
+	}
+	metadata.ServeDNS(context.TODO(), &test.ResponseWriter{}, new(dns.Msg))
+
+	// Verify that next plugin can find metadata in context from all Providers
+	for _, expected := range expectedMetadata {
+		md, ok := FromContext(next.ctx)
+		if !ok {
+			t.Fatalf("Metadata is expected but not present inside the context")
+		}
+		for expKey, expVal := range expected {
+			metadataVal, valOk := md.Value(expKey)
+			if !valOk {
+				t.Fatalf("Value by key %v can't be retrieved", expKey)
+			}
+			if metadataVal != expVal {
+				t.Errorf("Expected value %v, but got %v", expVal, metadataVal)
+			}
+		}
+		wrongKey := "wrong_key"
+		metadataVal, ok := md.Value(wrongKey)
+		if ok {
+			t.Fatalf("Value by key %v is not expected to be recieved, but got: %v", wrongKey, metadataVal)
+		}
+	}
+}

plugin/metadata/metadataer.go

@@ -0,0 +1,53 @@
+package metadata
+
+import (
+	"context"
+
+	"github.com/miekg/dns"
+)
+
+// Provider interface needs to be implemented by each plugin willing to provide
+// metadata information for other plugins.
+// Note: this method should work quickly, because it is called for every request
+// from the metadata plugin.
+type Provider interface {
+	// List of variables which are provided by current Provider. Must remain constant.
+	MetadataVarNames() []string
+	// Metadata is expected to return a value with metadata information by the key
+	// from 4th argument. Value can be later retrieved from context by any other plugin.
+	// If value is not available by some reason returned boolean value should be false.
+	Metadata(context.Context, dns.ResponseWriter, *dns.Msg, string) (interface{}, bool)
+}
+
+// MD is metadata information storage
+type MD map[string]interface{}
+
+// metadataKey defines the type of key that is used to save metadata into the context
+type metadataKey struct{}
+
+// newMD initializes MD and attaches it to context
+func newMD(ctx context.Context) (MD, context.Context) {
+	m := MD{}
+	return m, context.WithValue(ctx, metadataKey{}, m)
+}
+
+// FromContext retrieves MD struct from context.
+func FromContext(ctx context.Context) (md MD, ok bool) {
+	if metadata := ctx.Value(metadataKey{}); metadata != nil {
+		if md, ok := metadata.(MD); ok {
+			return md, true
+		}
+	}
+	return MD{}, false
+}
+
+// Value returns metadata value by key.
+func (m MD) Value(key string) (value interface{}, ok bool) {
+	value, ok = m[key]
+	return value, ok
+}
+
+// setValue adds metadata value.
+func (m MD) setValue(key string, val interface{}) {
+	m[key] = val
+}

plugin/metadata/metadataer_test.go

@@ -0,0 +1,47 @@
+package metadata
+
+import (
+	"context"
+	"reflect"
+	"testing"
+)
+
+func TestMD(t *testing.T) {
+	tests := []struct {
+		addValues      map[string]interface{}
+		expectedValues map[string]interface{}
+	}{
+		{
+			// Add initial metadata key/vals
+			map[string]interface{}{"key1": "val1", "key2": 2},
+			map[string]interface{}{"key1": "val1", "key2": 2},
+		},
+		{
+			// Add additional key/vals.
+			map[string]interface{}{"key3": 3, "key4": 4.5},
+			map[string]interface{}{"key1": "val1", "key2": 2, "key3": 3, "key4": 4.5},
+		},
+	}
+
+	// Using one same md and ctx for all test cases
+	ctx := context.TODO()
+	md, ctx := newMD(ctx)
+
+	for i, tc := range tests {
+		for k, v := range tc.addValues {
+			md.setValue(k, v)
+		}
+		if !reflect.DeepEqual(tc.expectedValues, map[string]interface{}(md)) {
+			t.Errorf("Test %d: Expected %v but got %v", i, tc.expectedValues, md)
+		}
+
+		// Make sure that MD is recieved from context successfullly
+		mdFromContext, ok := FromContext(ctx)
+		if !ok {
+			t.Errorf("Test %d: MD is not recieved from the context", i)
+		}
+		if !reflect.DeepEqual(md, mdFromContext) {
+			t.Errorf("Test %d: MD recieved from context differs from initial. Initial: %v, from context: %v", i, md, mdFromContext)
+		}
+	}
+}

plugin/metadata/setup.go

@@ -0,0 +1,71 @@
+package metadata
+
+import (
+	"fmt"
+
+	"github.com/coredns/coredns/core/dnsserver"
+	"github.com/coredns/coredns/plugin"
+
+	"github.com/mholt/caddy"
+)
+
+func init() {
+	caddy.RegisterPlugin("metadata", caddy.Plugin{
+		ServerType: "dns",
+		Action:     setup,
+	})
+}
+
+func setup(c *caddy.Controller) error {
+	m, err := metadataParse(c)
+	if err != nil {
+		return err
+	}
+	dnsserver.GetConfig(c).AddPlugin(func(next plugin.Handler) plugin.Handler {
+		m.Next = next
+		return m
+	})
+
+	c.OnStartup(func() error {
+		plugins := dnsserver.GetConfig(c).Handlers()
+		// Collect all plugins which implement Provider interface
+		metadataVariables := map[string]bool{}
+		for _, p := range plugins {
+			if met, ok := p.(Provider); ok {
+				for _, varName := range met.MetadataVarNames() {
+					if _, ok := metadataVariables[varName]; ok {
+						return fmt.Errorf("Metadata variable '%v' has duplicates", varName)
+					}
+					metadataVariables[varName] = true
+				}
+				m.Providers = append(m.Providers, met)
+			}
+		}
+		return nil
+	})
+
+	return nil
+}
+
+func metadataParse(c *caddy.Controller) (*Metadata, error) {
+	m := &Metadata{}
+	c.Next()
+	zones := c.RemainingArgs()
+
+	if len(zones) != 0 {
+		m.Zones = zones
+		for i := 0; i < len(m.Zones); i++ {
+			m.Zones[i] = plugin.Host(m.Zones[i]).Normalize()
+		}
+	} else {
+		m.Zones = make([]string, len(c.ServerBlockKeys))
+		for i := 0; i < len(c.ServerBlockKeys); i++ {
+			m.Zones[i] = plugin.Host(c.ServerBlockKeys[i]).Normalize()
+		}
+	}
+
+	if c.NextBlock() || c.Next() {
+		return nil, plugin.Error("metadata", c.ArgErr())
+	}
+	return m, nil
+}

plugin/metadata/setup_test.go

@@ -0,0 +1,70 @@
+package metadata
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/mholt/caddy"
+)
+
+func TestSetup(t *testing.T) {
+	tests := []struct {
+		input     string
+		zones     []string
+		shouldErr bool
+	}{
+		{"metadata", []string{}, false},
+		{"metadata example.com.", []string{"example.com."}, false},
+		{"metadata example.com. net.", []string{"example.com.", "net."}, false},
+
+		{"metadata example.com. { some_param }", []string{}, true},
+		{"metadata\nmetadata", []string{}, true},
+	}
+
+	for i, test := range tests {
+		c := caddy.NewTestController("dns", test.input)
+		err := setup(c)
+
+		if test.shouldErr && err == nil {
+			t.Errorf("Test %d: Setup call expected error but found none for input %s", i, test.input)
+		}
+
+		if !test.shouldErr && err != nil {
+			t.Errorf("Test %d: Setup call expected no error but found one for input %s. Error was: %v", i, test.input, err)
+		}
+	}
+}
+
+func TestSetupHealth(t *testing.T) {
+	tests := []struct {
+		input     string
+		zones     []string
+		shouldErr bool
+	}{
+		{"metadata", []string{}, false},
+		{"metadata example.com.", []string{"example.com."}, false},
+		{"metadata example.com. net.", []string{"example.com.", "net."}, false},
+
+		{"metadata example.com. { some_param }", []string{}, true},
+		{"metadata\nmetadata", []string{}, true},
+	}
+
+	for i, test := range tests {
+		c := caddy.NewTestController("dns", test.input)
+		m, err := metadataParse(c)
+
+		if test.shouldErr && err == nil {
+			t.Errorf("Test %d: Expected error but found none for input %s", i, test.input)
+		}
+
+		if !test.shouldErr && err != nil {
+			t.Errorf("Test %d: Expected no error but found one for input %s. Error was: %v", i, test.input, err)
+		}
+
+		if !test.shouldErr && err == nil {
+			if !reflect.DeepEqual(test.zones, m.Zones) {
+				t.Errorf("Test %d: Expected zones %s. Zones were: %v", i, test.zones, m.Zones)
+			}
+		}
+	}
+}