plugin/k8s_external: Resolve headless services (#5505)

*add option for resolving headless Services without external IPs in k8s_external

Signed-off-by: Tomas Kohout <tomas.kohout1995@gmail.com>

plugin/k8s_external/README.md

@@ -2,12 +2,12 @@
 
 ## Name
 
-*k8s_external* - resolves load balancer and external IPs from outside Kubernetes clusters.
+*k8s_external* - resolves load balancer, external IPs from outside Kubernetes clusters and if enabled headless services.
 
 ## Description
 
 This plugin allows an additional zone to resolve the external IP address(es) of a Kubernetes
-service. This plugin is only useful if the *kubernetes* plugin is also loaded.
+service and headless services. This plugin is only useful if the *kubernetes* plugin is also loaded.
 
 The plugin uses an external zone to resolve in-cluster IP addresses. It only handles queries for A,
 AAAA, SRV, and PTR records; all others result in NODATA responses. To make it a proper DNS zone, it handles
@@ -57,6 +57,16 @@ k8s_external [ZONE...] {
 * **APEX** is the name (DNS label) to use for the apex records; it defaults to `dns`.
 * `ttl` allows you to set a custom **TTL** for responses. The default is 5 (seconds).
 
+If you want to enable headless service resolution, you can do so by adding `headless` option.
+
+~~~
+k8s_external [ZONE...] {
+    headless
+}
+~~~
+
+* if there is a headless service with external IPs set, external IPs will be resolved
+
 ## Examples
 
 Enable names under `example.org` to be resolved to in-cluster DNS addresses.

plugin/k8s_external/apex.go

@@ -18,7 +18,7 @@ func (e *External) serveApex(state request.Request) (int, error) {
 	case dns.TypeNS:
 		m.Answer = []dns.RR{e.ns(state)}
 
-		addr := e.externalAddrFunc(state)
+		addr := e.externalAddrFunc(state, e.headless)
 		for _, rr := range addr {
 			rr.Header().Ttl = e.ttl
 			rr.Header().Name = dnsutil.Join("ns1", e.apex, state.QName())
@@ -58,7 +58,7 @@ func (e *External) serveSubApex(state request.Request) (int, error) {
 			return 0, nil
 		}
 
-		addr := e.externalAddrFunc(state)
+		addr := e.externalAddrFunc(state, e.headless)
 		for _, rr := range addr {
 			rr.Header().Ttl = e.ttl
 			rr.Header().Name = state.QName()

plugin/k8s_external/apex_test.go

@@ -17,6 +17,7 @@ func TestApex(t *testing.T) {
 	k.APIConn = &external{}
 
 	e := New()
+	e.headless = true
 	e.Zones = []string{"example.com."}
 	e.Next = test.NextHandler(dns.RcodeSuccess, nil)
 	e.externalFunc = k.External

plugin/k8s_external/external.go

@@ -26,11 +26,11 @@ import (
 type Externaler interface {
 	// External returns a slice of msg.Services that are looked up in the backend and match
 	// the request.
-	External(request.Request) ([]msg.Service, int)
+	External(request.Request, bool) ([]msg.Service, int)
 	// ExternalAddress should return a string slice of addresses for the nameserving endpoint.
-	ExternalAddress(state request.Request) []dns.RR
+	ExternalAddress(state request.Request, headless bool) []dns.RR
-	// ExternalServices returns all services in the given zone as a slice of msg.Service.
+	// ExternalServices returns all services in the given zone as a slice of msg.Service and if enabled, headless services as a map of services.
-	ExternalServices(zone string) []msg.Service
+	ExternalServices(zone string, headless bool) ([]msg.Service, map[string][]msg.Service)
 	// ExternalSerial gets the current serial.
 	ExternalSerial(string) uint32
 }
@@ -43,13 +43,14 @@ type External struct {
 	hostmaster 	string
 	apex       	string
 	ttl        	uint32
+	headless 	bool
 
 	upstream *upstream.Upstream
 
-	externalFunc         func(request.Request) ([]msg.Service, int)
+	externalFunc         func(request.Request, bool) ([]msg.Service, int)
-	externalAddrFunc     func(request.Request) []dns.RR
+	externalAddrFunc     func(request.Request, bool) []dns.RR
 	externalSerialFunc   func(string) uint32
-	externalServicesFunc func(string) []msg.Service
+	externalServicesFunc func(string, bool) ([]msg.Service, map[string][]msg.Service)
 }
 
 // New returns a new and initialized *External.
@@ -85,7 +86,7 @@ func (e *External) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Ms
 		}
 	}
 
-	svc, rcode := e.externalFunc(state)
+	svc, rcode := e.externalFunc(state, e.headless)
 
 	m := new(dns.Msg)
 	m.SetReply(state.Req)

plugin/k8s_external/external_test.go

@@ -21,6 +21,7 @@ func TestExternal(t *testing.T) {
 
 	e := New()
 	e.Zones = []string{"example.com.", "in-addr.arpa."}
+	e.headless = true
 	e.Next = test.NextHandler(dns.RcodeSuccess, nil)
 	e.externalFunc = k.External
 	e.externalAddrFunc = externalAddress  // internal test function
@@ -216,6 +217,66 @@ var tests = []test.Case{
 			test.SRV("svc12.testns.example.com.	5	IN	SRV	0 100 80 dummy.hostname."),
 		},
 	},
+	// headless service
+	{
+		Qname: "svc-headless.testns.example.com.", Qtype: dns.TypeA, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.A("svc-headless.testns.example.com.	5	IN	A	1.2.3.4"),
+			test.A("svc-headless.testns.example.com.	5	IN	A	1.2.3.5"),
+		},
+	},
+	{
+		Qname: "svc-headless.testns.example.com.", Qtype: dns.TypeSRV, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.SRV("svc-headless.testns.example.com.	5	IN	SRV	0	50	80	endpoint-svc-0.svc-headless.testns.example.com."),
+			test.SRV("svc-headless.testns.example.com.	5	IN	SRV	0	50	80	endpoint-svc-1.svc-headless.testns.example.com."),
+		},
+		Extra: []dns.RR{
+			test.A("endpoint-svc-0.svc-headless.testns.example.com.	5	IN	A	1.2.3.4"),
+			test.A("endpoint-svc-1.svc-headless.testns.example.com.	5	IN	A	1.2.3.5"),
+		},
+	},
+	{
+		Qname: "_http._tcp.svc-headless.testns.example.com.", Qtype: dns.TypeSRV, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.SRV("_http._tcp.svc-headless.testns.example.com.	5	IN	SRV	0	50	80	endpoint-svc-0.svc-headless.testns.example.com."),
+			test.SRV("_http._tcp.svc-headless.testns.example.com.	5	IN	SRV	0	50	80	endpoint-svc-1.svc-headless.testns.example.com."),
+		},
+		Extra: []dns.RR{
+			test.A("endpoint-svc-0.svc-headless.testns.example.com.	5	IN	A	1.2.3.4"),
+			test.A("endpoint-svc-1.svc-headless.testns.example.com.	5	IN	A	1.2.3.5"),
+		},
+	},
+	{
+		Qname: "endpoint-svc-0.svc-headless.testns.example.com.", Qtype: dns.TypeSRV, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.SRV("endpoint-svc-0.svc-headless.testns.example.com.		5	IN	SRV	0	100	80	endpoint-svc-0.svc-headless.testns.example.com."),
+		},
+		Extra: []dns.RR{
+			test.A("endpoint-svc-0.svc-headless.testns.example.com.	5	IN	A	1.2.3.4"),
+		},
+	},
+	{
+		Qname: "endpoint-svc-1.svc-headless.testns.example.com.", Qtype: dns.TypeSRV, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.SRV("endpoint-svc-1.svc-headless.testns.example.com.		5	IN	SRV	0	100	80	endpoint-svc-1.svc-headless.testns.example.com."),
+		},
+		Extra: []dns.RR{
+			test.A("endpoint-svc-1.svc-headless.testns.example.com.	5	IN	A	1.2.3.5"),
+		},
+	},
+	{
+		Qname: "endpoint-svc-0.svc-headless.testns.example.com.", Qtype: dns.TypeA, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.A("endpoint-svc-0.svc-headless.testns.example.com.	5	IN	A	1.2.3.4"),
+		},
+	},
+	{
+		Qname: "endpoint-svc-1.svc-headless.testns.example.com.", Qtype: dns.TypeA, Rcode: dns.RcodeSuccess,
+		Answer: []dns.RR{
+			test.A("endpoint-svc-1.svc-headless.testns.example.com.	5	IN	A	1.2.3.5"),
+		},
+	},
 }
 
 type external struct{}
@@ -226,8 +287,16 @@ func (external) Stop() error
 func (external) EpIndexReverse(string) []*object.Endpoints { return nil }
 func (external) SvcIndexReverse(string) []*object.Service  { return nil }
 func (external) Modified(bool) int64                       { return 0 }
-func (external) EpIndex(s string) []*object.Endpoints                              { return nil }
+func (external) EpIndex(s string) []*object.Endpoints {
-func (external) EndpointsList() []*object.Endpoints                                { return nil }
+	return epIndexExternal[s]
+}
+func (external) EndpointsList() []*object.Endpoints {
+	var eps []*object.Endpoints
+	for _, ep := range epIndexExternal {
+		eps = append(eps, ep...)
+	}
+	return eps
+}
 func (external) GetNodeByName(ctx context.Context, name string) (*api.Node, error) { return nil, nil }
 func (external) SvcIndex(s string) []*object.Service                               { return svcIndexExternal[s] }
 func (external) PodIndex(string) []*object.Pod                                     { return nil }
@@ -252,6 +321,41 @@ func (external) GetNamespaceByName(name string) (*object.Namespace, error) {
 	}, nil
 }
 
+var epIndexExternal = map[string][]*object.Endpoints{
+	"svc-headless.testns": {
+		{
+			Name:      "svc-headless",
+			Namespace: "testns",
+			Index:     "svc-headless.testns",
+			Subsets: []object.EndpointSubset{
+				{
+					Ports: []object.EndpointPort{
+						{
+							Port:     80,
+							Name:     "http",
+							Protocol: "TCP",
+						},
+					},
+					Addresses: []object.EndpointAddress{
+						{
+							IP:            "1.2.3.4",
+							Hostname:      "endpoint-svc-0",
+							NodeName:      "test-node",
+							TargetRefName: "endpoint-svc-0",
+						},
+						{
+							IP:            "1.2.3.5",
+							Hostname:      "endpoint-svc-1",
+							NodeName:      "test-node",
+							TargetRefName: "endpoint-svc-1",
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
 var svcIndexExternal = map[string][]*object.Service{
 	"svc1.testns": {
 		{
@@ -279,6 +383,7 @@ var svcIndexExternal = map[string][]*object.Service{
 			Namespace:   "testns",
 			Type:        api.ServiceTypeLoadBalancer,
 			ExternalIPs: []string{"2.3.4.5"},
+			ClusterIPs:  []string{"10.0.0.3"},
 			Ports:       []api.ServicePort{{Name: "http", Protocol: "tcp", Port: 80}},
 		},
 	},
@@ -287,10 +392,20 @@ var svcIndexExternal = map[string][]*object.Service{
 			Name:        "svc12",
 			Namespace:   "testns",
 			Type:        api.ServiceTypeLoadBalancer,
+			ClusterIPs:  []string{"10.0.0.3"},
 			ExternalIPs: []string{"dummy.hostname"},
 			Ports:       []api.ServicePort{{Name: "http", Protocol: "tcp", Port: 80}},
 		},
 	},
+	"svc-headless.testns": {
+		{
+			Name:       "svc-headless",
+			Namespace:  "testns",
+			Type:       api.ServiceTypeClusterIP,
+			ClusterIPs: []string{"None"},
+			Ports:      []api.ServicePort{{Name: "http", Protocol: "tcp", Port: 80}},
+		},
+	},
 }
 
 func (external) ServiceList() []*object.Service {
@@ -301,7 +416,7 @@ func (external) ServiceList() []*object.Service {
 	return svcs
 }
 
-func externalAddress(state request.Request) []dns.RR {
+func externalAddress(state request.Request, headless bool) []dns.RR {
 	a := test.A("example.org. IN A 127.0.0.1")
 	return []dns.RR{a}
 }

plugin/k8s_external/setup.go

@@ -68,6 +68,8 @@ func parse(c *caddy.Controller) (*External, error) {
 					return nil, c.ArgErr()
 				}
 				e.apex = args[0]
+			case "headless":
+				e.headless = true
 			default:
 				return nil, c.Errf("unknown property '%s'", c.Val())
 			}

plugin/k8s_external/setup_test.go

@@ -12,12 +12,16 @@ func TestSetup(t *testing.T) {
 		shouldErr        bool
 		expectedZone     string
 		expectedApex     string
+		expectedHeadless bool
 	}{
-		{`k8s_external`, false, "", "dns"},
+		{`k8s_external`, false, "", "dns", false},
-		{`k8s_external example.org`, false, "example.org.", "dns"},
+		{`k8s_external example.org`, false, "example.org.", "dns", false},
 		{`k8s_external example.org {
 			apex testdns
-}`, false, "example.org.", "testdns"},
+}`, false, "example.org.", "testdns", false},
+		{`k8s_external example.org {
+	headless
+}`, false, "example.org.", "dns", true},
 	}
 
 	for i, test := range tests {
@@ -44,5 +48,10 @@ func TestSetup(t *testing.T) {
 				t.Errorf("Test %d, expected apex %q for input %s, got: %q", i, test.expectedApex, test.input, e.apex)
 			}
 		}
+		if !test.shouldErr {
+			if test.expectedHeadless != e.headless {
+				t.Errorf("Test %d, expected headless %q for input %s, got: %v", i, test.expectedApex, test.input, e.headless)
+			}
+		}
 	}
 }

plugin/k8s_external/transfer.go

@@ -2,9 +2,11 @@ package external
 
 import (
 	"context"
+	"strings"
 
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/etcd/msg"
+	"github.com/coredns/coredns/plugin/kubernetes"
 	"github.com/coredns/coredns/plugin/transfer"
 	"github.com/coredns/coredns/request"
 
@@ -40,7 +42,7 @@ func (e *External) Transfer(zone string, serial uint32) (<-chan []dns.RR, error)
 		ch <- []dns.RR{&dns.NS{Hdr: nsHdr, Ns: nsName}}
 
 		// Add Nameserver A/AAAA records
-		nsRecords := e.externalAddrFunc(state)
+		nsRecords := e.externalAddrFunc(state, e.headless)
 		for i := range nsRecords {
 			// externalAddrFunc returns incomplete header names, correct here
 			nsRecords[i].Header().Name = nsName
@@ -48,10 +50,12 @@ func (e *External) Transfer(zone string, serial uint32) (<-chan []dns.RR, error)
 			ch <- []dns.RR{nsRecords[i]}
 		}
 
-		svcs := e.externalServicesFunc(zone)
+		svcs, headlessSvcs := e.externalServicesFunc(zone, e.headless)
 		srvSeen := make(map[string]struct{})
+
 		for i := range svcs {
 			name := msg.Domain(svcs[i].Key)
+
 			if svcs[i].TargetStrip == 0 {
 				// Add Service A/AAAA records
 				s := request.Request{Req: &dns.Msg{Question: []dns.Question{{Name: name}}}}
@@ -81,6 +85,55 @@ func (e *External) Transfer(zone string, serial uint32) (<-chan []dns.RR, error)
 				}
 			}
 		}
+		for key, svcs := range headlessSvcs {
+			// we have to strip the leading key because it's either port.protocol or endpoint
+			name := msg.Domain(key[:strings.LastIndex(key, "/")])
+			switchKey := key[strings.LastIndex(key, "/")+1:]
+			switch switchKey {
+			case kubernetes.Endpoint:
+				// headless.namespace.example.com records
+				s := request.Request{Req: &dns.Msg{Question: []dns.Question{{Name: name}}}}
+				as, _ := e.a(ctx, svcs, s)
+				if len(as) > 0 {
+					ch <- as
+				}
+				aaaas, _ := e.aaaa(ctx, svcs, s)
+				if len(aaaas) > 0 {
+					ch <- aaaas
+				}
+				// Add bare SRV record, ensuring uniqueness
+				recs, _ := e.srv(ctx, svcs, s)
+				ch <- recs
+				for _, srv := range recs {
+					ch <- []dns.RR{srv}
+				}
+
+				for i := range svcs {
+					// endpoint.headless.namespace.example.com record
+					s := request.Request{Req: &dns.Msg{Question: []dns.Question{{Name: msg.Domain(svcs[i].Key)}}}}
+
+					as, _ := e.a(ctx, []msg.Service{svcs[i]}, s)
+					if len(as) > 0 {
+						ch <- as
+					}
+					aaaas, _ := e.aaaa(ctx, []msg.Service{svcs[i]}, s)
+					if len(aaaas) > 0 {
+						ch <- aaaas
+					}
+					// Add bare SRV record, ensuring uniqueness
+					recs, _ := e.srv(ctx, []msg.Service{svcs[i]}, s)
+					ch <- recs
+					for _, srv := range recs {
+						ch <- []dns.RR{srv}
+					}
+				}
+
+			case kubernetes.PortProtocol:
+				s := request.Request{Req: &dns.Msg{Question: []dns.Question{{Name: name}}}}
+				recs, _ := e.srv(ctx, svcs, s)
+				ch <- recs
+			}
+		}
 		ch <- []dns.RR{soa}
 		close(ch)
 	}()

plugin/k8s_external/transfer_test.go

@@ -27,6 +27,7 @@ func TestTransferAXFR(t *testing.T) {
 	k.APIConn = &external{}
 
 	e := New()
+	e.headless = true
 	e.Zones = []string{"example.com."}
 	e.externalFunc = k.External
 	e.externalAddrFunc = externalAddress  // internal test function
@@ -64,6 +65,7 @@ func TestTransferAXFR(t *testing.T) {
 			if ans.Header().Rrtype == dns.TypePTR {
 				continue
 			}
+
 			expect = append(expect, ans)
 		}
 	}
@@ -92,6 +94,7 @@ func TestTransferIXFR(t *testing.T) {
 
 	e := New()
 	e.Zones = []string{"example.com."}
+	e.headless = true
 	e.externalFunc = k.External
 	e.externalAddrFunc = externalAddress  // internal test function
 	e.externalSerialFunc = externalSerial // internal test function

plugin/kubernetes/external.go

@@ -11,9 +11,21 @@ import (
 	"github.com/miekg/dns"
 )
 
+// Those constants are used to distinguish between records in ExternalServices headless
+// return values.
+// They are always appendedn to key in a map which is
+// either base service key eg. /com/example/namespace/service/endpoint or
+// /com/example/namespace/service/_http/_tcp/port.protocol
+// this will allow us to distinguish services in implementation of Transfer protocol
+// see plugin/k8s_external/transfer.go
+const (
+	Endpoint     = "endpoint"
+	PortProtocol = "port.protocol"
+)
+
 // External implements the ExternalFunc call from the external plugin.
-// It returns any services matching in the services' ExternalIPs.
+// It returns any services matching in the services' ExternalIPs and if enabled, headless endpoints..
-func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
+func (k *Kubernetes) External(state request.Request, headless bool) ([]msg.Service, int) {
 	if state.QType() == dns.TypePTR {
 		ip := dnsutil.ExtractAddressFromReverse(state.Name())
 		if ip != "" {
@@ -33,10 +45,11 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 	if last < 0 {
 		return nil, dns.RcodeServerFailure
 	}
-	// We are dealing with a fairly normal domain name here, but we still need to have the service
+	// We are dealing with a fairly normal domain name here, but we still need to have the service,
-	// and the namespace:
+	// namespace and if present, endpoint:
-	// service.namespace.<base>
+	// service.namespace.<base> or
-	var port, protocol string
+	// endpoint.service.namespace.<base>
+	var port, protocol, endpoint string
 	namespace := segs[last]
 	if !k.namespaceExposed(namespace) {
 		return nil, dns.RcodeNameError
@@ -49,7 +62,10 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 
 	service := segs[last]
 	last--
-	if last == 1 {
+	if last == 0 {
+		endpoint = stripUnderscore(segs[last])
+		last--
+	} else if last == 1 {
 		protocol = stripUnderscore(segs[last])
 		port = stripUnderscore(segs[last-1])
 		last -= 2
@@ -60,8 +76,13 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 		return nil, dns.RcodeNameError
 	}
 
+	var (
+		endpointsList []*object.Endpoints
+		serviceList   []*object.Service
+	)
+
 	idx := object.ServiceKey(service, namespace)
-	serviceList := k.APIConn.SvcIndex(idx)
+	serviceList = k.APIConn.SvcIndex(idx)
 
 	services := []msg.Service{}
 	zonePath := msg.Path(state.Zone, coredns)
@@ -75,6 +96,36 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 			continue
 		}
 
+		if headless && len(svc.ExternalIPs) == 0 && (svc.Headless() || endpoint != "") {
+			if endpointsList == nil {
+				endpointsList = k.APIConn.EpIndex(idx)
+			}
+			// Endpoint query or headless service
+			for _, ep := range endpointsList {
+				if object.EndpointsKey(svc.Name, svc.Namespace) != ep.Index {
+					continue
+				}
+
+				for _, eps := range ep.Subsets {
+					for _, addr := range eps.Addresses {
+						if endpoint != "" && !match(endpoint, endpointHostname(addr, k.endpointNameMode)) {
+							continue
+						}
+
+						for _, p := range eps.Ports {
+							if !(matchPortAndProtocol(port, p.Name, protocol, p.Protocol)) {
+								continue
+							}
+							s := msg.Service{Host: addr.IP, Port: int(p.Port), TTL: k.ttl}
+							s.Key = strings.Join([]string{zonePath, svc.Namespace, svc.Name, endpointHostname(addr, k.endpointNameMode)}, "/")
+
+							services = append(services, s)
+						}
+					}
+				}
+			}
+			continue
+		} else {
 			for _, ip := range svc.ExternalIPs {
 				for _, p := range svc.Ports {
 					if !(matchPortAndProtocol(port, p.Name, protocol, string(p.Protocol))) {
@@ -88,6 +139,7 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 				}
 			}
 		}
+	}
 	if state.QType() == dns.TypePTR {
 		// if this was a PTR request, return empty service list, but retain rcode for proper nxdomain/nodata response
 		return nil, rcode
@@ -96,19 +148,52 @@ func (k *Kubernetes) External(state request.Request) ([]msg.Service, int) {
 }
 
 // ExternalAddress returns the external service address(es) for the CoreDNS service.
-func (k *Kubernetes) ExternalAddress(state request.Request) []dns.RR {
+func (k *Kubernetes) ExternalAddress(state request.Request, headless bool) []dns.RR {
 	// If CoreDNS is running inside the Kubernetes cluster: k.nsAddrs() will return the external IPs of the services
 	// targeting the CoreDNS Pod.
 	// If CoreDNS is running outside of the Kubernetes cluster: k.nsAddrs() will return the first non-loopback IP
 	// address seen on the local system it is running on. This could be the wrong answer if coredns is using the *bind*
 	// plugin to bind to a different IP address.
-	return k.nsAddrs(true, state.Zone)
+	return k.nsAddrs(true, headless, state.Zone)
 }
 
-// ExternalServices returns all services with external IPs
+// ExternalServices returns all services with external IPs and if enabled headless services
-func (k *Kubernetes) ExternalServices(zone string) (services []msg.Service) {
+func (k *Kubernetes) ExternalServices(zone string, headless bool) (services []msg.Service, headlessServices map[string][]msg.Service) {
 	zonePath := msg.Path(zone, coredns)
+	headlessServices = make(map[string][]msg.Service)
 	for _, svc := range k.APIConn.ServiceList() {
+		// Endpoints and headless services
+		if headless && len(svc.ExternalIPs) == 0 && svc.Headless() {
+			idx := object.ServiceKey(svc.Name, svc.Namespace)
+			endpointsList := k.APIConn.EpIndex(idx)
+
+			for _, ep := range endpointsList {
+				for _, eps := range ep.Subsets {
+					for _, addr := range eps.Addresses {
+						// we need to have some answers grouped together
+						// 1. for endpoint requests eg. endpoint-0.service.example.com - will always have one endpoint
+						// 2. for service requests eg. service.example.com - can have multiple endpoints
+						// 3. for port.protocol requests eg. _http._tcp.service.example.com - can have multiple endpoints
+						for _, p := range eps.Ports {
+							s := msg.Service{Host: addr.IP, Port: int(p.Port), TTL: k.ttl}
+							baseSvc := strings.Join([]string{zonePath, svc.Namespace, svc.Name}, "/")
+							s.Key = strings.Join([]string{baseSvc, endpointHostname(addr, k.endpointNameMode)}, "/")
+							headlessServices[strings.Join([]string{baseSvc, Endpoint}, "/")] = append(headlessServices[strings.Join([]string{baseSvc, Endpoint}, "/")], s)
+
+							// As per spec unnamed ports do not have a srv record
+							// https://github.com/kubernetes/dns/blob/master/docs/specification.md#232---srv-records
+							if p.Name == "" {
+								continue
+							}
+							s.Host = msg.Domain(s.Key)
+							s.Key = strings.Join(append([]string{zonePath, svc.Namespace, svc.Name}, strings.ToLower("_"+string(p.Protocol)), strings.ToLower("_"+string(p.Name))), "/")
+							headlessServices[strings.Join([]string{s.Key, PortProtocol}, "/")] = append(headlessServices[strings.Join([]string{s.Key, PortProtocol}, "/")], s)
+						}
+					}
+				}
+			}
+			continue
+		} else {
 			for _, ip := range svc.ExternalIPs {
 				for _, p := range svc.Ports {
 					s := msg.Service{Host: ip, Port: int(p.Port), TTL: k.ttl}
@@ -120,7 +205,8 @@ func (k *Kubernetes) ExternalServices(zone string) (services []msg.Service) {
 				}
 			}
 		}
-	return services
+	}
+	return services, headlessServices
 }
 
 //ExternalSerial returns the serial of the external zone

plugin/kubernetes/external_test.go

@@ -43,6 +43,22 @@ var extCases = []struct {
 	{
 		Qname: "svc0.svc-nons.example.com.", Rcode: dns.RcodeNameError,
 	},
+	{
+		Qname: "svc-headless.testns.example.com.", Rcode: dns.RcodeSuccess,
+		Msg: []msg.Service{
+			{Host: "1.2.3.4", Port: 80, TTL: 5, Weight: 50, Key: "/c/org/example/testns/svc-headless"},
+			{Host: "1.2.3.5", Port: 80, TTL: 5, Weight: 50, Key: "/c/org/example/testns/svc-headless"},
+		},
+	},
+	{
+		Qname: "endpoint-0.svc-headless.testns.example.com.", Rcode: dns.RcodeSuccess,
+		Msg: []msg.Service{
+			{Host: "1.2.3.4", Port: 80, TTL: 5, Weight: 100, Key: "/c/org/example/testns/svc-headless/endpoint-0"},
+		},
+	},
+	{
+		Qname: "endpoint-1.svc-nons.testns.example.com.", Rcode: dns.RcodeNameError,
+	},
 }
 
 func TestExternal(t *testing.T) {
@@ -54,7 +70,7 @@ func TestExternal(t *testing.T) {
 	for i, tc := range extCases {
 		state := testRequest(tc.Qname)
 
-		svc, rcode := k.External(state)
+		svc, rcode := k.External(state, true)
 
 		if x := tc.Rcode; x != rcode {
 			t.Errorf("Test %d, expected rcode %d, got %d", i, x, rcode)
@@ -82,8 +98,16 @@ func (external) EpIndexReverse(string) []*object.Endpoints
 func (external) SvcIndexReverse(string) []*object.Service    { return nil }
 func (external) SvcExtIndexReverse(string) []*object.Service { return nil }
 func (external) Modified(bool) int64                         { return 0 }
-func (external) EpIndex(s string) []*object.Endpoints                              { return nil }
+func (external) EpIndex(s string) []*object.Endpoints {
-func (external) EndpointsList() []*object.Endpoints                                { return nil }
+	return epIndexExternal[s]
+}
+func (external) EndpointsList() []*object.Endpoints {
+	var eps []*object.Endpoints
+	for _, ep := range epIndexExternal {
+		eps = append(eps, ep...)
+	}
+	return eps
+}
 func (external) GetNodeByName(ctx context.Context, name string) (*api.Node, error) { return nil, nil }
 func (external) SvcIndex(s string) []*object.Service                               { return svcIndexExternal[s] }
 func (external) PodIndex(string) []*object.Pod                                     { return nil }
@@ -94,6 +118,41 @@ func (external) GetNamespaceByName(name string) (*object.Namespace, error) {
 	}, nil
 }
 
+var epIndexExternal = map[string][]*object.Endpoints{
+	"svc-headless.testns": {
+		{
+			Name:      "svc-headless",
+			Namespace: "testns",
+			Index:     "svc-headless.testns",
+			Subsets: []object.EndpointSubset{
+				{
+					Ports: []object.EndpointPort{
+						{
+							Port:     80,
+							Name:     "http",
+							Protocol: "TCP",
+						},
+					},
+					Addresses: []object.EndpointAddress{
+						{
+							IP:            "1.2.3.4",
+							Hostname:      "endpoint-svc-0",
+							NodeName:      "test-node",
+							TargetRefName: "endpoint-svc-0",
+						},
+						{
+							IP:            "1.2.3.5",
+							Hostname:      "endpoint-svc-1",
+							NodeName:      "test-node",
+							TargetRefName: "endpoint-svc-1",
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
 var svcIndexExternal = map[string][]*object.Service{
 	"svc1.testns": {
 		{
@@ -115,6 +174,15 @@ var svcIndexExternal = map[string][]*object.Service{
 			Ports:       []api.ServicePort{{Name: "http", Protocol: "tcp", Port: 80}},
 		},
 	},
+	"svc-headless.testns": {
+		{
+			Name:       "svc-headless",
+			Namespace:  "testns",
+			Type:       api.ServiceTypeClusterIP,
+			ClusterIPs: []string{api.ClusterIPNone},
+			Ports:      []api.ServicePort{{Name: "http", Protocol: "tcp", Port: 80}},
+		},
+	},
 }
 
 func (external) ServiceList() []*object.Service {

plugin/kubernetes/kubernetes.go

@@ -112,7 +112,7 @@ func (k *Kubernetes) Services(ctx context.Context, state request.Request, exact
 
 	case dns.TypeNS:
 		// We can only get here if the qname equals the zone, see ServeDNS in handler.go.
-		nss := k.nsAddrs(false, state.Zone)
+		nss := k.nsAddrs(false, false, state.Zone)
 		var svcs []msg.Service
 		for _, ns := range nss {
 			if ns.Header().Rrtype == dns.TypeA {
@@ -127,7 +127,7 @@ func (k *Kubernetes) Services(ctx context.Context, state request.Request, exact
 	}
 
 	if isDefaultNS(state.Name(), state.Zone) {
-		nss := k.nsAddrs(false, state.Zone)
+		nss := k.nsAddrs(false, false, state.Zone)
 		var svcs []msg.Service
 		for _, ns := range nss {
 			if ns.Header().Rrtype == dns.TypeA && state.QType() == dns.TypeA {

plugin/kubernetes/ns.go

@@ -13,7 +13,7 @@ func isDefaultNS(name, zone string) bool {
 
 // nsAddrs returns the A or AAAA records for the CoreDNS service in the cluster. If the service cannot be found,
 // it returns a record for the local address of the machine we're running on.
-func (k *Kubernetes) nsAddrs(external bool, zone string) []dns.RR {
+func (k *Kubernetes) nsAddrs(external, headless bool, zone string) []dns.RR {
 	var (
 		svcNames      []string
 		svcIPs        []net.IP
@@ -31,10 +31,21 @@ func (k *Kubernetes) nsAddrs(external bool, zone string) []dns.RR {
 			for _, svc := range svcs {
 				if external {
 					svcName := strings.Join([]string{svc.Name, svc.Namespace, zone}, ".")
+
+					if headless && svc.Headless() {
+						for _, s := range endpoint.Subsets {
+							for _, a := range s.Addresses {
+								svcNames = append(svcNames, endpointHostname(a, k.endpointNameMode)+"."+svcName)
+								svcIPs = append(svcIPs, net.ParseIP(a.IP))
+							}
+						}
+					} else {
 						for _, exIP := range svc.ExternalIPs {
 							svcNames = append(svcNames, svcName)
 							svcIPs = append(svcIPs, net.ParseIP(exIP))
 						}
+					}
+
 					continue
 				}
 				svcName := strings.Join([]string{svc.Name, svc.Namespace, Svc, zone}, ".")

plugin/kubernetes/ns_test.go

@@ -103,7 +103,7 @@ func TestNsAddrs(t *testing.T) {
 	k.APIConn = &APIConnTest{}
 	k.localIPs = []net.IP{net.ParseIP("10.244.0.20")}
 
-	cdrs := k.nsAddrs(false, k.Zones[0])
+	cdrs := k.nsAddrs(false, false, k.Zones[0])
 
 	if len(cdrs) != 3 {
 		t.Fatalf("Expected 3 results, got %v", len(cdrs))
@@ -137,13 +137,36 @@ func TestNsAddrs(t *testing.T) {
 	}
 }
 
+func TestNsAddrsExternalHeadless(t *testing.T) {
+	k := New([]string{"example.com."})
+	k.APIConn = &APIConnTest{}
+	k.localIPs = []net.IP{net.ParseIP("10.244.0.20")}
+
+	// there are only headless sevices
+	cdrs := k.nsAddrs(true, true, k.Zones[0])
+
+	if len(cdrs) != 1 {
+		t.Fatalf("Expected 0 results, got %v", cdrs)
+	}
+
+	cdr := cdrs[0]
+	expected := "10.244.0.20"
+	if cdr.(*dns.A).A.String() != expected {
+		t.Errorf("Expected A address to be %q, got %q", expected, cdr.(*dns.A).A.String())
+	}
+	expected = "10-244-0-20.hdls-dns-service.kube-system.example.com."
+	if cdr.Header().Name != expected {
+		t.Errorf("Expected record name to be %q, got %q", expected, cdr.Header().Name)
+	}
+}
+
 func TestNsAddrsExternal(t *testing.T) {
 	k := New([]string{"example.com."})
 	k.APIConn = &APIConnTest{}
 	k.localIPs = []net.IP{net.ParseIP("10.244.0.20")}
 
 	// initially no services have an external IP ...
-	cdrs := k.nsAddrs(true, k.Zones[0])
+	cdrs := k.nsAddrs(true, false, k.Zones[0])
 
 	if len(cdrs) != 0 {
 		t.Fatalf("Expected 0 results, got %v", len(cdrs))
@@ -151,7 +174,35 @@ func TestNsAddrsExternal(t *testing.T) {
 
 	// Add an external IP to one of the services ...
 	svcs[0].ExternalIPs = []string{"1.2.3.4"}
-	cdrs = k.nsAddrs(true, k.Zones[0])
+	cdrs = k.nsAddrs(true, false, k.Zones[0])
+
+	if len(cdrs) != 1 {
+		t.Fatalf("Expected 1 results, got %v", len(cdrs))
+	}
+	cdr := cdrs[0]
+	expected := "1.2.3.4"
+	if cdr.(*dns.A).A.String() != expected {
+		t.Errorf("Expected A address to be %q, got %q", expected, cdr.(*dns.A).A.String())
+	}
+	expected = "dns-service.kube-system.example.com."
+	if cdr.Header().Name != expected {
+		t.Errorf("Expected record name to be %q, got %q", expected, cdr.Header().Name)
+	}
+}
+
+func TestNsAddrsExternalWithPreexistingExternalIP(t *testing.T) {
+	k := New([]string{"example.com."})
+	k.APIConn = &APIConnTest{}
+	k.localIPs = []net.IP{net.ParseIP("10.244.0.20")}
+
+	svcs[0].ExternalIPs = []string{"1.2.3.4"}
+
+	// initially no services have an external IP ...
+	cdrs := k.nsAddrs(true, false, k.Zones[0])
+
+	if len(cdrs) != 1 {
+		t.Fatalf("Expected 1 results, got %v", len(cdrs))
+	}
 
 	if len(cdrs) != 1 {
 		t.Fatalf("Expected 1 results, got %v", len(cdrs))

plugin/kubernetes/xfr.go

@@ -42,7 +42,7 @@ func (k *Kubernetes) Transfer(zone string, serial uint32) (<-chan []dns.RR, erro
 		}
 		ch <- soa
 
-		nsAddrs := k.nsAddrs(false, zone)
+		nsAddrs := k.nsAddrs(false, false, zone)
 		nsHosts := make(map[string]struct{})
 		for _, nsAddr := range nsAddrs {
 			nsHost := nsAddr.Header().Name