Support multiple k8s api servers specification and load balance among api servers (#820)

* Support multiple k8s api servers specification and load balance among api servers

This fix adds supports for multiple k8s api servers specification,
load balance among api servers.

When two or more api servers are specified in kubernetes block (endpoint ...),
a proxy is created locally (with randomly generately port). The coredns
will points to the generated proxy so that load balancing could be achieved.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Setup initial healthcheck at the beginning

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update README.md for kubernetes middleware and remove whitespaces.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Use middleware/pkg/healthcheck in middleware/kubernetes

for api proxy

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

test/kubernetes_test.go

@@ -450,7 +450,7 @@ func doIntegrationTests(t *testing.T, corefile string, testCases []test.Case) {
 
 	// Work-around for timing condition that results in no-data being returned in
 	// test environment.
-	time.Sleep(1 * time.Second)
+	time.Sleep(3 * time.Second)
 
 	for _, tc := range testCases {
 
@@ -513,6 +513,27 @@ func TestKubernetesIntegration(t *testing.T) {
 	doIntegrationTests(t, corefile, dnsTestCases)
 }
 
+func TestKubernetesIntegrationAPIProxy(t *testing.T) {
+
+	removeUpstreamConfig, upstreamServer, udp := createUpstreamServer(t)
+	defer upstreamServer.Stop()
+	defer removeUpstreamConfig()
+
+	corefile :=
+		`.:0 {
+    kubernetes cluster.local 0.0.10.in-addr.arpa {
+        endpoint http://nonexistance:8080,http://invalidip:8080,http://localhost:8080
+        namespaces test-1
+        pods disabled
+        upstream ` + udp + `
+    }
+    erratic . {
+        drop 0
+    }
+`
+	doIntegrationTests(t, corefile, dnsTestCases)
+}
+
 func TestKubernetesIntegrationPodsInsecure(t *testing.T) {
 	corefile :=
 		`.:0 {