plugin/health: add lameduck mode (#1379)

* plugin/health: add lameduck mode

Add a way to configure lameduck more, i.e. set health to false, stop
polling plugins. Then wait for a duration before shutting down. As the
health middleware is configured early on in the plugin list, it will
hold up all other shutdown, meaning we still answer queries.

* Add New

* More tests

* golint

* remove confusing text

plugin/health/README.md

@@ -21,6 +21,17 @@ a 503. *health* periodically (1s) polls plugin that exports health information.
 plugin signals that it is unhealthy, the server will go unhealthy too. Each plugin that
 supports health checks has a section "Health" in their README.
 
+More options can be set with this extended syntax:
+
+~~~
+health [ADDRESS] {
+    lameduck DURATION
+}
+~~~
+
+* Where `lameduck` will make the process unhealthy then *wait* for **DURATION** before the process
+  shuts down.
+
 ## Plugins
 
 Any plugin that implements the Healther interface will be used to report health.
@@ -42,3 +53,13 @@ Run another health endpoint on http://localhost:8091.
     health localhost:8091
 }
 ~~~
+
+Set a lameduck duration of 1 second:
+
+~~~ corefile
+. {
+    health localhost:8091 {
+        lameduck 1s
+    }
+}
+~~~

plugin/health/health.go

@@ -7,12 +7,15 @@ import (
 	"net"
 	"net/http"
 	"sync"
+	"time"
 )
 
 var once sync.Once
 
+// Health implements healthchecks by polling plugins.
 type health struct {
 	Addr     string
+	lameduck time.Duration
 
 	ln  net.Listener
 	mux *http.ServeMux
@@ -23,6 +26,12 @@ type health struct {
 	ok bool // ok is the global boolean indicating an all healthy plugin stack
 
 	stop     chan bool
+	pollstop chan bool
+}
+
+// newHealth returns a new initialized health.
+func newHealth(addr string) *health {
+	return &health{Addr: addr, stop: make(chan bool), pollstop: make(chan bool)}
 }
 
 func (h *health) OnStartup() error {
@@ -61,12 +70,21 @@ func (h *health) OnStartup() error {
 }
 
 func (h *health) OnShutdown() error {
+	// Stop polling plugins
+	h.pollstop <- true
+	// NACK health
+	h.SetOk(false)
+
+	if h.lameduck > 0 {
+		log.Printf("[INFO] Going into lameduck mode for %s", h.lameduck)
+		time.Sleep(h.lameduck)
+	}
+
 	if h.ln != nil {
 		return h.ln.Close()
 	}
 
 	h.stop <- true
-
 	return nil
 }
 

plugin/health/health_test.go

@@ -5,12 +5,13 @@ import (
 	"io/ioutil"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/coredns/coredns/plugin/erratic"
 )
 
 func TestHealth(t *testing.T) {
-	h := health{Addr: ":0"}
+	h := newHealth(":0")
 	h.h = append(h.h, &erratic.Erratic{})
 
 	if err := h.OnStartup(); err != nil {
@@ -18,6 +19,11 @@ func TestHealth(t *testing.T) {
 	}
 	defer h.OnShutdown()
 
+	go func() {
+		<-h.pollstop
+		return
+	}()
+
 	// Reconstruct the http address based on the port allocated by operating system.
 	address := fmt.Sprintf("http://%s%s", h.ln.Addr().String(), path)
 
@@ -50,3 +56,22 @@ func TestHealth(t *testing.T) {
 		t.Errorf("Invalid response body: expecting 'OK', got '%s'", string(content))
 	}
 }
+
+func TestHealthLameduck(t *testing.T) {
+	h := newHealth(":0")
+	h.lameduck = 250 * time.Millisecond
+	h.h = append(h.h, &erratic.Erratic{})
+
+	if err := h.OnStartup(); err != nil {
+		t.Fatalf("Unable to startup the health server: %v", err)
+	}
+
+	// Both these things are behind a sync.Once, fake reading from the channels.
+	go func() {
+		<-h.pollstop
+		<-h.stop
+		return
+	}()
+
+	h.OnShutdown()
+}

plugin/health/setup.go

@@ -1,6 +1,7 @@
 package health
 
 import (
+	"fmt"
 	"net"
 	"time"
 
@@ -19,12 +20,13 @@ func init() {
 }
 
 func setup(c *caddy.Controller) error {
-	addr, err := healthParse(c)
+	addr, lame, err := healthParse(c)
 	if err != nil {
 		return plugin.Error("health", err)
 	}
 
-	h := &health{Addr: addr, stop: make(chan bool)}
+	h := newHealth(addr)
+	h.lameduck = lame
 
 	c.OnStartup(func() error {
 		plugins := dnsserver.GetConfig(c).Handlers()
@@ -41,8 +43,12 @@ func setup(c *caddy.Controller) error {
 		h.poll()
 		go func() {
 			for {
-				<-time.After(1 * time.Second)
+				select {
+				case <-time.After(1 * time.Second):
 					h.poll()
+				case <-h.pollstop:
+					return
+				}
 			}
 		}()
 		return nil
@@ -68,8 +74,9 @@ func setup(c *caddy.Controller) error {
 	return nil
 }
 
-func healthParse(c *caddy.Controller) (string, error) {
+func healthParse(c *caddy.Controller) (string, time.Duration, error) {
 	addr := ""
+	dur := time.Duration(0)
 	for c.Next() {
 		args := c.RemainingArgs()
 
@@ -78,11 +85,28 @@ func healthParse(c *caddy.Controller) (string, error) {
 		case 1:
 			addr = args[0]
 			if _, _, e := net.SplitHostPort(addr); e != nil {
-				return "", e
+				return "", 0, e
 			}
 		default:
-			return "", c.ArgErr()
+			return "", 0, c.ArgErr()
+		}
+
+		for c.NextBlock() {
+			switch c.Val() {
+			case "lameduck":
+				args := c.RemainingArgs()
+				if len(args) != 1 {
+					return "", 0, c.ArgErr()
+				}
+				l, err := time.ParseDuration(args[0])
+				if err != nil {
+					return "", 0, fmt.Errorf("unable to parse lameduck duration value: '%v' : %v", args[0], err)
+				}
+				dur = l
+			default:
+				return "", 0, c.ArgErr()
 			}
 		}
-	return addr, nil
+	}
+	return addr, dur, nil
 }

plugin/health/setup_test.go

@@ -13,17 +13,27 @@ func TestSetupHealth(t *testing.T) {
 	}{
 		{`health`, false},
 		{`health localhost:1234`, false},
+		{`health localhost:1234 {
+			lameduck 4s
+}`, false},
 		{`health bla:a`, false},
+
 		{`health bla`, true},
 		{`health bla bla`, true},
+		{`health localhost:1234 {
+			lameduck a
+}`, true},
+		{`health localhost:1234 {
+			lamedudk 4
+} `, true},
 	}
 
 	for i, test := range tests {
 		c := caddy.NewTestController("dns", test.input)
-		_, err := healthParse(c)
+		_, _, err := healthParse(c)
 
 		if test.shouldErr && err == nil {
-			t.Errorf("Test %d: Expected error but found %s for input %s", i, err, test.input)
+			t.Errorf("Test %d: Expected error but found none for input %s", i, test.input)
 		}
 
 		if err != nil {