plugin/health: add lameduck mode (#1379)

* plugin/health: add lameduck mode

Add a way to configure lameduck more, i.e. set health to false, stop
polling plugins. Then wait for a duration before shutting down. As the
health middleware is configured early on in the plugin list, it will
hold up all other shutdown, meaning we still answer queries.

* Add New

* More tests

* golint

* remove confusing text

plugin/health/health.go

@@ -7,12 +7,15 @@ import (
 	"net"
 	"net/http"
 	"sync"
+	"time"
 )
 
 var once sync.Once
 
+// Health implements healthchecks by polling plugins.
 type health struct {
-	Addr string
+	Addr     string
+	lameduck time.Duration
 
 	ln  net.Listener
 	mux *http.ServeMux
@@ -22,7 +25,13 @@ type health struct {
 	sync.RWMutex
 	ok bool // ok is the global boolean indicating an all healthy plugin stack
 
-	stop chan bool
+	stop     chan bool
+	pollstop chan bool
+}
+
+// newHealth returns a new initialized health.
+func newHealth(addr string) *health {
+	return &health{Addr: addr, stop: make(chan bool), pollstop: make(chan bool)}
 }
 
 func (h *health) OnStartup() error {
@@ -61,12 +70,21 @@ func (h *health) OnStartup() error {
 }
 
 func (h *health) OnShutdown() error {
+	// Stop polling plugins
+	h.pollstop <- true
+	// NACK health
+	h.SetOk(false)
+
+	if h.lameduck > 0 {
+		log.Printf("[INFO] Going into lameduck mode for %s", h.lameduck)
+		time.Sleep(h.lameduck)
+	}
+
 	if h.ln != nil {
 		return h.ln.Close()
 	}
 
 	h.stop <- true
-
 	return nil
 }