Add *ready* plugin (#2616)

Add a ready plugin that allows plugin to signal when they are ready.
Once a plugin is ready it is not queried again.

This uses same mechanism as the health plugin: each plugin needs to
implement an interface.

Implement readines for the *erratic* plugin to aid in testing.

Add README.md and tests moduled after the health plugin; which will be
relegated to just providing process health. In similar vein to health
this is a process wide setting.

With this Corefile:
~~~
. {
    erratic
    whoami
    ready
}

bla {
    erratic
    whoami
}
~~~

ready will lead to:

~~~ sh
% curl localhost:8181/ready
% dig @localhost -p 1053 mx example.org
% curl localhost:8181/ready
OK%
~~~

Meanwhile CoreDNS logs:

~~~
.:1053
bla.:1053
2019-02-26T20:59:07.137Z [INFO] CoreDNS-1.3.1
2019-02-26T20:59:07.137Z [INFO] linux/amd64, go1.11.4,
CoreDNS-1.3.1
linux/amd64, go1.11.4,
2019-02-26T20:59:11.415Z [INFO] plugin/ready: Still waiting on: "erratic"
2019-02-26T20:59:13.510Z [INFO] plugin/ready: Still waiting on: "erratic"
~~~

*ready* can be used in multiple server blocks and will do the right
thing; query all those plugins from all server blocks for readiness.
This does a similar thing to the prometheus plugin.

Signed-off-by: Miek Gieben <miek@miek.nl>

plugin/ready/ready.go

@@ -0,0 +1,81 @@
+// Package ready is used to signal readiness of the CoreDNS process. Once all
+// plugins have called in the plugin will signal readiness by returning a 200
+// OK on the HTTP handler (on port 8181). If not ready yet, the handler will
+// return a 503.
+package ready
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"sync"
+
+	clog "github.com/coredns/coredns/plugin/pkg/log"
+	"github.com/coredns/coredns/plugin/pkg/uniq"
+)
+
+var (
+	log      = clog.NewWithPlugin("ready")
+	plugins  = &list{}
+	uniqAddr = uniq.New()
+)
+
+type ready struct {
+	Addr string
+
+	sync.RWMutex
+	ln   net.Listener
+	done bool
+	mux  *http.ServeMux
+}
+
+func (rd *ready) onStartup() error {
+	if rd.Addr == "" {
+		rd.Addr = defAddr
+	}
+
+	ln, err := net.Listen("tcp", rd.Addr)
+	if err != nil {
+		return err
+	}
+
+	rd.Lock()
+	rd.ln = ln
+	rd.mux = http.NewServeMux()
+	rd.done = true
+	rd.Unlock()
+
+	rd.mux.HandleFunc("/ready", func(w http.ResponseWriter, _ *http.Request) {
+		ok, todo := plugins.Ready()
+		if ok {
+			w.WriteHeader(http.StatusOK)
+			io.WriteString(w, "OK")
+			return
+		}
+		log.Infof("Still waiting on: %q", todo)
+		w.WriteHeader(http.StatusServiceUnavailable)
+		io.WriteString(w, todo)
+	})
+
+	go func() { http.Serve(rd.ln, rd.mux) }()
+
+	return nil
+}
+
+func (rd *ready) onRestart() error { return rd.onFinalShutdown() }
+
+func (rd *ready) onFinalShutdown() error {
+	rd.Lock()
+	defer rd.Unlock()
+	if !rd.done {
+		return nil
+	}
+
+	uniqAddr.Unset(rd.Addr)
+
+	rd.ln.Close()
+	rd.done = false
+	return nil
+}
+
+const defAddr = ":8181"