Make middleware survive a restart (#142)

Make middleware that sets up a (http) handler survive a graceful
restart. We calls the middleware's Shutdown function(s). If restart
fails the Start function is called again.

* middleware/health: OK
* middleware/pprof: OK
* middleware/metrics: OK

All restart OK.

core/restart.go

@@ -117,6 +117,9 @@ func Restart(newCorefile Input) error {
 	}
 	wpipe.Close()
 
+	// Run all shutdown functions for the middleware, if child start fails, restart them all...
+	executeShutdownCallbacks("SIGUSR1")
+
 	// Determine whether child startup succeeded
 	answer, readErr := ioutil.ReadAll(sigrpipe)
 	if answer == nil || len(answer) == 0 {
@@ -125,6 +128,9 @@ func Restart(newCorefile Input) error {
 		if readErr != nil {
 			log.Printf("[ERROR] Restart: additionally, error communicating with child process: %v", readErr)
 		}
+		// re-call all startup functions.
+		// TODO(miek): this needs to be tested, somehow.
+		executeStartupCallbacks("SIGUSR1")
 		return errIncompleteRestart
 	}
 

core/setup/health.go

@@ -11,8 +11,9 @@ func Health(c *Controller) (middleware.Middleware, error) {
 		return nil, err
 	}
 
-	h := health.Health{Addr: addr}
-	c.Startup = append(c.Startup, h.ListenAndServe)
+	h := &health.Health{Addr: addr}
+	c.Startup = append(c.Startup, h.Start)
+	c.Shutdown = append(c.Shutdown, h.Shutdown)
 	return nil, nil
 }
 

core/setup/metrics.go

@@ -12,18 +12,19 @@ const addr = "localhost:9153"
 var metricsOnce sync.Once
 
 func Prometheus(c *Controller) (middleware.Middleware, error) {
-	met, err := parsePrometheus(c)
+	m, err := parsePrometheus(c)
 	if err != nil {
 		return nil, err
 	}
 
 	metricsOnce.Do(func() {
-		c.Startup = append(c.Startup, met.Start)
+		c.Startup = append(c.Startup, m.Start)
+		c.Shutdown = append(c.Shutdown, m.Shutdown)
 	})
 
 	return func(next middleware.Handler) middleware.Handler {
-		met.Next = next
-		return met
+		m.Next = next
+		return m
 	}, nil
 }
 

core/setup/pprof.go

@@ -27,6 +27,7 @@ func PProf(c *Controller) (middleware.Middleware, error) {
 	handler := &pprof.Handler{}
 	pprofOnce.Do(func() {
 		c.Startup = append(c.Startup, handler.Start)
+		c.Shutdown = append(c.Shutdown, handler.Shutdown)
 	})
 
 	return func(next middleware.Handler) middleware.Handler {

core/sigtrap.go

@@ -68,4 +68,26 @@ func executeShutdownCallbacks(signame string) (exitCode int) {
 	return
 }
 
-var shutdownCallbacksOnce sync.Once
+// executeStartupCallbacks executes the startup callbacks as initiated
+// by signame. This is used when on restart when the child failed to start and
+// all middleware executed their shutdown functions
+func executeStartupCallbacks(signame string) (exitCode int) {
+	startupCallbacksOnce.Do(func() {
+		serversMu.Lock()
+		errs := server.StartupCallbacks(servers)
+		serversMu.Unlock()
+
+		if len(errs) > 0 {
+			for _, err := range errs {
+				log.Printf("[ERROR] %s shutdown: %v", signame, err)
+			}
+			exitCode = 1
+		}
+	})
+	return
+}
+
+var (
+	shutdownCallbacksOnce sync.Once
+	startupCallbacksOnce  sync.Once
+)