middleware/proxy: async health checks (#749)

* Switches out Unhealthy bool for OkUntil timestamp

* Make sure servers are healthy forever if there are no health checks

* Moves health check off into a go routine to avoid blocking conditions

* Improved logging info

* Fixes initial date

* Fixes health checking; alters tests to adapt to async health checking

* Moves future variable into static upstream and populates it in more places

* Restores silencing of stdout during testing

* Restores silencing of stdout during testing

* keeps check url string once built

* Removes debug message

* uses zero value to signal no checking; reduces in-mutex code to a fetch

middleware/proxy/proxy.go

@@ -59,9 +59,11 @@ type UpstreamHost struct {
 	Name              string // IP address (and port) of this upstream host
 	Fails             int32
 	FailTimeout       time.Duration
-	Unhealthy         bool
+	OkUntil           time.Time
 	CheckDown         UpstreamHostDownFunc
+	CheckUrl          string
 	WithoutPathPrefix string
+	Checking          bool
 	checkMu           sync.Mutex
 }
 
@@ -72,7 +74,17 @@ func (uh *UpstreamHost) Down() bool {
 	if uh.CheckDown == nil {
 		// Default settings
 		fails := atomic.LoadInt32(&uh.Fails)
-		return uh.Unhealthy || fails > 0
+		after := false
+
+		uh.checkMu.Lock()
+		until := uh.OkUntil
+		uh.checkMu.Unlock()
+
+		if !until.IsZero() && time.Now().After(until) {
+			after = true
+		}
+
+		return after || fails > 0
 	}
 	return uh.CheckDown(uh)
 }