pkg/up: implement backoff (#2342)

* pkg/up: implement backoff

Every 2nd failure we double the interval until we hit 4 * interval. This
to have some sort of backoff, esp when a large cluster of coredns shares
an upstream (original intent of up package) they will hammer the
upstream. This put some back pressure on that.

Signed-off-by: Miek Gieben <miek@miek.nl>

* Update plugin/pkg/up/up.go

Co-Authored-By: miekg <miek@miek.nl>

plugin/pkg/up/up.go

@@ -9,10 +9,12 @@ import (
 
 // Probe is used to run a single Func until it returns true (indicating a target is healthy). If an Func
 // is already in progress no new one will be added, i.e. there is always a maximum of 1 checks in flight.
+// When failures start to happen we will back off every second failure up to maximum of 4 intervals.
 type Probe struct {
 	sync.Mutex
 	inprogress int
 	interval   time.Duration
+	max        time.Duration
 }
 
 // Func is used to determine if a target is alive. If so this function must return nil.
@@ -34,17 +36,22 @@ func (p *Probe) Do(f Func) {
 	// Passed the lock. Now run f for as long it returns false. If a true is returned
 	// we return from the goroutine and we can accept another Func to run.
 	go func() {
+		i := 1
 		for {
 			if err := f(); err == nil {
 				break
 			}
 			time.Sleep(interval)
+			if i%2 == 0 && i < 4 { // 4 is 2 doubles, so no need to increase anymore - this is *also* checked in double()
+				p.double()
+			}
 			p.Lock()
 			if p.inprogress == stop {
 				p.Unlock()
 				return
 			}
 			p.Unlock()
+			i++
 		}
 
 		p.Lock()
@@ -53,6 +60,15 @@ func (p *Probe) Do(f Func) {
 	}()
 }
 
+func (p *Probe) double() {
+	p.Lock()
+	p.interval *= 2
+	if p.interval > p.max {
+		p.interval = p.max
+	}
+	p.Unlock()
+}
+
 // Stop stops the probing.
 func (p *Probe) Stop() {
 	p.Lock()
@@ -61,12 +77,10 @@ func (p *Probe) Stop() {
 }
 
 // Start will initialize the probe manager, after which probes can be initiated with Do.
-func (p *Probe) Start(interval time.Duration) { p.SetInterval(interval) }
-
-// SetInterval sets the probing interval to be used by upcoming probes initiated with Do.
-func (p *Probe) SetInterval(interval time.Duration) {
+func (p *Probe) Start(interval time.Duration) {
 	p.Lock()
 	p.interval = interval
+	p.max = interval * multiplier
 	p.Unlock()
 }
 
@@ -74,4 +88,6 @@ const (
 	idle = iota
 	active
 	stop
+
+	multiplier = 4
 )