pkg/up: add generic run-this-functions (#1481)

This adds a generic way of start a check function to check a backend.
This package can be used to kick off healthchecks. The package makes
sure only 1 is run at any one time.

It should allow for:
See upstream error -> kick off healthcheck

and not to worry about overwhelming the upstream with a barrage of
queries.

plugin/pkg/up/up_test.go

@@ -0,0 +1,40 @@
+package up
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestUp(t *testing.T) {
+	pr := New()
+	wg := sync.WaitGroup{}
+	hits := int32(0)
+
+	upfunc := func(s string) bool {
+		atomic.AddInt32(&hits, 1)
+		// Sleep tiny amount so that our other pr.Do() calls hit the lock.
+		time.Sleep(3 * time.Millisecond)
+		wg.Done()
+		return true
+	}
+
+	pr.Start("nonexistent", 5*time.Millisecond)
+	defer pr.Stop()
+
+	// These functions AddInt32 to the same hits variable, but we only want to wait when
+	// upfunc finishes, as that only calls Done() on the waitgroup.
+	upfuncNoWg := func(s string) bool { atomic.AddInt32(&hits, 1); return true }
+	wg.Add(1)
+	pr.Do(upfunc)
+	pr.Do(upfuncNoWg)
+	pr.Do(upfuncNoWg)
+
+	wg.Wait()
+
+	h := atomic.LoadInt32(&hits)
+	if h != 1 {
+		t.Errorf("Expected hits to be %d, got %d", 1, h)
+	}
+}