New cache implementation and prefetch handing in mw/cache (#731)

* cache: add sharded cache implementation

Add Cache impl and a few tests. This cache is 256-way sharded, mainly
so each shard has it's own lock. The main cache structure is a readonly
jump plane into the right shard.

This should remove the single lock contention on the main lock and
provide more concurrent throughput - Obviously this hasn't been tested
or measured.

The key into the cache was made a uint32 (hash.fnv) and the hashing op
is not using strings.ToLower anymore remove any GC in that code path.

* here too

* Minimum shard size

* typos

* blurp

* small cleanups no defer

* typo

* Add freq based on Johns idea

* cherry-pick conflict resolv

* typo

* update from early code review from john

* add prefetch to the cache

* mw/cache: add prefetch

* remove println

* remove comment

* Fix tests

* Test prefetch in setup

* Add start of cache

* try add diff cache options

* Add hacky testcase

* not needed

* allow the use of a percentage for prefetch

If the TTL falls below xx% do a prefetch, if the record was popular.
Some other fixes and correctly prefetch only popular records.

middleware/cache/freq/freq.go

@@ -0,0 +1,54 @@
+// Package freq keeps track of last X seen events. The events themselves are not stored
+// here. So the Freq type should be added next to the thing it is tracking.
+package freq
+
+import (
+	"sync"
+	"time"
+)
+
+type Freq struct {
+	// Last time we saw a query for this element.
+	last time.Time
+	// Number of this in the last time slice.
+	hits int
+
+	sync.RWMutex
+}
+
+// New returns a new initialized Freq.
+func New(t time.Time) *Freq {
+	return &Freq{last: t, hits: 0}
+}
+
+// Updates updates the number of hits. Last time seen will be set to now.
+// If the last time we've seen this entity is within now - d, we increment hits, otherwise
+// we reset hits to 1. It returns the number of hits.
+func (f *Freq) Update(d time.Duration, now time.Time) int {
+	earliest := now.Add(-1 * d)
+	f.Lock()
+	defer f.Unlock()
+	if f.last.Before(earliest) {
+		f.last = now
+		f.hits = 1
+		return f.hits
+	}
+	f.last = now
+	f.hits++
+	return f.hits
+}
+
+// Hits returns the number of hits that we have seen, according to the updates we have done to f.
+func (f *Freq) Hits() int {
+	f.RLock()
+	defer f.RUnlock()
+	return f.hits
+}
+
+// Reset resets f to time t and hits to hits.
+func (f *Freq) Reset(t time.Time, hits int) {
+	f.Lock()
+	defer f.Unlock()
+	f.last = t
+	f.hits = hits
+}

middleware/cache/freq/freq_test.go

@@ -0,0 +1,36 @@
+package freq
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFreqUpdate(t *testing.T) {
+	now := time.Now().UTC()
+	f := New(now)
+	window := 1 * time.Minute
+
+	f.Update(window, time.Now().UTC())
+	f.Update(window, time.Now().UTC())
+	f.Update(window, time.Now().UTC())
+	hitsCheck(t, f, 3)
+
+	f.Reset(now, 0)
+	history := time.Now().UTC().Add(-3 * time.Minute)
+	f.Update(window, history)
+	hitsCheck(t, f, 1)
+}
+
+func TestReset(t *testing.T) {
+	f := New(time.Now().UTC())
+	f.Update(1*time.Minute, time.Now().UTC())
+	hitsCheck(t, f, 1)
+	f.Reset(time.Now().UTC(), 0)
+	hitsCheck(t, f, 0)
+}
+
+func hitsCheck(t *testing.T, f *Freq, expected int) {
+	if x := f.Hits(); x != expected {
+		t.Fatalf("Expected hits to be %d, got %d", expected, x)
+	}
+}