plugin/secondary: Retry initial transfer until successful (#4663)

* retry initial transfer

Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* fix import grouping

Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* add test; use backoff timeout

Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* fix import order

Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

* manual backoff

Signed-off-by: Chris O'Haver <cohaver@infoblox.com>

plugin/secondary/setup.go

@@ -1,14 +1,19 @@
 package secondary
 
 import (
+	"time"
+
 	"github.com/coredns/caddy"
 	"github.com/coredns/coredns/core/dnsserver"
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/file"
+	clog "github.com/coredns/coredns/plugin/pkg/log"
 	"github.com/coredns/coredns/plugin/pkg/parse"
 	"github.com/coredns/coredns/plugin/pkg/upstream"
 )
 
+var log = clog.NewWithPlugin("secondary")
+
 func init() { plugin.Register("secondary", setup) }
 
 func setup(c *caddy.Controller) error {
@@ -24,7 +29,21 @@ func setup(c *caddy.Controller) error {
 			c.OnStartup(func() error {
 				z.StartupOnce.Do(func() {
 					go func() {
-						z.TransferIn()
+						dur := time.Millisecond * 250
+						step := time.Duration(2)
+						max := time.Second * 10
+						for {
+							err := z.TransferIn()
+							if err == nil {
+								break
+							}
+							log.Warningf("All '%s' masters failed to transfer, retrying in %s: %s", n, dur.String(), err)
+							time.Sleep(dur)
+							dur = step * dur
+							if dur > max {
+								dur = max
+							}
+						}
 						z.Update()
 					}()
 				})