plugin/forward: added option failfast_all_unhealthy_upstreams to return servfail if all upstreams are down (#6999)

* feat: option to return servfail if upstreams are down

Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>

* fix based on review comments and added to Readme

Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>

* add tests to improve code coverage

Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>

* added failfast_all_unhealthy_upstreams option to forward plugin

Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>

---------

Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>
Co-authored-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>

plugin/forward/forward.go

@@ -45,11 +45,12 @@ type Forward struct {
 
 	nextAlternateRcodes []int
 
-	tlsConfig     *tls.Config
-	tlsServerName string
-	maxfails      uint32
-	expire        time.Duration
-	maxConcurrent int64
+	tlsConfig                  *tls.Config
+	tlsServerName              string
+	maxfails                   uint32
+	expire                     time.Duration
+	maxConcurrent              int64
+	failfastUnhealthyUpstreams bool
 
 	opts proxy.Options // also here for testing
 
@@ -126,12 +127,16 @@ func (f *Forward) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg
 			if fails < len(f.proxies) {
 				continue
 			}
-			// All upstream proxies are dead, assume healthcheck is completely broken and randomly
+
+			healthcheckBrokenCount.Add(1)
+			// All upstreams are dead, return servfail if all upstreams are down
+			if f.failfastUnhealthyUpstreams {
+				break
+			}
+			// assume healthcheck is completely broken and randomly
 			// select an upstream to connect to.
 			r := new(random)
 			proxy = r.List(f.proxies)[0]
-
-			healthcheckBrokenCount.Add(1)
 		}
 
 		if span != nil {