plugin/forward: added option failfast_all_unhealthy_upstreams to return servfail if all upstreams are down (#6999)

* feat: option to return servfail if upstreams are down Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net> * fix based on review comments and added to Readme Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net> * add tests to improve code coverage Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net> * added failfast_all_unhealthy_upstreams option to forward plugin Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net> --------- Signed-off-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net> Co-authored-by: Puneet Loya <puneetloya@Puneets-MBP.attlocal.net>
2025-11-23 04:12:17 -05:00 · 2025-03-07 08:37:25 -08:00
parent 669ff527bf
commit 4de8fb57b2
5 changed files with 138 additions and 8 deletions
--- a/plugin/forward/health_test.go
+++ b/plugin/forward/health_test.go
@@ -277,3 +277,79 @@ func TestHealthDomain(t *testing.T) {
 		t.Errorf("Expected number of health checks with Domain==%s to be %d, got %d", hcDomain, 1, i1)
 	}
 }
+
+func TestAllUpstreamsDown(t *testing.T) {
+	qs := uint32(0)
+	s := dnstest.NewServer(func(w dns.ResponseWriter, r *dns.Msg) {
+		// count non-healthcheck queries
+		if r.Question[0].Name != "." {
+			atomic.AddUint32(&qs, 1)
+		}
+		// timeout
+	})
+	defer s.Close()
+
+	s1 := dnstest.NewServer(func(w dns.ResponseWriter, r *dns.Msg) {
+		// count non-healthcheck queries
+		if r.Question[0].Name != "." {
+			atomic.AddUint32(&qs, 1)
+		}
+		// timeout
+	})
+	defer s1.Close()
+
+	p := proxy.NewProxy("TestHealthAllUpstreamsDown", s.Addr, transport.DNS)
+	p1 := proxy.NewProxy("TestHealthAllUpstreamsDown2", s1.Addr, transport.DNS)
+	p.GetHealthchecker().SetReadTimeout(10 * time.Millisecond)
+	p1.GetHealthchecker().SetReadTimeout(10 * time.Millisecond)
+
+	f := New()
+	f.SetProxy(p)
+	f.SetProxy(p1)
+	f.failfastUnhealthyUpstreams = true
+	f.maxfails = 1
+	// Make proxys fail by checking health twice
+	// i.e, fails > maxfails
+	for range f.maxfails + 1 {
+		p.GetHealthchecker().Check(p)
+		p1.GetHealthchecker().Check(p1)
+	}
+
+	defer f.OnShutdown()
+
+	// Check if all proxies are down
+	if !p.Down(f.maxfails) || !p1.Down(f.maxfails) {
+		t.Fatalf("Expected all proxies to be down")
+	}
+	req := new(dns.Msg)
+	req.SetQuestion("example.org.", dns.TypeA)
+	resp, err := f.ServeDNS(context.TODO(), &test.ResponseWriter{}, req)
+
+	if resp != dns.RcodeServerFailure {
+		t.Errorf("Expected Response code: %d, Got: %d", dns.RcodeServerFailure, resp)
+	}
+
+	if err != ErrNoHealthy {
+		t.Errorf("Expected error message: no healthy proxies, Got: %s", err.Error())
+	}
+
+	q1 := atomic.LoadUint32(&qs)
+	if q1 != 0 {
+		t.Errorf("Expected queries to the upstream: 0, Got: %d", q1)
+	}
+
+	// set failfast to false to check if queries get answered
+	f.failfastUnhealthyUpstreams = false
+
+	req = new(dns.Msg)
+	req.SetQuestion("example.org.", dns.TypeA)
+	_, err = f.ServeDNS(context.TODO(), &test.ResponseWriter{}, req)
+	if err == ErrNoHealthy {
+		t.Error("Unexpected error message: no healthy proxies")
+	}
+
+	q1 = atomic.LoadUint32(&qs)
+	if q1 != 1 {
+		t.Errorf("Expected queries to the upstream: 1, Got: %d", q1)
+	}
+}