plugin/pkg/proxy: set IP_RECVERR on upstream UDP sockets (#8628)

Without IP_RECVERR, Linux passes only some ICMP errors, such as port
unreachable, to a connected UDP socket. It drops net and host unreachable
(net/ipv4/udp.c, __udp4_lib_err), so a query to an upstream behind an
unreachable route waits out the read timeout before forward tries the
next upstream, and health checks wait out theirs before marking it down.

Set IP_RECVERR (IPV6_RECVERR for IPv6) on the UDP sockets for upstream
queries and health checks, so the read fails when the ICMP error arrives.
TCP sockets are left alone, and nothing changes outside Linux. The health
check client gets an explicit 2s dial timeout, since a Dialer replaces
dns.Client's default one. Because the Dialer now exists from the start,
setDialer no longer creates a bare one when source_address is set, which
also lost the dial timeout before.

Fixes #8621

Signed-off-by: MohibShaikh <mohibuddin9@gmail.com>

plugin/pkg/proxy/connect.go

@@ -118,7 +118,7 @@ func (t *Transport) dial(ctx context.Context, proto string, tlsDeadline time.Tim
 	}
 	reqTime := time.Now()
 	timeout := t.dialTimeout()
-	dialer := &net.Dialer{Timeout: timeout}
+	dialer := &net.Dialer{Timeout: timeout, Control: recvErr}
 
 	if t.localAddress != nil {
 		if proto == "udp" {

plugin/pkg/proxy/health.go

@@ -53,6 +53,8 @@ func NewHealthChecker(proxyName, protocol string, recursionDesired bool, domain
 	case transport.DNS, transport.TLS:
 		c := new(dns.Client)
 		c.Net = "udp"
+		// A Dialer replaces dns.Client's own 2s dial timeout, so keep it.
+		c.Dialer = &net.Dialer{Timeout: 2 * time.Second, Control: recvErr}
 		setDefaultTimeout(c)
 
 		return &dnsHc{

plugin/pkg/proxy/health_test.go

@@ -217,6 +217,10 @@ func TestHealthLocalAddress(t *testing.T) {
 	if dnsClient.Dialer.LocalAddr.String() != "127.0.0.1:0" {
 		t.Errorf("Expected local address to be 127.0.0.1:0, got %s", dnsClient.Dialer.LocalAddr.String())
 	}
+	// A Dialer replaces dns.Client's own dial timeout, so it must carry one.
+	if dnsClient.Dialer.Timeout != 2*time.Second {
+		t.Errorf("Expected dial timeout to be 2s, got %s", dnsClient.Dialer.Timeout)
+	}
 
 	// check type of underlying transport
 	_, ok := dnsClient.Dialer.LocalAddr.(*net.UDPAddr)

plugin/pkg/proxy/recverr_linux.go

@@ -0,0 +1,37 @@
+//go:build linux
+
+package proxy
+
+import (
+	"syscall"
+
+	"github.com/coredns/coredns/plugin/pkg/log"
+
+	"golang.org/x/sys/unix"
+)
+
+// recvErr is a net.Dialer Control function that sets IP_RECVERR or IPV6_RECVERR
+// on UDP sockets. Without it Linux passes only some ICMP errors, such as port
+// unreachable, to a connected UDP socket, and drops net and host unreachable
+// (net/ipv4/udp.c, __udp4_lib_err). A read from an upstream behind an
+// unreachable route then waits out its timeout. With it, the read fails as soon
+// as the ICMP error arrives. Other networks are left alone.
+func recvErr(network, _ string, c syscall.RawConn) error {
+	if network != "udp4" && network != "udp6" {
+		return nil
+	}
+	return c.Control(func(fd uintptr) {
+		const maxInt = int(^uint(0) >> 1)
+		if fd > uintptr(maxInt) {
+			log.Warningf("Failed to set IP_RECVERR on socket: invalid file descriptor %d", fd)
+			return
+		}
+		level, opt := unix.SOL_IP, unix.IP_RECVERR
+		if network == "udp6" {
+			level, opt = unix.SOL_IPV6, unix.IPV6_RECVERR
+		}
+		if err := unix.SetsockoptInt(int(fd), level, opt, 1); err != nil { // #nosec G115 -- fd is range-checked above
+			log.Warningf("Failed to set IP_RECVERR on socket: %s", err)
+		}
+	})
+}

plugin/pkg/proxy/recverr_linux_test.go

@@ -0,0 +1,110 @@
+//go:build linux
+
+package proxy
+
+import (
+	"net"
+	"syscall"
+	"testing"
+
+	"github.com/coredns/coredns/plugin/pkg/transport"
+
+	"golang.org/x/sys/unix"
+)
+
+func sockopt(t *testing.T, c syscall.Conn, level, opt int) int {
+	t.Helper()
+	rc, err := c.SyscallConn()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var v int
+	var serr error
+	if err := rc.Control(func(fd uintptr) { v, serr = unix.GetsockoptInt(int(fd), level, opt) }); err != nil {
+		t.Fatal(err)
+	}
+	if serr != nil {
+		t.Fatal(serr)
+	}
+	return v
+}
+
+// TestRecvErrUpstreamSockets checks that the UDP sockets for upstream queries and
+// health checks get IP_RECVERR, and TCP sockets do not.
+func TestRecvErrUpstreamSockets(t *testing.T) {
+	udp, err := net.ListenPacket("udp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer udp.Close()
+	tcp, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tcp.Close()
+
+	tr := newTransport("test_recverr", udp.LocalAddr().String())
+	tr.Start()
+	defer tr.Stop()
+	pc, _, err := tr.Dial("udp")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pc.c.Close()
+	if v := sockopt(t, pc.c.Conn.(*net.UDPConn), unix.SOL_IP, unix.IP_RECVERR); v != 1 {
+		t.Errorf("upstream UDP socket: IP_RECVERR = %d, want 1", v)
+	}
+
+	hc := NewHealthChecker("test_recverr", transport.DNS, true, ".").(*dnsHc)
+	hconn, err := hc.c.Dial(udp.LocalAddr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer hconn.Close()
+	if v := sockopt(t, hconn.Conn.(*net.UDPConn), unix.SOL_IP, unix.IP_RECVERR); v != 1 {
+		t.Errorf("health check UDP socket: IP_RECVERR = %d, want 1", v)
+	}
+
+	// source_address goes through setDialer, which must keep the option.
+	hc.SetLocalAddress(net.ParseIP("127.0.0.1"))
+	lconn, err := hc.c.Dial(udp.LocalAddr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer lconn.Close()
+	if v := sockopt(t, lconn.Conn.(*net.UDPConn), unix.SOL_IP, unix.IP_RECVERR); v != 1 {
+		t.Errorf("health check UDP socket with source address: IP_RECVERR = %d, want 1", v)
+	}
+
+	ttr := newTransport("test_recverr_tcp", tcp.Addr().String())
+	ttr.Start()
+	defer ttr.Stop()
+	tpc, _, err := ttr.Dial("tcp")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tpc.c.Close()
+	if v := sockopt(t, tpc.c.Conn.(*net.TCPConn), unix.SOL_IP, unix.IP_RECVERR); v != 0 {
+		t.Errorf("upstream TCP socket: IP_RECVERR = %d, want 0", v)
+	}
+}
+
+// TestRecvErrUDP6 checks that IPv6 upstream sockets get IPV6_RECVERR.
+func TestRecvErrUDP6(t *testing.T) {
+	udp, err := net.ListenPacket("udp6", "[::1]:0")
+	if err != nil {
+		t.Skip("no IPv6 loopback:", err)
+	}
+	defer udp.Close()
+	tr := newTransport("test_recverr6", udp.LocalAddr().String())
+	tr.Start()
+	defer tr.Stop()
+	pc, _, err := tr.Dial("udp")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pc.c.Close()
+	if v := sockopt(t, pc.c.Conn.(*net.UDPConn), unix.SOL_IPV6, unix.IPV6_RECVERR); v != 1 {
+		t.Errorf("upstream UDP6 socket: IPV6_RECVERR = %d, want 1", v)
+	}
+}

plugin/pkg/proxy/recverr_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package proxy
+
+import "syscall"
+
+// recvErr does nothing outside Linux.
+func recvErr(_, _ string, _ syscall.RawConn) error { return nil }