create pkg/reuseport (#3455)

* create pkg/reuseport

Move the core server listening functions to a new package so plugins can
use them.

Also make *all* servers use the functions here; as only the udp/tcp
listeners where using SO_REUSEPORT (if available). This is the only
actual change in this PR; in it's core it's just a move of 2 files.

This can also be used to cleanup the dance we're doing now for
re-acquiring the sockets in e.g. the metrics plugins and the ready
plugin.

Signed-off-by: Miek Gieben <miek@miek.nl>

* Also push a small doc update

Signed-off-by: Miek Gieben <miek@miek.nl>

core/dnsserver/listen_go111.go

@@ -1,33 +0,0 @@
-// +build go1.11
-// +build aix darwin dragonfly freebsd linux netbsd openbsd
-
-package dnsserver
-
-import (
-	"context"
-	"net"
-	"syscall"
-
-	"github.com/coredns/coredns/plugin/pkg/log"
-
-	"golang.org/x/sys/unix"
-)
-
-func reuseportControl(network, address string, c syscall.RawConn) error {
-	c.Control(func(fd uintptr) {
-		if err := unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
-			log.Warningf("Failed to set SO_REUSEPORT on socket: %s", err)
-		}
-	})
-	return nil
-}
-
-func listen(network, addr string) (net.Listener, error) {
-	lc := net.ListenConfig{Control: reuseportControl}
-	return lc.Listen(context.Background(), network, addr)
-}
-
-func listenPacket(network, addr string) (net.PacketConn, error) {
-	lc := net.ListenConfig{Control: reuseportControl}
-	return lc.ListenPacket(context.Background(), network, addr)
-}

core/dnsserver/listen_go_not111.go

@@ -1,11 +0,0 @@
-// +build !go1.11 !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd
-
-package dnsserver
-
-import "net"
-
-func listen(network, addr string) (net.Listener, error) { return net.Listen(network, addr) }
-
-func listenPacket(network, addr string) (net.PacketConn, error) {
-	return net.ListenPacket(network, addr)
-}

core/dnsserver/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/edns"
 	"github.com/coredns/coredns/plugin/pkg/log"
 	"github.com/coredns/coredns/plugin/pkg/rcode"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/trace"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 	"github.com/coredns/coredns/request"
@@ -126,7 +127,7 @@ func (s *Server) ServePacket(p net.PacketConn) error {
 
 // Listen implements caddy.TCPServer interface.
 func (s *Server) Listen() (net.Listener, error) {
-	l, err := listen("tcp", s.Addr[len(transport.DNS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.DNS+"://"):])
 	if err != nil {
 		return nil, err
 	}
@@ -140,7 +141,7 @@ func (s *Server) WrapListener(ln net.Listener) net.Listener {
 
 // ListenPacket implements caddy.UDPServer interface.
 func (s *Server) ListenPacket() (net.PacketConn, error) {
-	p, err := listenPacket("udp", s.Addr[len(transport.DNS+"://"):])
+	p, err := reuseport.ListenPacket("udp", s.Addr[len(transport.DNS+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_grpc.go

@@ -8,6 +8,7 @@ import (
 	"net"
 
 	"github.com/coredns/coredns/pb"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 
 	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
@@ -72,7 +73,7 @@ func (s *ServergRPC) ServePacket(p net.PacketConn) error { return nil }
 // Listen implements caddy.TCPServer interface.
 func (s *ServergRPC) Listen() (net.Listener, error) {
 
-	l, err := net.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.GRPC+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_https.go

@@ -12,6 +12,7 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/dnsutil"
 	"github.com/coredns/coredns/plugin/pkg/doh"
 	"github.com/coredns/coredns/plugin/pkg/response"
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 )
 
@@ -61,7 +62,7 @@ func (s *ServerHTTPS) ServePacket(p net.PacketConn) error { return nil }
 // Listen implements caddy.TCPServer interface.
 func (s *ServerHTTPS) Listen() (net.Listener, error) {
 
-	l, err := net.Listen("tcp", s.Addr[len(transport.HTTPS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.HTTPS+"://"):])
 	if err != nil {
 		return nil, err
 	}

core/dnsserver/server_tls.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/coredns/coredns/plugin/pkg/reuseport"
 	"github.com/coredns/coredns/plugin/pkg/transport"
 
 	"github.com/miekg/dns"
@@ -57,7 +58,7 @@ func (s *ServerTLS) ServePacket(p net.PacketConn) error { return nil }
 
 // Listen implements caddy.TCPServer interface.
 func (s *ServerTLS) Listen() (net.Listener, error) {
-	l, err := net.Listen("tcp", s.Addr[len(transport.TLS+"://"):])
+	l, err := reuseport.Listen("tcp", s.Addr[len(transport.TLS+"://"):])
 	if err != nil {
 		return nil, err
 	}

plugin.md

@@ -64,6 +64,11 @@ a *Metrics* section detailing the metrics.
 If the plugin supports signalling readiness it should have a *Ready* section detailing how it
 works, and implement the `ready.Readiness` interface.
 
+## Opening Sockets
+
+See the plugin/pkg/reuseport for `Listen` and `ListenPacket` functions. Using these functions makes
+you plugin handle reload events better.
+
 ## Documentation
 
 Each plugin should have a README.md explaining what the plugin does and how it is configured. The

plugin/pkg/reuseport/listen_go111.go

@@ -0,0 +1,37 @@
+// +build go1.11
+// +build aix darwin dragonfly freebsd linux netbsd openbsd
+
+package reuseport
+
+import (
+	"context"
+	"net"
+	"syscall"
+
+	"github.com/coredns/coredns/plugin/pkg/log"
+
+	"golang.org/x/sys/unix"
+)
+
+func control(network, address string, c syscall.RawConn) error {
+	c.Control(func(fd uintptr) {
+		if err := unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			log.Warningf("Failed to set SO_REUSEPORT on socket: %s", err)
+		}
+	})
+	return nil
+}
+
+// Listen announces on the local network address. See net.Listen for more information.
+// If SO_REUSEPORT is available it will be set on the socket.
+func Listen(network, addr string) (net.Listener, error) {
+	lc := net.ListenConfig{Control: control}
+	return lc.Listen(context.Background(), network, addr)
+}
+
+// ListenPacket announces on the local network address. See net.ListenPacket for more information.
+// If SO_REUSEPORT is available it will be set on the socket.
+func ListenPacket(network, addr string) (net.PacketConn, error) {
+	lc := net.ListenConfig{Control: control}
+	return lc.ListenPacket(context.Background(), network, addr)
+}

plugin/pkg/reuseport/listen_go_not111.go

@@ -0,0 +1,13 @@
+// +build !go1.11 !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd
+
+package reuseport
+
+import "net"
+
+// Listen is a wrapper around net.Listen.
+func Listen(network, addr string) (net.Listener, error) { return net.Listen(network, addr) }
+
+// ListenPacket is a wrapper around net.ListenPacket.
+func ListenPacket(network, addr string) (net.PacketConn, error) {
+	return net.ListenPacket(network, addr)
+}