Add max conn limit to https3 (#8187)

* Add max conn limit to https3

This PR adds max conn limit to https3, similiar to https

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add Test to cover change

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Address review feedback

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update README and setup

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update plugin/https3/README.md

Co-authored-by: Ville Vesilehto <ville@vesilehto.fi>
Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---------

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
Co-authored-by: Ville Vesilehto <ville@vesilehto.fi>

core/dnsserver/config.go

@@ -118,6 +118,10 @@ type Config struct {
 	// This is nil if not specified, allowing for a default to be used.
 	MaxHTTPS3Streams *int
 
+	// MaxHTTPS3Connections defines the maximum number of concurrent HTTPS3 connections.
+	// This is nil if not specified, allowing for a default to be used.
+	MaxHTTPS3Connections *int
+
 	// Timeouts for connection-oriented servers. Exact applicability depends on transport.
 	ReadTimeout  time.Duration
 	WriteTimeout time.Duration

core/dnsserver/server_https3.go

@@ -34,12 +34,45 @@ const (
 // ServerHTTPS3 represents a DNS-over-HTTP/3 server.
 type ServerHTTPS3 struct {
 	*Server
-	httpsServer  *http3.Server
+	httpsServer    *http3.Server
-	listenAddr   net.Addr
+	listenAddr     net.Addr
-	tlsConfig    *tls.Config
+	tlsConfig      *tls.Config
-	quicConfig   *quic.Config
+	quicConfig     *quic.Config
-	validRequest func(*http.Request) bool
+	validRequest   func(*http.Request) bool
-	maxStreams   int
+	maxStreams     int
+	maxConnections int
+}
+
+type limitQUICListener struct {
+	http3.QUICListener
+	sem chan struct{}
+}
+
+func newLimitQUICListener(ln http3.QUICListener, maxConnections int) http3.QUICListener {
+	return &limitQUICListener{
+		QUICListener: ln,
+		sem:          make(chan struct{}, maxConnections),
+	}
+}
+
+func (l *limitQUICListener) Accept(ctx context.Context) (*quic.Conn, error) {
+	for {
+		conn, err := l.QUICListener.Accept(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		select {
+		case l.sem <- struct{}{}:
+			go func() {
+				<-conn.Context().Done()
+				<-l.sem
+			}()
+			return conn, nil
+		default:
+			_ = conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeExcessiveLoad), "too many connections")
+		}
+	}
 }
 
 // NewServerHTTPS3 builds the HTTP/3 (DoH3) server.
@@ -79,6 +112,11 @@ func NewServerHTTPS3(addr string, group []*Config) (*ServerHTTPS3, error) {
 		maxStreams = *group[0].MaxHTTPS3Streams
 	}
 
+	maxConnections := DefaultHTTPSMaxConnections
+	if len(group) > 0 && group[0] != nil && group[0].MaxHTTPS3Connections != nil {
+		maxConnections = *group[0].MaxHTTPS3Connections
+	}
+
 	// QUIC transport config with stream limits (0 means use QUIC default)
 	qconf := &quic.Config{
 		MaxIdleTimeout: s.IdleTimeout,
@@ -99,14 +137,14 @@ func NewServerHTTPS3(addr string, group []*Config) (*ServerHTTPS3, error) {
 	}
 
 	sh := &ServerHTTPS3{
-		Server:       s,
+		Server:         s,
-		tlsConfig:    tlsConfig,
+		tlsConfig:      tlsConfig,
-		httpsServer:  h3srv,
+		httpsServer:    h3srv,
-		quicConfig:   qconf,
+		quicConfig:     qconf,
-		validRequest: validator,
+		validRequest:   validator,
-		maxStreams:   maxStreams,
+		maxStreams:     maxStreams,
+		maxConnections: maxConnections,
 	}
-
 	h3srv.Handler = sh
 
 	return sh, nil
@@ -131,8 +169,26 @@ func (s *ServerHTTPS3) ServePacket(pc net.PacketConn) error {
 	s.m.Lock()
 	s.listenAddr = pc.LocalAddr()
 	s.m.Unlock()
-	// Serve HTTP/3 over QUIC
+
-	return s.httpsServer.Serve(pc)
+	if s.maxConnections <= 0 {
+		return s.httpsServer.Serve(pc)
+	}
+
+	quicConfig := s.quicConfig.Clone()
+	if s.httpsServer.EnableDatagrams {
+		quicConfig.EnableDatagrams = true
+	}
+
+	tr := &quic.Transport{Conn: pc}
+	defer tr.Close()
+
+	ln, err := tr.ListenEarly(http3.ConfigureTLSConfig(s.tlsConfig), quicConfig)
+	if err != nil {
+		return err
+	}
+	defer ln.Close()
+
+	return s.httpsServer.ServeListener(newLimitQUICListener(ln, s.maxConnections))
 }
 
 // Listen function not used in HTTP/3, but defined for compatibility

core/dnsserver/server_https3_test.go

@@ -2,9 +2,11 @@ package dnsserver
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"errors"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -14,6 +16,8 @@ import (
 	"github.com/coredns/coredns/plugin"
 
 	"github.com/miekg/dns"
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
 )
 
 const (
@@ -406,3 +410,129 @@ func TestServeHTTP3DoesNotLeakBodyReadError(t *testing.T) {
 		t.Fatalf("expected sanitized body %q, got %q", "invalid request", got)
 	}
 }
+
+func requireHTTPS3ConnectionRejected(t *testing.T, err error) {
+	t.Helper()
+
+	var appErr *quic.ApplicationError
+	if errors.As(err, &appErr) {
+		if !appErr.Remote {
+			t.Fatalf("connection closed with local application error: %v", err)
+		}
+		if appErr.ErrorCode != quic.ApplicationErrorCode(http3.ErrCodeExcessiveLoad) {
+			t.Fatalf("application error code = %#x, want %#x", appErr.ErrorCode, http3.ErrCodeExcessiveLoad)
+		}
+		return
+	}
+
+	// An application close sent before 1-RTT keys are available is encoded as
+	// the generic transport-level APPLICATION_ERROR. In that case, the peer
+	// cannot observe the HTTP/3 application code or reason phrase.
+	var transportErr *quic.TransportError
+	if errors.As(err, &transportErr) {
+		if !transportErr.Remote {
+			t.Fatalf("connection closed with local transport error: %v", err)
+		}
+		if transportErr.ErrorCode != quic.ApplicationErrorErrorCode {
+			t.Fatalf("transport error code = %#x, want APPLICATION_ERROR", transportErr.ErrorCode)
+		}
+		return
+	}
+
+	t.Fatalf("second connection error has unexpected type %T: %v", err, err)
+}
+
+func TestServerHTTPS3MaxConnections(t *testing.T) {
+	maxConnections := 1
+
+	config := testConfig("https3", echoPlugin{})
+	config.TLSConfig = mustMakeQUICServerTLSConfig(t)
+	config.MaxHTTPS3Connections = &maxConnections
+
+	server, err := NewServerHTTPS3("127.0.0.1:0", []*Config{config})
+	if err != nil {
+		t.Fatalf("NewServerHTTPS3() failed: %v", err)
+	}
+
+	accepted := make(chan struct{}, 1)
+	server.httpsServer.ConnContext = func(ctx context.Context, _ *quic.Conn) context.Context {
+		accepted <- struct{}{}
+		return ctx
+	}
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("net.ListenPacket() failed: %v", err)
+	}
+	defer pc.Close()
+
+	serveErrCh := make(chan error, 1)
+	go func() {
+		serveErrCh <- server.ServePacket(pc)
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	clientTLS := mustMakeQUICClientTLSConfig()
+	clientTLS.NextProtos = []string{"h3"}
+
+	first, err := quic.DialAddr(ctx, pc.LocalAddr().String(), clientTLS, &quic.Config{})
+	if err != nil {
+		t.Fatalf("first quic.DialAddr() failed: %v", err)
+	}
+
+	select {
+	case <-accepted:
+	case <-time.After(2 * time.Second):
+		t.Fatal("first connection was not accepted by the HTTP/3 server")
+	}
+
+	second, err := quic.DialAddr(ctx, pc.LocalAddr().String(), clientTLS, &quic.Config{})
+	if err == nil {
+		defer second.CloseWithError(0, "")
+
+		select {
+		case <-second.Context().Done():
+			err = context.Cause(second.Context())
+		case <-time.After(2 * time.Second):
+			t.Fatal("second connection remained open after the maximum connection count was reached")
+		}
+	}
+	if err == nil {
+		t.Fatal("second connection closed without reporting an error")
+	}
+	requireHTTPS3ConnectionRejected(t, err)
+
+	if err := first.CloseWithError(0, ""); err != nil {
+		t.Fatalf("first.CloseWithError() failed: %v", err)
+	}
+	select {
+	case <-first.Context().Done():
+	case <-time.After(2 * time.Second):
+		t.Fatal("first connection did not close")
+	}
+
+	stopErrCh := make(chan error, 1)
+	go func() {
+		stopErrCh <- server.Stop()
+	}()
+
+	select {
+	case err := <-stopErrCh:
+		if err != nil {
+			t.Fatalf("Stop() failed: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Stop() did not return")
+	}
+
+	select {
+	case err := <-serveErrCh:
+		if !errors.Is(err, http.ErrServerClosed) {
+			t.Fatalf("ServePacket() error = %v, want http.ErrServerClosed", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("ServePacket() did not stop after Stop()")
+	}
+}

plugin/https3/README.md

@@ -15,10 +15,12 @@ This plugin can only be used once per HTTPS3 listener block.
 ```txt
 https3 {
     max_streams POSITIVE_INTEGER
+    max_connections POSITIVE_INTEGER
 }
 ```
 
 * `max_streams` limits the number of concurrent QUIC streams per connection. This helps prevent unbounded streams on a single connection, exhausting server resources. The default value is 256 if not specified. Set to 0 to use underlying QUIC transport default.
+* `max_connections` limits the number of concurrent HTTPS/3 connections accepted by the server. The default value is 200 if not specified. Connections above the configured limit are rejected. Set to 0 to disable the CoreDNS connection limit.
 
 ## Examples
 

plugin/https3/setup.go

@@ -54,6 +54,22 @@ func parseDOH3(c *caddy.Controller) error {
 				return c.Err("max_streams already defined for this server block")
 			}
 			config.MaxHTTPS3Streams = &val
+		case "max_connections":
+			args := c.RemainingArgs()
+			if len(args) != 1 {
+				return c.ArgErr()
+			}
+			val, err := strconv.Atoi(args[0])
+			if err != nil {
+				return c.Errf("invalid max_connections value '%s': %v", args[0], err)
+			}
+			if val < 0 {
+				return c.Errf("max_connections must be a non-negative integer: %d", val)
+			}
+			if config.MaxHTTPS3Connections != nil {
+				return c.Err("max_connections already defined for this server block")
+			}
+			config.MaxHTTPS3Connections = &val
 		default:
 			return c.Errf("unknown property '%s'", c.Val())
 		}