core: Add connection-level concurrency limiting to DNS-over-QUIC (#8213)

* core: Add connection-level concurrency limiting to DNS-over-QUIC

This PR adds max connections to prevent unbounded connection goroutine growth
for DoQ

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test to cover change

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---------

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

core/dnsserver/server_quic.go

@@ -43,6 +43,9 @@ const (
 
 	// DefaultQUICStreamWorkers is the default number of workers for processing QUIC streams.
 	DefaultQUICStreamWorkers = 1024
+
+	// DefaultQUICMaxConnections is the default maximum number of concurrent connections.
+	DefaultQUICMaxConnections = 200
 )
 
 // ServerQUIC represents an instance of a DNS-over-QUIC server.
@@ -54,6 +57,8 @@ type ServerQUIC struct {
 	quicListener      *quic.Listener
 	maxStreams        int
 	streamProcessPool chan struct{}
+	maxConnections    int
+	connSem           chan struct{}
 }
 
 // NewServerQUIC returns a new CoreDNS QUIC server and compiles all plugin in to it.
@@ -93,6 +98,15 @@ func NewServerQUIC(addr string, group []*Config) (*ServerQUIC, error) {
 		// Enable 0-RTT by default for all connections on the server-side.
 		Allow0RTT: true,
 	}
+	maxConnections := DefaultQUICMaxConnections
+	if len(group) > 0 && group[0] != nil && group[0].MaxQUICConnections != nil {
+		maxConnections = *group[0].MaxQUICConnections
+	}
+
+	var connSem chan struct{}
+	if maxConnections > 0 {
+		connSem = make(chan struct{}, maxConnections)
+	}
 
 	return &ServerQUIC{
 		Server:            s,
@@ -100,6 +114,8 @@ func NewServerQUIC(addr string, group []*Config) (*ServerQUIC, error) {
 		quicConfig:        quicConfig,
 		maxStreams:        maxStreams,
 		streamProcessPool: make(chan struct{}, streamProcessPoolSize),
+		maxConnections:    maxConnections,
+		connSem:           connSem,
 	}, nil
 }
 
@@ -133,8 +149,21 @@ func (s *ServerQUIC) ServeQUIC() error {
 			s.closeQUICConn(conn, DoQCodeInternalError)
 			return err
 		}
+		if s.connSem == nil {
+			go s.serveQUICConnection(conn)
+			continue
+		}
 
-		go s.serveQUICConnection(conn)
+		select {
+		case s.connSem <- struct{}{}:
+			go func(c *quic.Conn) {
+				defer func() { <-s.connSem }()
+				s.serveQUICConnection(c)
+			}(conn)
+
+		default:
+			_ = conn.CloseWithError(0, "too many connections")
+		}
 	}
 }