fix: release listeners when frps fails to start (#5574)

A constructor that binds a port before its fallible preparation leaks the
listener when a later step returns an error, and no one can close it because
the returned service is nil.

pkg/util/http: load the TLS keypair and build the auth middleware before
net.Listen, so binding is the last step that can run and a failure leaves no
bound port behind.

server: create the web server after auth, the tcpmux, main, kcp, quic, ssh
gateway and vhost listeners, and the nat hole controller, following what frpc
already does. If the web server itself fails, close the resources created so
far before returning. The port managers start their cleanup workers on
creation, so they are created after every fallible step too, and the metrics
collectors keep being enabled only when the web server is configured.

Add regression tests for all three failure paths.

pkg/util/http/server.go

@@ -53,11 +53,6 @@ func NewServer(cfg v1.WebServerConfig) (*Server, error) {
 		addr = ":http"
 	}
 
-	ln, err := net.Listen("tcp", addr)
-	if err != nil {
-		return nil, err
-	}
-
 	router := mux.NewRouter()
 	hs := &http.Server{
 		Addr:         addr,
@@ -67,7 +62,6 @@ func NewServer(cfg v1.WebServerConfig) (*Server, error) {
 	}
 	s := &Server{
 		addr:   addr,
-		ln:     ln,
 		hs:     hs,
 		router: router,
 	}
@@ -84,6 +78,13 @@ func NewServer(cfg v1.WebServerConfig) (*Server, error) {
 		}
 	}
 	s.authMiddleware = netpkg.NewHTTPAuthMiddleware(cfg.User, cfg.Password).SetAuthFailDelay(200 * time.Millisecond).Middleware
+
+	// Bind only after all fallible preparation so an error cannot leak the listener.
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+	s.ln = ln
 	return s, nil
 }
 

pkg/util/http/server_test.go

@@ -0,0 +1,55 @@
+// Copyright 2026 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	v1 "github.com/fatedier/frp/pkg/config/v1"
+)
+
+func TestNewServerDoesNotLeakListenerOnTLSFailure(t *testing.T) {
+	dir := t.TempDir()
+	certFile := filepath.Join(dir, "cert.pem")
+	keyFile := filepath.Join(dir, "key.pem")
+	require.NoError(t, os.WriteFile(certFile, []byte("invalid certificate"), 0o600))
+	require.NoError(t, os.WriteFile(keyFile, []byte("invalid key"), 0o600))
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	address := ln.Addr().String()
+	port := ln.Addr().(*net.TCPAddr).Port
+	require.NoError(t, ln.Close())
+
+	svr, err := NewServer(v1.WebServerConfig{
+		Addr: "127.0.0.1",
+		Port: port,
+		TLS:  &v1.TLSConfig{CertFile: certFile, KeyFile: keyFile},
+	})
+	if svr != nil {
+		defer svr.Close()
+	}
+	require.ErrorContains(t, err, "failed to find any PEM data in certificate input")
+	require.Nil(t, svr)
+
+	ln, err = net.Listen("tcp", address)
+	require.NoError(t, err, "web server port was not released")
+	require.NoError(t, ln.Close())
+}