test/e2e: add compatibility test suite for wire protocol version negotiation (#5297)

.gitignore

@@ -18,6 +18,7 @@ release/
 test/bin/
 vendor/
 lastversion/
+.cache/
 dist/
 .idea/
 .vscode/

Makefile

@@ -2,8 +2,10 @@ export PATH := $(PATH):`go env GOPATH`/bin
 export GO111MODULE=on
 LDFLAGS := -s -w
 NOWEB_TAG = $(shell [ ! -d web/frps/dist ] || [ ! -d web/frpc/dist ] && echo ',noweb')
+FRP_COMPAT_BASELINE_COUNT ?= 8
+FRP_COMPAT_FLOOR_VERSION ?= 0.61.0
 
-.PHONY: web frps-web frpc-web frps frpc
+.PHONY: web frps-web frpc-web frps frpc e2e-compatibility-smoke e2e-compatibility e2e-compatibility-floor
 
 all: env fmt web build
 
@@ -53,6 +55,15 @@ e2e:
 e2e-trace:
 	DEBUG=true LOG_LEVEL=trace ./hack/run-e2e.sh
 
+e2e-compatibility-smoke: build
+	FRP_COMPAT_BASELINE_COUNT=1 ./hack/run-e2e-compatibility.sh
+
+e2e-compatibility: build
+	FRP_COMPAT_BASELINE_COUNT="$(FRP_COMPAT_BASELINE_COUNT)" ./hack/run-e2e-compatibility.sh
+
+e2e-compatibility-floor: build
+	FRP_COMPAT_BASELINE_VERSIONS="$(FRP_COMPAT_FLOOR_VERSION)" ./hack/run-e2e-compatibility.sh
+
 e2e-compatibility-last-frpc:
 	if [ ! -d "./lastversion" ]; then \
 		TARGET_DIRNAME=lastversion ./hack/download.sh; \
@@ -73,3 +84,5 @@ clean:
 	rm -f ./bin/frpc
 	rm -f ./bin/frps
 	rm -rf ./lastversion
+	rm -rf ./.cache
+	rm -rf ./.compat

hack/run-e2e-compatibility.sh

@@ -0,0 +1,162 @@
+#!/bin/sh
+
+set -eu
+
+SCRIPT=$(readlink -f "$0")
+ROOT=$(unset CDPATH && cd "$(dirname "$SCRIPT")/.." && pwd)
+
+if ! command -v ginkgo >/dev/null 2>&1; then
+    echo "ginkgo not found, try to install..."
+    go install github.com/onsi/ginkgo/v2/ginkgo@v2.23.4
+fi
+
+debug=false
+if [ "x${DEBUG:-}" = "xtrue" ]; then
+    debug=true
+fi
+logLevel=debug
+if [ "${LOG_LEVEL:-}" ]; then
+    logLevel="${LOG_LEVEL}"
+fi
+
+currentFrpsPath=${CURRENT_FRPS_PATH:-${ROOT}/bin/frps}
+currentFrpcPath=${CURRENT_FRPC_PATH:-${ROOT}/bin/frpc}
+baselineCount=${FRP_COMPAT_BASELINE_COUNT:-8}
+targetOS=${TARGET_OS:-$(go env GOOS)}
+targetArch=${TARGET_ARCH:-$(go env GOARCH)}
+targetPlatform="${targetOS}_${targetArch}"
+cacheRoot=${FRP_COMPAT_CACHE_DIR:-${ROOT}/.cache/e2e-compat}
+
+check_file() {
+    if [ ! -f "$2" ]; then
+        echo "$1 not found: $2"
+        exit 1
+    fi
+}
+
+check_file "current frps" "${currentFrpsPath}"
+check_file "current frpc" "${currentFrpcPath}"
+
+run_current_current=true
+
+run_compatibility() {
+    baselineVersion=$1
+    baselineFrpsPath=$2
+    baselineFrpcPath=$3
+
+    check_file "baseline frps" "${baselineFrpsPath}"
+    check_file "baseline frpc" "${baselineFrpcPath}"
+
+    echo "Running compatibility e2e with baseline ${baselineVersion}"
+    ginkgo -nodes=1 --poll-progress-after=60s "${ROOT}/test/e2e/compatibility" -- \
+        -current-frps-path="${currentFrpsPath}" \
+        -current-frpc-path="${currentFrpcPath}" \
+        -baseline-frps-path="${baselineFrpsPath}" \
+        -baseline-frpc-path="${baselineFrpcPath}" \
+        -baseline-version="${baselineVersion}" \
+        -run-current-current="${run_current_current}" \
+        -log-level="${logLevel}" \
+        -debug="${debug}"
+    run_current_current=false
+}
+
+github_api_curl() {
+    if [ "${GITHUB_TOKEN:-}" ]; then
+        curl -fsSL \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            "$1"
+    else
+        curl -fsSL "$1"
+    fi
+}
+
+resolve_versions() {
+    if [ "${FRP_COMPAT_BASELINE_VERSIONS:-}" ]; then
+        printf "%s\n" "${FRP_COMPAT_BASELINE_VERSIONS}"
+        return
+    fi
+
+    case "${baselineCount}" in
+        '' | *[!0-9]*)
+            echo "FRP_COMPAT_BASELINE_COUNT must be a positive integer: ${baselineCount}" >&2
+            exit 1
+            ;;
+    esac
+    if [ "${baselineCount}" -eq 0 ]; then
+        echo "FRP_COMPAT_BASELINE_COUNT must be greater than 0" >&2
+        exit 1
+    fi
+    if [ "${baselineCount}" -gt 100 ]; then
+        echo "FRP_COMPAT_BASELINE_COUNT must be less than or equal to 100" >&2
+        exit 1
+    fi
+
+    releaseURL="https://api.github.com/repos/fatedier/frp/releases?per_page=100"
+    resolvedVersions=""
+    if releases=$(github_api_curl "${releaseURL}" 2>/dev/null); then
+        resolvedVersions=$(printf "%s\n" "${releases}" |
+            sed -n 's/.*"tag_name":[[:space:]]*"v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)".*/\1/p' |
+            awk '!seen[$0]++' |
+            head -n "${baselineCount}" |
+            tr '\n' ' ' |
+            sed 's/[[:space:]]*$//')
+    else
+        echo "Failed to fetch release metadata from GitHub API, falling back to GitHub releases page." >&2
+    fi
+
+    if [ -z "${resolvedVersions}" ]; then
+        releasesPageURL="https://github.com/fatedier/frp/releases"
+        if ! releases=$(curl -fsSL "${releasesPageURL}"); then
+            echo "Failed to fetch release metadata from GitHub: ${releasesPageURL}" >&2
+            echo "Set FRP_COMPAT_BASELINE_VERSIONS to run with explicit baseline versions." >&2
+            exit 1
+        fi
+        resolvedVersions=$(printf "%s\n" "${releases}" |
+            grep -o 'releases/tag/v[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*"' |
+            sed 's#.*/v##; s/"$//' |
+            awk '!seen[$0]++' |
+            head -n "${baselineCount}" |
+            tr '\n' ' ' |
+            sed 's/[[:space:]]*$//')
+    fi
+
+    set -- ${resolvedVersions}
+    if [ "$#" -lt "${baselineCount}" ]; then
+        echo "Only resolved $# stable release versions from GitHub, expected ${baselineCount}." >&2
+        echo "Set FRP_COMPAT_BASELINE_VERSIONS to run with explicit baseline versions." >&2
+        exit 1
+    fi
+
+    printf "%s\n" "${resolvedVersions}"
+}
+
+if [ "${BASELINE_FRPS_PATH:-}" ] || [ "${BASELINE_FRPC_PATH:-}" ]; then
+    if [ -z "${BASELINE_FRPS_PATH:-}" ] || [ -z "${BASELINE_FRPC_PATH:-}" ]; then
+        echo "BASELINE_FRPS_PATH and BASELINE_FRPC_PATH must be set together"
+        exit 1
+    fi
+    run_compatibility "${FRP_COMPAT_BASELINE_VERSION:-custom}" "${BASELINE_FRPS_PATH}" "${BASELINE_FRPC_PATH}"
+    exit 0
+fi
+
+versions=$(resolve_versions)
+echo "Compatibility baseline versions: ${versions}"
+
+mkdir -p "${cacheRoot}"
+for version in ${versions}; do
+    baselineDir="${cacheRoot}/${version}/${targetPlatform}"
+    if [ ! -f "${baselineDir}/frps" ] || [ ! -f "${baselineDir}/frpc" ]; then
+        tmpDir="${cacheRoot}/.download-${version}-${targetPlatform}"
+        rm -rf "${tmpDir}"
+        (
+            cd "${cacheRoot}"
+            FRP_VERSION="${version}" TARGET_DIRNAME="$(basename "${tmpDir}")" "${ROOT}/hack/download.sh"
+        )
+        mkdir -p "$(dirname "${baselineDir}")"
+        rm -rf "${baselineDir}"
+        mv "${tmpDir}" "${baselineDir}"
+    fi
+
+    run_compatibility "${version}" "${baselineDir}/frps" "${baselineDir}/frpc"
+done

test/e2e/compatibility/compatibility_test.go

@@ -0,0 +1,236 @@
+package compatibility
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+
+	"github.com/fatedier/frp/pkg/util/log"
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/test/e2e/framework/consts"
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+)
+
+type compatTestContext struct {
+	CurrentFRPSPath   string
+	CurrentFRPCPath   string
+	BaselineFRPSPath  string
+	BaselineFRPCPath  string
+	BaselineVersion   string
+	LogLevel          string
+	Debug             bool
+	RunCurrentCurrent bool
+}
+
+var compatCtx compatTestContext
+
+func registerFlags(flags *flag.FlagSet) {
+	flags.StringVar(&compatCtx.CurrentFRPSPath, "current-frps-path", "../../bin/frps", "The current frps binary to use.")
+	flags.StringVar(&compatCtx.CurrentFRPCPath, "current-frpc-path", "../../bin/frpc", "The current frpc binary to use.")
+	flags.StringVar(&compatCtx.BaselineFRPSPath, "baseline-frps-path", "", "The baseline frps binary to use.")
+	flags.StringVar(&compatCtx.BaselineFRPCPath, "baseline-frpc-path", "", "The baseline frpc binary to use.")
+	flags.StringVar(&compatCtx.BaselineVersion, "baseline-version", "custom", "The baseline version label for reporting.")
+	flags.StringVar(&compatCtx.LogLevel, "log-level", "debug", "Log level.")
+	flags.BoolVar(&compatCtx.Debug, "debug", false, "Enable debug mode to print detailed info.")
+	flags.BoolVar(&compatCtx.RunCurrentCurrent, "run-current-current", true, "Run current frps/current frpc sanity checks.")
+}
+
+func validateCompatContext(t *compatTestContext) error {
+	paths := map[string]string{
+		"current-frps-path":  t.CurrentFRPSPath,
+		"current-frpc-path":  t.CurrentFRPCPath,
+		"baseline-frps-path": t.BaselineFRPSPath,
+		"baseline-frpc-path": t.BaselineFRPCPath,
+	}
+	for name, path := range paths {
+		if path == "" {
+			return fmt.Errorf("%s can't be empty", name)
+		}
+		if _, err := os.Stat(path); err != nil {
+			return fmt.Errorf("load %s error: %v", name, err)
+		}
+	}
+	return nil
+}
+
+func TestMain(m *testing.M) {
+	registerFlags(flag.CommandLine)
+	flag.Parse()
+
+	if err := validateCompatContext(&compatCtx); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	framework.TestContext.Debug = compatCtx.Debug
+	framework.TestContext.LogLevel = compatCtx.LogLevel
+	log.InitLogger("console", compatCtx.LogLevel, 0, true)
+	os.Exit(m.Run())
+}
+
+func TestCompatibilityE2E(t *testing.T) {
+	gomega.RegisterFailHandler(framework.Fail)
+
+	suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
+	suiteConfig.EmitSpecProgress = true
+	suiteConfig.RandomizeAllSpecs = true
+	ginkgo.RunSpecs(t, "frp compatibility e2e suite", suiteConfig, reporterConfig)
+}
+
+var _ = ginkgo.Describe("[Compatibility: WireProtocol]", func() {
+	f := framework.NewDefaultFramework()
+
+	ginkgo.It("current frps and current frpc support v1 and v2", func() {
+		if !compatCtx.RunCurrentCurrent {
+			ginkgo.Skip("current/current sanity checks already ran")
+		}
+
+		webPort := f.AllocPort()
+		serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
+webServer.port = %d
+		`, webPort)
+
+		v1PortName := port.GenName("CompatCurrentV1")
+		v1ClientConf := tcpClientConfig("compat-current-v1", v1PortName, `
+clientID = "compat-current-v1"
+transport.wireProtocol = "v1"
+`)
+
+		v2PortName := port.GenName("CompatCurrentV2")
+		v2ClientConf := tcpClientConfig("compat-current-v2", v2PortName, `
+clientID = "compat-current-v2"
+transport.wireProtocol = "v2"
+`)
+
+		f.RunProcessesWithBinaries(
+			compatCtx.CurrentFRPSPath,
+			compatCtx.CurrentFRPCPath,
+			serverConf,
+			[]string{v1ClientConf, v2ClientConf},
+		)
+
+		framework.NewRequestExpect(f).PortName(v1PortName).Ensure()
+		framework.NewRequestExpect(f).PortName(v2PortName).Ensure()
+		expectClientWireProtocol(webPort, "compat-current-v1", "v1")
+		expectClientWireProtocol(webPort, "compat-current-v2", "v2")
+	})
+
+	ginkgo.It("current frps accepts baseline frpc using v1", func() {
+		webPort := f.AllocPort()
+		serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
+webServer.port = %d
+`, webPort)
+
+		portName := port.GenName("CompatBaselineFRPC")
+		clientConf := tcpClientConfig("tcp", portName, "")
+
+		f.RunProcessesWithBinaries(
+			compatCtx.CurrentFRPSPath,
+			compatCtx.BaselineFRPCPath,
+			serverConf,
+			[]string{clientConf},
+		)
+
+		framework.NewRequestExpect(f).PortName(portName).Ensure()
+		expectSingleClientWireProtocol(webPort, "v1")
+	})
+
+	ginkgo.It("baseline frps accepts current frpc defaulting to v1", func() {
+		portName := port.GenName("CompatBaselineFRPS")
+		clientConf := tcpClientConfig("tcp", portName, "")
+
+		f.RunProcessesWithBinaries(
+			compatCtx.BaselineFRPSPath,
+			compatCtx.CurrentFRPCPath,
+			consts.DefaultServerConfig,
+			[]string{clientConf},
+		)
+
+		framework.NewRequestExpect(f).PortName(portName).Ensure()
+	})
+
+	ginkgo.It("baseline frps rejects current frpc forced to v2", func() {
+		portName := port.GenName("CompatBaselineFRPSForcedV2")
+		clientConf := tcpClientConfig("tcp", portName, `
+transport.wireProtocol = "v2"
+`)
+
+		_, clientProcesses := f.RunProcessesWithBinaries(
+			compatCtx.BaselineFRPSPath,
+			compatCtx.CurrentFRPCPath,
+			consts.DefaultServerConfig,
+			[]string{clientConf},
+		)
+		expectProcessExit(clientProcesses[0], 5*time.Second)
+		framework.NewRequestExpect(f).PortName(portName).ExpectError(true).Ensure()
+	})
+})
+
+func tcpClientConfig(proxyName string, remotePortName string, extra string) string {
+	return fmt.Sprintf(`
+serverAddr = "127.0.0.1"
+serverPort = {{ .%s }}
+loginFailExit = true
+log.level = "trace"
+%s
+
+[[proxies]]
+name = "%s"
+type = "tcp"
+localPort = {{ .%s }}
+remotePort = {{ .%s }}
+`, consts.PortServerName, extra, proxyName, framework.TCPEchoServerPort, remotePortName)
+}
+
+func expectProcessExit(p *process.Process, timeout time.Duration) {
+	select {
+	case <-p.Done():
+	case <-time.After(timeout):
+		framework.Failf("process did not exit within %s; output:\n%s", timeout, p.Output())
+	}
+}
+
+type wireClientInfo struct {
+	ClientID     string `json:"clientID"`
+	WireProtocol string `json:"wireProtocol"`
+}
+
+func expectSingleClientWireProtocol(webPort int, wireProtocol string) {
+	clients := getWireClientInfos(webPort)
+	framework.ExpectEqual(len(clients), 1)
+	framework.ExpectEqual(clients[0].WireProtocol, wireProtocol)
+}
+
+func expectClientWireProtocol(webPort int, clientID string, wireProtocol string) {
+	for _, client := range getWireClientInfos(webPort) {
+		if client.ClientID == clientID {
+			framework.ExpectEqual(client.WireProtocol, wireProtocol)
+			return
+		}
+	}
+	framework.Failf("client %q not found in /api/clients response", clientID)
+}
+
+func getWireClientInfos(webPort int) []wireClientInfo {
+	client := http.Client{Timeout: consts.DefaultTimeout}
+	resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/api/clients", webPort))
+	framework.ExpectNoError(err)
+	defer resp.Body.Close()
+	framework.ExpectEqual(resp.StatusCode, http.StatusOK)
+
+	content, err := io.ReadAll(resp.Body)
+	framework.ExpectNoError(err)
+
+	var clients []wireClientInfo
+	framework.ExpectNoError(json.Unmarshal(content, &clients))
+	return clients
+}

test/e2e/framework/process.go

@@ -17,6 +17,16 @@ import (
 
 // RunProcesses starts one frps and zero or more frpc processes from templates.
 func (f *Framework) RunProcesses(serverTemplate string, clientTemplates []string) (*process.Process, []*process.Process) {
+	return f.RunProcessesWithBinaries(TestContext.FRPServerPath, TestContext.FRPClientPath, serverTemplate, clientTemplates)
+}
+
+// RunProcessesWithBinaries starts one frps and zero or more frpc processes with explicit binary paths.
+func (f *Framework) RunProcessesWithBinaries(
+	serverBinaryPath string,
+	clientBinaryPath string,
+	serverTemplate string,
+	clientTemplates []string,
+) (*process.Process, []*process.Process) {
 	templates := append([]string{serverTemplate}, clientTemplates...)
 	outs, ports, err := f.RenderTemplates(templates)
 	ExpectNoError(err)
@@ -32,7 +42,7 @@ func (f *Framework) RunProcesses(serverTemplate string, clientTemplates []string
 		flog.Debugf("[%s] %s", serverPath, outs[0])
 	}
 
-	serverProcess := process.NewWithEnvs(TestContext.FRPServerPath, []string{"-c", serverPath}, f.osEnvs)
+	serverProcess := process.NewWithEnvs(serverBinaryPath, []string{"-c", serverPath}, f.osEnvs)
 	f.serverConfPaths = append(f.serverConfPaths, serverPath)
 	f.serverProcesses = append(f.serverProcesses, serverProcess)
 	err = serverProcess.Start()
@@ -55,7 +65,7 @@ func (f *Framework) RunProcesses(serverTemplate string, clientTemplates []string
 			flog.Debugf("[%s] %s", path, outs[1+i])
 		}
 
-		p := process.NewWithEnvs(TestContext.FRPClientPath, []string{"-c", path}, f.osEnvs)
+		p := process.NewWithEnvs(clientBinaryPath, []string{"-c", path}, f.osEnvs)
 		f.clientConfPaths = append(f.clientConfPaths, path)
 		f.clientProcesses = append(f.clientProcesses, p)
 		clientProcesses = append(clientProcesses, p)