mirror of
https://github.com/fatedier/frp.git
synced 2026-10-05 16:55:58 +08:00
599 lines
18 KiB
Go
599 lines
18 KiB
Go
package features
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/onsi/ginkgo/v2"
|
|
"github.com/onsi/gomega"
|
|
|
|
"github.com/fatedier/frp/pkg/sdk/client"
|
|
"github.com/fatedier/frp/pkg/transport"
|
|
"github.com/fatedier/frp/test/e2e/framework"
|
|
"github.com/fatedier/frp/test/e2e/framework/consts"
|
|
"github.com/fatedier/frp/test/e2e/mock/server/httpserver"
|
|
"github.com/fatedier/frp/test/e2e/mock/server/streamserver"
|
|
"github.com/fatedier/frp/test/e2e/pkg/request"
|
|
)
|
|
|
|
func waitForProxyStatus(proxyClient *client.Client, proxyName, want string) error {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
ticker := time.NewTicker(25 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
|
|
var lastStatus string
|
|
var lastErr error
|
|
for {
|
|
status, err := proxyClient.GetProxyStatus(ctx, proxyName)
|
|
if err == nil {
|
|
lastErr = nil
|
|
lastStatus = status.Status
|
|
if status.Status == want {
|
|
return nil
|
|
}
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
if lastErr != nil {
|
|
return fmt.Errorf("timeout waiting for proxy %q status %q: last error: %w", proxyName, want, lastErr)
|
|
}
|
|
return fmt.Errorf("timeout waiting for proxy %q status %q: last status %q", proxyName, want, lastStatus)
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func waitForSignal(signal <-chan struct{}) error {
|
|
timer := time.NewTimer(5 * time.Second)
|
|
defer timer.Stop()
|
|
|
|
select {
|
|
case <-signal:
|
|
return nil
|
|
case <-timer.C:
|
|
return fmt.Errorf("timeout waiting for backend health-check signal")
|
|
}
|
|
}
|
|
|
|
// Keep this longer than client/proxy.statusCheckInterval (3s). The wrapper's
|
|
// health notification is deliberately non-blocking, so the E2E assertion must
|
|
// also cover the fallback poll plus scheduling margin while recovery is gated.
|
|
const proxyFallbackObservationWindow = 4 * time.Second
|
|
|
|
func waitForServerProxyStatus(port int, proxyName, want string, timeout time.Duration) error {
|
|
return waitForLifecycleCondition(timeout, func() error {
|
|
body, err := getLifecycleEndpoint(port, "/api/proxies/"+url.PathEscape(proxyName))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var status struct {
|
|
Status string `json:"status"`
|
|
}
|
|
if err := json.Unmarshal([]byte(body), &status); err != nil {
|
|
return err
|
|
}
|
|
if status.Status != want {
|
|
return fmt.Errorf("frps proxy %q status %q, want %q", proxyName, status.Status, want)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
type httpHealthStage struct {
|
|
healthy bool
|
|
signal chan<- struct{}
|
|
release <-chan struct{}
|
|
}
|
|
|
|
var _ = ginkgo.Describe("[Feature: Group]", func() {
|
|
f := framework.NewDefaultFramework()
|
|
|
|
newHTTPServer := func(port int, respContent string) *httpserver.Server {
|
|
return httpserver.New(
|
|
httpserver.WithBindPort(port),
|
|
httpserver.WithHandler(framework.SpecifiedHTTPBodyHandler([]byte(respContent))),
|
|
)
|
|
}
|
|
|
|
validateFooBarResponse := func(resp *request.Response) bool {
|
|
if string(resp.Content) == "foo" || string(resp.Content) == "bar" {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
doFooBarHTTPRequest := func(vhostPort int, host string) []string {
|
|
results := []string{}
|
|
var wait sync.WaitGroup
|
|
var mu sync.Mutex
|
|
expectFn := func() {
|
|
framework.NewRequestExpect(f).Port(vhostPort).
|
|
RequestModify(func(r *request.Request) {
|
|
r.HTTP().HTTPHost(host)
|
|
}).
|
|
Ensure(validateFooBarResponse, func(resp *request.Response) bool {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
results = append(results, string(resp.Content))
|
|
return true
|
|
})
|
|
}
|
|
for range 10 {
|
|
wait.Go(func() {
|
|
expectFn()
|
|
})
|
|
}
|
|
|
|
wait.Wait()
|
|
return results
|
|
}
|
|
|
|
ginkgo.Describe("Load Balancing", func() {
|
|
ginkgo.It("TCP", func() {
|
|
serverConf := consts.DefaultServerConfig
|
|
clientConf := consts.DefaultClientConfig
|
|
|
|
fooPort := f.AllocPort()
|
|
fooServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(fooPort), streamserver.WithRespContent([]byte("foo")))
|
|
f.RunServer("", fooServer)
|
|
|
|
barPort := f.AllocPort()
|
|
barServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(barPort), streamserver.WithRespContent([]byte("bar")))
|
|
f.RunServer("", barServer)
|
|
|
|
remotePort := f.AllocPort()
|
|
clientConf += fmt.Sprintf(`
|
|
[[proxies]]
|
|
name = "foo"
|
|
type = "tcp"
|
|
localPort = %d
|
|
remotePort = %d
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
|
|
[[proxies]]
|
|
name = "bar"
|
|
type = "tcp"
|
|
localPort = %d
|
|
remotePort = %d
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
`, fooPort, remotePort, barPort, remotePort)
|
|
|
|
f.RunProcesses(serverConf, []string{clientConf})
|
|
|
|
fooCount := 0
|
|
barCount := 0
|
|
for i := range 10 {
|
|
framework.NewRequestExpect(f).Explain("times " + strconv.Itoa(i)).Port(remotePort).Ensure(func(resp *request.Response) bool {
|
|
switch string(resp.Content) {
|
|
case "foo":
|
|
fooCount++
|
|
case "bar":
|
|
barCount++
|
|
default:
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
framework.ExpectTrue(fooCount > 1 && barCount > 1, "fooCount: %d, barCount: %d", fooCount, barCount)
|
|
})
|
|
|
|
ginkgo.It("HTTPS", func() {
|
|
vhostHTTPSPort := f.AllocPort()
|
|
serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
|
|
vhostHTTPSPort = %d
|
|
`, vhostHTTPSPort)
|
|
clientConf := consts.DefaultClientConfig
|
|
|
|
tlsConfig, err := transport.NewServerTLSConfig("", "", "")
|
|
framework.ExpectNoError(err)
|
|
|
|
fooPort := f.AllocPort()
|
|
fooServer := httpserver.New(
|
|
httpserver.WithBindPort(fooPort),
|
|
httpserver.WithHandler(framework.SpecifiedHTTPBodyHandler([]byte("foo"))),
|
|
httpserver.WithTLSConfig(tlsConfig),
|
|
)
|
|
f.RunServer("", fooServer)
|
|
|
|
barPort := f.AllocPort()
|
|
barServer := httpserver.New(
|
|
httpserver.WithBindPort(barPort),
|
|
httpserver.WithHandler(framework.SpecifiedHTTPBodyHandler([]byte("bar"))),
|
|
httpserver.WithTLSConfig(tlsConfig),
|
|
)
|
|
f.RunServer("", barServer)
|
|
|
|
clientConf += fmt.Sprintf(`
|
|
[[proxies]]
|
|
name = "foo"
|
|
type = "https"
|
|
localPort = %d
|
|
customDomains = ["example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
|
|
[[proxies]]
|
|
name = "bar"
|
|
type = "https"
|
|
localPort = %d
|
|
customDomains = ["example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
`, fooPort, barPort)
|
|
|
|
f.RunProcesses(serverConf, []string{clientConf})
|
|
|
|
fooCount := 0
|
|
barCount := 0
|
|
for i := range 10 {
|
|
framework.NewRequestExpect(f).
|
|
Explain("times " + strconv.Itoa(i)).
|
|
Port(vhostHTTPSPort).
|
|
RequestModify(func(r *request.Request) {
|
|
r.HTTPS().HTTPHost("example.com").TLSConfig(&tls.Config{
|
|
ServerName: "example.com",
|
|
InsecureSkipVerify: true,
|
|
})
|
|
}).
|
|
Ensure(func(resp *request.Response) bool {
|
|
switch string(resp.Content) {
|
|
case "foo":
|
|
fooCount++
|
|
case "bar":
|
|
barCount++
|
|
default:
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
framework.ExpectTrue(fooCount > 1 && barCount > 1, "fooCount: %d, barCount: %d", fooCount, barCount)
|
|
})
|
|
|
|
ginkgo.It("TCPMux httpconnect", func() {
|
|
vhostPort := f.AllocPort()
|
|
serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
|
|
tcpmuxHTTPConnectPort = %d
|
|
`, vhostPort)
|
|
clientConf := consts.DefaultClientConfig
|
|
|
|
fooPort := f.AllocPort()
|
|
fooServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(fooPort), streamserver.WithRespContent([]byte("foo")))
|
|
f.RunServer("", fooServer)
|
|
|
|
barPort := f.AllocPort()
|
|
barServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(barPort), streamserver.WithRespContent([]byte("bar")))
|
|
f.RunServer("", barServer)
|
|
|
|
clientConf += fmt.Sprintf(`
|
|
[[proxies]]
|
|
name = "foo"
|
|
type = "tcpmux"
|
|
multiplexer = "httpconnect"
|
|
localPort = %d
|
|
customDomains = ["tcpmux-group.example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
|
|
[[proxies]]
|
|
name = "bar"
|
|
type = "tcpmux"
|
|
multiplexer = "httpconnect"
|
|
localPort = %d
|
|
customDomains = ["tcpmux-group.example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
`, fooPort, barPort)
|
|
|
|
f.RunProcesses(serverConf, []string{clientConf})
|
|
|
|
proxyURL := fmt.Sprintf("http://127.0.0.1:%d", vhostPort)
|
|
fooCount := 0
|
|
barCount := 0
|
|
for i := range 10 {
|
|
framework.NewRequestExpect(f).
|
|
Explain("times " + strconv.Itoa(i)).
|
|
RequestModify(func(r *request.Request) {
|
|
r.Addr("tcpmux-group.example.com").Proxy(proxyURL)
|
|
}).
|
|
Ensure(func(resp *request.Response) bool {
|
|
switch string(resp.Content) {
|
|
case "foo":
|
|
fooCount++
|
|
case "bar":
|
|
barCount++
|
|
default:
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
|
|
framework.ExpectTrue(fooCount > 1 && barCount > 1, "fooCount: %d, barCount: %d", fooCount, barCount)
|
|
})
|
|
})
|
|
|
|
ginkgo.Describe("Health Check", func() {
|
|
ginkgo.It("TCP", func() {
|
|
dashboardPort := f.AllocPort()
|
|
serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
|
|
webServer.port = %d
|
|
`, dashboardPort)
|
|
clientConf := consts.DefaultClientConfig
|
|
|
|
fooPort := f.AllocPort()
|
|
fooServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(fooPort), streamserver.WithRespContent([]byte("foo")))
|
|
f.RunServer("", fooServer)
|
|
|
|
barPort := f.AllocPort()
|
|
newBarServer := func() *streamserver.Server {
|
|
return streamserver.New(streamserver.TCP, streamserver.WithBindPort(barPort), streamserver.WithRespContent([]byte("bar")))
|
|
}
|
|
barServer := newBarServer()
|
|
f.RunServer("", barServer)
|
|
|
|
remotePort := f.AllocPort()
|
|
adminPort := f.AllocPort()
|
|
clientConf += fmt.Sprintf(`
|
|
webServer.port = %d
|
|
|
|
[[proxies]]
|
|
name = "foo"
|
|
type = "tcp"
|
|
localPort = %d
|
|
remotePort = %d
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
healthCheck.type = "tcp"
|
|
healthCheck.intervalSeconds = 1
|
|
|
|
[[proxies]]
|
|
name = "bar"
|
|
type = "tcp"
|
|
localPort = %d
|
|
remotePort = %d
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
healthCheck.type = "tcp"
|
|
healthCheck.intervalSeconds = 1
|
|
healthCheck.maxFailed = 3
|
|
`, adminPort, fooPort, remotePort, barPort, remotePort)
|
|
|
|
f.RunProcesses(serverConf, []string{clientConf})
|
|
proxyClient := f.APIClientForFrpc(adminPort)
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "foo", "running"))
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running"))
|
|
|
|
// Both requests traverse frps, the load-balancing group, frpc, and the
|
|
// corresponding local backend.
|
|
results := []string{}
|
|
for range 10 {
|
|
framework.NewRequestExpect(f).Port(remotePort).Ensure(validateFooBarResponse, func(resp *request.Response) bool {
|
|
results = append(results, string(resp.Content))
|
|
return true
|
|
})
|
|
}
|
|
framework.ExpectContainElements(results, []string{"foo", "bar"})
|
|
|
|
// frps removes the group listener before deleting the proxy from
|
|
// its manager. Its offline status is the removal barrier; frpc's
|
|
// local check-failed status alone does not acknowledge that work.
|
|
for range 2 {
|
|
framework.ExpectNoError(barServer.Close())
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "check failed"))
|
|
framework.ExpectNoError(waitForServerProxyStatus(dashboardPort, "bar", "offline", 5*time.Second))
|
|
for range 10 {
|
|
framework.NewRequestExpect(f).Port(remotePort).ExpectResp([]byte("foo")).Ensure()
|
|
}
|
|
|
|
barServer = newBarServer()
|
|
f.RunServer("", barServer)
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running"))
|
|
results = []string{}
|
|
for range 10 {
|
|
framework.NewRequestExpect(f).Port(remotePort).Ensure(validateFooBarResponse, func(resp *request.Response) bool {
|
|
results = append(results, string(resp.Content))
|
|
return true
|
|
})
|
|
}
|
|
framework.ExpectContainElements(results, []string{"foo", "bar"})
|
|
}
|
|
})
|
|
|
|
ginkgo.It("HTTP", func() {
|
|
vhostPort := f.AllocPort()
|
|
dashboardPort := f.AllocPort()
|
|
serverConf := consts.DefaultServerConfig + fmt.Sprintf(`
|
|
vhostHTTPPort = %d
|
|
webServer.port = %d
|
|
`, vhostPort, dashboardPort)
|
|
clientConf := consts.DefaultClientConfig
|
|
|
|
fooPort := f.AllocPort()
|
|
fooServer := newHTTPServer(fooPort, "foo")
|
|
f.RunServer("", fooServer)
|
|
|
|
barPort := f.AllocPort()
|
|
var stageMu sync.RWMutex
|
|
stage := &httpHealthStage{healthy: true, signal: make(chan struct{}, 1)}
|
|
barServer := httpserver.New(
|
|
httpserver.WithBindPort(barPort),
|
|
httpserver.WithHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
stageMu.RLock()
|
|
currentStage := stage
|
|
stageMu.RUnlock()
|
|
|
|
if r.URL.Path == "/healthz" {
|
|
if !currentStage.healthy {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}
|
|
select {
|
|
case currentStage.signal <- struct{}{}:
|
|
default:
|
|
}
|
|
if currentStage.release != nil {
|
|
select {
|
|
case <-currentStage.release:
|
|
case <-r.Context().Done():
|
|
return
|
|
}
|
|
}
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte("bar"))
|
|
})),
|
|
)
|
|
f.RunServer("", barServer)
|
|
|
|
adminPort := f.AllocPort()
|
|
clientConf += fmt.Sprintf(`
|
|
webServer.port = %d
|
|
|
|
[[proxies]]
|
|
name = "foo"
|
|
type = "http"
|
|
localPort = %d
|
|
customDomains = ["example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
healthCheck.type = "http"
|
|
healthCheck.intervalSeconds = 1
|
|
healthCheck.path = "/healthz"
|
|
|
|
[[proxies]]
|
|
name = "bar"
|
|
type = "http"
|
|
localPort = %d
|
|
customDomains = ["example.com"]
|
|
loadBalancer.group = "test"
|
|
loadBalancer.groupKey = "123"
|
|
healthCheck.type = "http"
|
|
healthCheck.intervalSeconds = 1
|
|
healthCheck.maxFailed = 3
|
|
healthCheck.timeoutSeconds = 10
|
|
healthCheck.path = "/healthz"
|
|
`, adminPort, fooPort, barPort)
|
|
|
|
f.RunProcesses(serverConf, []string{clientConf})
|
|
proxyClient := f.APIClientForFrpc(adminPort)
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "foo", "running"))
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running"))
|
|
|
|
// Ordinary requests traverse the real HTTP proxy path.
|
|
var contents []string
|
|
framework.NewRequestExpect(f).Port(vhostPort).
|
|
RequestModify(func(r *request.Request) {
|
|
r.HTTP().HTTPHost("example.com")
|
|
}).
|
|
Ensure(func(resp *request.Response) bool {
|
|
contents = append(contents, string(resp.Content))
|
|
return true
|
|
})
|
|
framework.NewRequestExpect(f).Port(vhostPort).
|
|
RequestModify(func(r *request.Request) {
|
|
r.HTTP().HTTPHost("example.com")
|
|
}).
|
|
Ensure(func(resp *request.Response) bool {
|
|
contents = append(contents, string(resp.Content))
|
|
return true
|
|
})
|
|
framework.ExpectContainElements(contents, []string{"foo", "bar"})
|
|
|
|
// Every failed health response is gated by the test. This gives each
|
|
// stage a request-level barrier and avoids cumulative process logs.
|
|
runFailureRecovery := func() {
|
|
failureSignals := make(chan struct{}, 1)
|
|
release := make(chan struct{}, 1)
|
|
stageMu.Lock()
|
|
stage = &httpHealthStage{healthy: false, signal: failureSignals, release: release}
|
|
stageMu.Unlock()
|
|
|
|
framework.ExpectNoError(waitForSignal(failureSignals))
|
|
release <- struct{}{}
|
|
framework.ExpectNoError(waitForSignal(failureSignals))
|
|
|
|
recoverySignals := make(chan struct{}, 1)
|
|
recoveryRelease := make(chan struct{})
|
|
stageMu.Lock()
|
|
stage = &httpHealthStage{healthy: true, signal: recoverySignals, release: recoveryRelease}
|
|
stageMu.Unlock()
|
|
release <- struct{}{}
|
|
|
|
// The next request can only start after the worker consumed the
|
|
// second failed response. Hold this success response until the
|
|
// assertion finishes, so premature removal cannot be hidden by
|
|
// recovery. Consistently also gives the proxy worker time to act
|
|
// on an erroneous failure notification without retrying it away.
|
|
framework.ExpectNoError(waitForSignal(recoverySignals))
|
|
gomega.Consistently(func() string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
status, err := proxyClient.GetProxyStatus(ctx, "bar")
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
return status.Status
|
|
}, proxyFallbackObservationWindow, 25*time.Millisecond).Should(gomega.Equal("running"))
|
|
close(recoveryRelease)
|
|
}
|
|
runFailureRecovery()
|
|
results := doFooBarHTTPRequest(vhostPort, "example.com")
|
|
framework.ExpectContainElements(results, []string{"foo", "bar"})
|
|
|
|
// Repeat the two-failure window to verify that recovery cleared the
|
|
// counter before another failure sequence began.
|
|
runFailureRecovery()
|
|
results = doFooBarHTTPRequest(vhostPort, "example.com")
|
|
framework.ExpectContainElements(results, []string{"foo", "bar"})
|
|
|
|
// Three failed responses reach MaxFailed and remove bar from the
|
|
// group; a healthy response then re-registers it.
|
|
failureSignals := make(chan struct{}, 1)
|
|
release := make(chan struct{}, 1)
|
|
stageMu.Lock()
|
|
stage = &httpHealthStage{healthy: false, signal: failureSignals, release: release}
|
|
stageMu.Unlock()
|
|
for range 3 {
|
|
framework.ExpectNoError(waitForSignal(failureSignals))
|
|
release <- struct{}{}
|
|
}
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "check failed"))
|
|
// HTTPProxy.Close unregisters the group route before frps reports
|
|
// offline. Only then assert that fresh data-plane requests use foo.
|
|
framework.ExpectNoError(waitForServerProxyStatus(dashboardPort, "bar", "offline", 5*time.Second))
|
|
results = doFooBarHTTPRequest(vhostPort, "example.com")
|
|
framework.ExpectContainElements(results, []string{"foo"})
|
|
framework.ExpectNotContainElements(results, []string{"bar"})
|
|
|
|
recoverySignals := make(chan struct{}, 1)
|
|
stageMu.Lock()
|
|
stage = &httpHealthStage{healthy: true, signal: recoverySignals}
|
|
stageMu.Unlock()
|
|
// Release any additional failed request already waiting while the
|
|
// removal and data-plane assertions were in progress.
|
|
close(release)
|
|
framework.ExpectNoError(waitForSignal(recoverySignals))
|
|
framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running"))
|
|
results = doFooBarHTTPRequest(vhostPort, "example.com")
|
|
framework.ExpectContainElements(results, []string{"foo", "bar"})
|
|
})
|
|
})
|
|
})
|