From 25a4ba306afbaca592f757f463e48a658b045382 Mon Sep 17 00:00:00 2001 From: fatedier Date: Mon, 17 Aug 2026 00:48:09 +0800 Subject: [PATCH 01/23] ci: pin golangci-lint to v2.12.2 (#5496) --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 36e8d50b..79eaa6df 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -28,4 +28,4 @@ jobs: uses: golangci/golangci-lint-action@v9 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v2.11 + version: v2.12.2 From 758f07d59e155e1eb044d3ca6bb3151fdab737dc Mon Sep 17 00:00:00 2001 From: fatedier Date: Tue, 18 Aug 2026 00:24:11 +0800 Subject: [PATCH 02/23] deps: migrate to fatedier/yamux v0.2.0 (#5498) --- client/connector.go | 2 +- client/proxy/xtcp.go | 2 +- client/visitor/xtcp.go | 2 +- go.mod | 5 +---- go.sum | 4 ++-- server/service.go | 2 +- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/client/connector.go b/client/connector.go index 34b24f0d..41332378 100644 --- a/client/connector.go +++ b/client/connector.go @@ -24,7 +24,7 @@ import ( "time" libnet "github.com/fatedier/golib/net" - fmux "github.com/hashicorp/yamux" + fmux "github.com/fatedier/yamux" quic "github.com/quic-go/quic-go" "github.com/samber/lo" diff --git a/client/proxy/xtcp.go b/client/proxy/xtcp.go index aef66780..fcd1487f 100644 --- a/client/proxy/xtcp.go +++ b/client/proxy/xtcp.go @@ -22,7 +22,7 @@ import ( "reflect" "time" - fmux "github.com/hashicorp/yamux" + fmux "github.com/fatedier/yamux" "github.com/quic-go/quic-go" v1 "github.com/fatedier/frp/pkg/config/v1" diff --git a/client/visitor/xtcp.go b/client/visitor/xtcp.go index e7a60895..114f47be 100644 --- a/client/visitor/xtcp.go +++ b/client/visitor/xtcp.go @@ -25,7 +25,7 @@ import ( "time" libio "github.com/fatedier/golib/io" - fmux "github.com/hashicorp/yamux" + fmux "github.com/fatedier/yamux" quic "github.com/quic-go/quic-go" "golang.org/x/time/rate" diff --git a/go.mod b/go.mod index 9e02c333..87d47021 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/coreos/go-oidc/v3 v3.18.0 github.com/fatedier/golib v0.8.2 + github.com/fatedier/yamux v0.2.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.0 - github.com/hashicorp/yamux v0.1.1 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.36.3 github.com/pelletier/go-toml/v2 v2.2.0 @@ -73,6 +73,3 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -// TODO(fatedier): Temporary use the modified version, update to the official version after merging into the official repository. -replace github.com/hashicorp/yamux => github.com/fatedier/yamux v0.0.0-20250825093530-d0154be01cd6 diff --git a/go.sum b/go.sum index 30641d78..ce3512c9 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatedier/golib v0.8.2 h1:02n2Dg7KJ7rR7p7n4/6hBUjaLQf2J7EiHYZQsgGTvww= github.com/fatedier/golib v0.8.2/go.mod h1:ArUGvPg2cOw/py2RAuBt46nNZH2VQ5Z70p109MAZpJw= -github.com/fatedier/yamux v0.0.0-20250825093530-d0154be01cd6 h1:u92UUy6FURPmNsMBUuongRWC0rBqN6gd01Dzu+D21NE= -github.com/fatedier/yamux v0.0.0-20250825093530-d0154be01cd6/go.mod h1:c5/tk6G0dSpXGzJN7Wk1OEie8grdSJAmeawId9Zvd34= +github.com/fatedier/yamux v0.2.0 h1:H+2A9iBVh7aJlEOc1Ws1FXWOaecBf2nRv9zpFMPUWg8= +github.com/fatedier/yamux v0.2.0/go.mod h1:d4FtRDrC9sHvRpiDL6J5EnfjLhqzZZplHe5yToSn2Ac= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= diff --git a/server/service.go b/server/service.go index 82883bb2..6ea24cc4 100644 --- a/server/service.go +++ b/server/service.go @@ -29,7 +29,7 @@ import ( "github.com/fatedier/golib/crypto" "github.com/fatedier/golib/net/mux" - fmux "github.com/hashicorp/yamux" + fmux "github.com/fatedier/yamux" quic "github.com/quic-go/quic-go" "github.com/samber/lo" From 40bb45d1928ee81a260977751903dcd2f5b00577 Mon Sep 17 00:00:00 2001 From: flying <54491871+flyingcoding@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:33:45 +0800 Subject: [PATCH 03/23] vnet: fix route cleanup and reconnect backoff (#5492) --- pkg/plugin/visitor/virtual_net.go | 29 ++++++++++--- pkg/plugin/visitor/virtual_net_test.go | 47 +++++++++++++++++++++ pkg/vnet/controller.go | 13 ++++-- pkg/vnet/controller_test.go | 57 ++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 pkg/plugin/visitor/virtual_net_test.go create mode 100644 pkg/vnet/controller_test.go diff --git a/pkg/plugin/visitor/virtual_net.go b/pkg/plugin/visitor/virtual_net.go index fa8c700e..87a67744 100644 --- a/pkg/plugin/visitor/virtual_net.go +++ b/pkg/plugin/visitor/virtual_net.go @@ -48,6 +48,11 @@ type VirtualNetPlugin struct { cancel context.CancelFunc } +const ( + virtualNetReconnectBaseDelay = 60 * time.Second + virtualNetReconnectMaxDelay = 300 * time.Second +) + func NewVirtualNetPlugin(pluginCtx PluginContext, options v1.VisitorPluginOptions) (Plugin, error) { opts := options.(*v1.VirtualNetVisitorPluginOptions) @@ -152,8 +157,7 @@ func (p *VirtualNetPlugin) run() { p.pluginCtx.Name, p.consecutiveErrors, closeErr) // Exponential backoff: 60s, 120s, 240s, 300s (capped) - baseDelay := 60 * time.Second - reconnectDelay = min(baseDelay*time.Duration(1< 0 { @@ -182,6 +186,18 @@ func (p *VirtualNetPlugin) run() { } } +// virtualNetReconnectDelay returns a bounded reconnect delay without allowing +// the exponential shift to overflow for large consecutive error counts. +func virtualNetReconnectDelay(consecutiveErrors int) time.Duration { + if consecutiveErrors <= 1 { + return virtualNetReconnectBaseDelay + } + if consecutiveErrors >= 4 { + return virtualNetReconnectMaxDelay + } + return virtualNetReconnectBaseDelay * time.Duration(1< Date: Fri, 28 Aug 2026 10:56:05 +0800 Subject: [PATCH 04/23] vnet: serialize VirtualNet route lifecycle (#5512) * vnet: serialize virtual net route lifecycle * test: remove fixed virtual net helper parameters --- pkg/plugin/visitor/virtual_net.go | 98 ++++++++---- pkg/plugin/visitor/virtual_net_test.go | 198 +++++++++++++++++++++++++ pkg/vnet/controller_test.go | 7 + 3 files changed, 273 insertions(+), 30 deletions(-) diff --git a/pkg/plugin/visitor/virtual_net.go b/pkg/plugin/visitor/virtual_net.go index 87a67744..f95856c2 100644 --- a/pkg/plugin/visitor/virtual_net.go +++ b/pkg/plugin/visitor/virtual_net.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "io" "net" "sync" "time" @@ -33,10 +34,16 @@ func init() { Register(v1.VisitorPluginVirtualNet, NewVirtualNetPlugin) } +type clientRouteController interface { + RegisterClientRoute(context.Context, string, []net.IPNet, io.ReadWriteCloser) + UnregisterClientRoute(string, io.Writer) bool +} + type VirtualNetPlugin struct { pluginCtx PluginContext - routes []net.IPNet + routeController clientRouteController + routes []net.IPNet mu sync.Mutex controllerConn net.Conn @@ -60,6 +67,9 @@ func NewVirtualNetPlugin(pluginCtx PluginContext, options v1.VisitorPluginOption pluginCtx: pluginCtx, routes: make([]net.IPNet, 0), } + if pluginCtx.VnetController != nil { + p.routeController = pluginCtx.VnetController + } p.ctx, p.cancel = context.WithCancel(pluginCtx.Ctx) @@ -90,7 +100,7 @@ func (p *VirtualNetPlugin) Name() string { func (p *VirtualNetPlugin) Start() { xl := xlog.FromContextSafe(p.pluginCtx.Ctx) - if p.pluginCtx.VnetController == nil { + if p.routeController == nil { return } @@ -116,16 +126,17 @@ func (p *VirtualNetPlugin) run() { select { case <-p.ctx.Done(): xl.Infof("VirtualNetPlugin run loop for visitor [%s] stopping (context cancelled before pipe creation).", p.pluginCtx.Name) - p.cleanupControllerConn(xl) + p.cleanupCurrentControllerConn(xl) return default: } controllerConn, pluginConn := net.Pipe() - - p.mu.Lock() - p.controllerConn = controllerConn - p.mu.Unlock() + xl.Infof("attempting to register client route for visitor [%s]", p.pluginCtx.Name) + if !p.registerControllerConn(controllerConn, pluginConn) { + xl.Infof("VirtualNetPlugin run loop for visitor [%s] stopping (context cancelled before route registration).", p.pluginCtx.Name) + return + } // Wrap with CloseNotifyConn which supports both close notification and error recording var closeErr error @@ -134,8 +145,6 @@ func (p *VirtualNetPlugin) run() { close(currentCloseSignal) // Signal the run loop on close. }) - xl.Infof("attempting to register client route for visitor [%s]", p.pluginCtx.Name) - p.pluginCtx.VnetController.RegisterClientRoute(p.ctx, p.pluginCtx.Name, p.routes, controllerConn) xl.Infof("successfully registered client route for visitor [%s]. Starting connection handler with CloseNotifyConn.", p.pluginCtx.Name) // Pass the CloseNotifyConn to the visitor for handling. @@ -146,7 +155,7 @@ func (p *VirtualNetPlugin) run() { select { case <-p.ctx.Done(): xl.Infof("VirtualNetPlugin run loop stopping for visitor [%s] (context cancelled while waiting).", p.pluginCtx.Name) - p.cleanupControllerConn(xl) + p.cleanupControllerConn(xl, controllerConn) return case <-currentCloseSignal: // Determine reconnect delay based on error with exponential backoff @@ -171,7 +180,7 @@ func (p *VirtualNetPlugin) run() { } // The visitor closed the plugin side. Close the controller side. - p.cleanupControllerConn(xl) + p.cleanupControllerConn(xl, controllerConn) xl.Infof("waiting %v before attempting reconnection for visitor [%s]...", reconnectDelay, p.pluginCtx.Name) select { @@ -186,6 +195,23 @@ func (p *VirtualNetPlugin) run() { } } +// registerControllerConn publishes and registers controllerConn atomically with +// respect to Close. A canceled plugin cannot register a new route. +func (p *VirtualNetPlugin) registerControllerConn(controllerConn, pluginConn net.Conn) bool { + p.mu.Lock() + if p.ctx.Err() != nil || p.routeController == nil { + p.mu.Unlock() + _ = controllerConn.Close() + _ = pluginConn.Close() + return false + } + + p.controllerConn = controllerConn + p.routeController.RegisterClientRoute(p.ctx, p.pluginCtx.Name, p.routes, controllerConn) + p.mu.Unlock() + return true +} + // virtualNetReconnectDelay returns a bounded reconnect delay without allowing // the exponential shift to overflow for large consecutive error counts. func virtualNetReconnectDelay(consecutiveErrors int) time.Duration { @@ -198,16 +224,37 @@ func virtualNetReconnectDelay(consecutiveErrors int) time.Duration { return virtualNetReconnectBaseDelay * time.Duration(1< Date: Sat, 29 Aug 2026 14:41:43 +0800 Subject: [PATCH 05/23] docs: update release notes (#5521) --- README.md | 15 ++++++++------- README_zh.md | 15 ++++++++------- Release.md | 8 +------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c3bc2f60..cf0e0d4c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,14 @@ frp is an open source project with its ongoing development made possible entirel

Gold Sponsors

+

+ + +
+ The complete IDE crafted for professional Go developers + +

+

@@ -32,13 +40,6 @@ an API that records Zoom, Google Meet, Microsoft Teams, in-person meetings, and -

- - -
- The complete IDE crafted for professional Go developers - -

## What is frp? diff --git a/README_zh.md b/README_zh.md index 600cd4fd..246ebb7a 100644 --- a/README_zh.md +++ b/README_zh.md @@ -14,6 +14,14 @@ frp 是一个完全开源的项目，我们的开发工作完全依靠赞助者

Gold Sponsors

+

+ + +
+ The complete IDE crafted for professional Go developers + +

+

@@ -34,13 +42,6 @@ an API that records Zoom, Google Meet, Microsoft Teams, in-person meetings, and -

- - -
- The complete IDE crafted for professional Go developers - -

## 为什么使用 frp ？ diff --git a/Release.md b/Release.md index 33fc0aea..5199f328 100644 --- a/Release.md +++ b/Release.md @@ -1,9 +1,3 @@ -## Features - -* UDP packet payloads for ordinary UDP proxies and SUDP now use a dedicated binary codec when frpc and frps successfully negotiate the capability under wire protocol v2, using a more compact wire representation. Wire protocol v1 remains JSON; wire protocol v2 falls back to JSON `UDPPacket` when the peer does not support or did not negotiate the capability. - ## Fixes -* Fixed a server panic and remote denial of service caused by a client sending a negative `pool_count`. Negative values are now rejected before work-connection pool resources are allocated. -* Fixed `frpc verify` ignoring configured `featureGates`, which caused VirtualNet configurations to be rejected even when the feature was enabled. -* Fixed a case-insensitive validation bypass that allowed `customDomains` under the configured `subDomainHost` to be registered using mixed-case domain names. +* Fixed VirtualNet route lifecycle issues during reconnect and shutdown, including stale route cleanup, shutdown races, and reconnect backoff overflow. From 832df8dff66d1b0caa0d776c450418f982756df1 Mon Sep 17 00:00:00 2001 From: Rui JingAn <166888152+racequite@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:55:32 +0800 Subject: [PATCH 06/23] fix(health): reset failed count after successful check (#5502) Signed-off-by: racequite --- client/health/health.go | 1 + client/health/health_test.go | 65 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 client/health/health_test.go diff --git a/client/health/health.go b/client/health/health.go index d298e615..8cb1122a 100644 --- a/client/health/health.go +++ b/client/health/health.go @@ -119,6 +119,7 @@ func (monitor *Monitor) checkWorker() { if err == nil { xl.Tracef("do one health check success") + monitor.failedTimes = 0 if !monitor.statusOK && monitor.statusNormalFn != nil { xl.Infof("health check status change to success") monitor.statusOK = true diff --git a/client/health/health_test.go b/client/health/health_test.go new file mode 100644 index 00000000..f43edd04 --- /dev/null +++ b/client/health/health_test.go @@ -0,0 +1,65 @@ +// Copyright 2026 The frp Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package health + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + v1 "github.com/fatedier/frp/pkg/config/v1" +) + +func TestMonitorResetsFailedTimesAfterSuccess(t *testing.T) { + var checkCount atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + count := checkCount.Add(1) + if count == 1 || count == 2 || count == 4 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + var failedCount atomic.Int32 + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{ + Type: "http", + Path: "/health", + TimeoutSeconds: 1, + IntervalSeconds: 1, + MaxFailed: 3, + }, + strings.TrimPrefix(server.URL, "http://"), + func() {}, + func() { failedCount.Add(1) }, + ) + monitor.interval = 10 * time.Millisecond + monitor.Start() + defer monitor.Stop() + + require.Eventually(t, func() bool { + return checkCount.Load() >= 5 + }, time.Second, 10*time.Millisecond) + require.Equal(t, int32(0), failedCount.Load()) +} From c5508627c4c496a4e5518f5a207ffb86c92d884f Mon Sep 17 00:00:00 2001 From: fatedier Date: Thu, 10 Sep 2026 00:02:51 +0800 Subject: [PATCH 07/23] docs: add rapidproxy as gold sponsor (#5529) --- README.md | 12 ++++++++++++ README_zh.md | 12 ++++++++++++ doc/pic/sponsor_rapidproxy.png | Bin 0 -> 50558 bytes 3 files changed, 24 insertions(+) create mode 100644 doc/pic/sponsor_rapidproxy.png diff --git a/README.md b/README.md index cf0e0d4c..1c72a7bf 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,18 @@ frp is an open source project with its ongoing development made possible entirel

Gold Sponsors

+

+ + +
+ High-performance residential and ISP proxies for developers + +
+ 90M+ residential IPs worldwide. Rotating IPs, sticky sessions, and traffic that never expires. +
+ From $0.55/GB. Use RAPID10 for 10% off. Try it for free. +

+

diff --git a/README_zh.md b/README_zh.md index 246ebb7a..e4d28410 100644 --- a/README_zh.md +++ b/README_zh.md @@ -14,6 +14,18 @@ frp 是一个完全开源的项目，我们的开发工作完全依靠赞助者

Gold Sponsors

+

+ + +
+ High-performance residential and ISP proxies for developers + +
+ 90M+ residential IPs worldwide. Rotating IPs, sticky sessions, and traffic that never expires. +
+ From $0.55/GB. Use RAPID10 for 10% off. Try it for free. +

+

diff --git a/doc/pic/sponsor_rapidproxy.png b/doc/pic/sponsor_rapidproxy.png new file mode 100644 index 0000000000000000000000000000000000000000..11ec7c3a359e9194ac21e5c753f76f027a1708eb GIT binary patch literal 50558 zcmV*ZKvutrP)&_Ys^Og|6c^euVc*F)mo+iXjCXY7hHF=E91bWu$o>(FwS%I`AI+za!-)lhrkgR+4 zarJSp{xuJKSJkg-?{ohDOEZ~`xjO%I_O4x3yS{eqefHto?|C~3)xzZ0qeY0=u7y|| zmOo4^wTe)EII9nd51AZ5eJ<2IVMSox(o~hEtM)?C6JnmLGjRy%o^qz!M;{bb_Q^ce zE1t@j?!!VvpUe!0x}fTaD?4HEnF78uXiQuxt(aM6GFieuWL(XdQsjmX-#p_eQou66 zM^bpO(^jUuH}APTlqgh%e>OaEs6>dESTD#}r;M{7O;H{fXcajoA`xjA&O}op24pKt zs2N3@SV$(s70rsWFjR}fP~Zx);!s)5)|s&#`OHL9Y5I<&Dx4~Vp;KUdEC1e*I1E^q zrKlFAiXp}X{ABi@Tsf4m&wrC}mIs4c6i1L7mKaP#z<$J1!mS8p@*%77i37CUqFFN~IL!-vOBkV$ecIHTl79aEKR`xwHx{+;Co2On_f$4nGz zkKCwx=+T&uHp7P-{M!g}YC~K|2E8C~Q+-IFW?*30FC60mAr!daVgXo8kvW*LuEPl| zCiOb1i6*g3#MQ*;1G6P=G~I8J)-{FZYQBm;QG1O;%O=`NlF5ImEtM#Ok=Dv!VThQD zmEhvBB3nhhchP$1O1UFYDGXo*SeV2>u#5VXOxL}$f$N?WH$&F?2h^#F8H2)hWJ`0?lXcbblA58hi&Vkz2{ldrMK73D)Qw+Cru}Cz z0aF>hZ5L@c24tEI74V3&N;i(_RH7Q8tNH9#oQ1vy``)}2z0jcABcrPi)&7E?t(b<{ z!~BJvfn#3K5Xd!hBe#UJI_`A{DyOj?$2P%Y4Wru_L~tBD3psbv#E}vi?X+XvQi)bi zq*@z8sqdj3osKxxIULG^8QV~oK}*e4MZ1Rc1ZF7+8WI)>xPgy6cKvccUxA1rW7xS) zGqI>A2Qf~{7qoHW1hdM26z(+zQsj z_SF=ZIUwo_S3VaX5}HWpj!j%zcff`XGnc<2Y;`3!$!4seqJAQhVe-?kqxwJdp6)ui z*a6XLJ^HAc=_qg&m+FADLl@_UsRCptq%e4huE??_%2+k^>gJVK$fs$gXR>gNI zehgfh^$M4ey-|eN8uA@ zYWtWOjj8rLP#i4SAR!i_8a`L}QWHl>U@T|k>ZIWhOu{Biby}v+Z60Z!*tFfbVl~|y z6h2Sfww5wA$z!ELWEzFWLJ(mv+w4BH4kFlZ8yt;3A(>W{T2m%WG*uDHqR^X-D!W6- zXrgRx5>>lVbG)HLRl<>Abes0Qrges|wx5|jdN7za6bXo#)X>2MiDsGzvB

<@U2N zMouea4wJ7OlI>;FQ zI_3q#-(c^&A5KI=dm$M8;a=Q-2ubzRmR>V6S1WPk6B%oCU1d@=z)Y(~)8dIJduCXv zi?9$uR3v07LNX~j%T;-695Nkq)k>Qfb-TM^FqP!0bHypDKK#ARM7 zs-(KM+H*WQ5L~Tg4cbk7(f=eN5*-|8Ii(_MIC6%bR5}|?j^4oNba)MjrU=-2r;b@f zYCaidHVL44M^6kJR5yltbE@NEDTr5x`4Sz=GiVXidm`uS>OMm3&Q9#)6#IFg z1#CJk82pdBe|HEoI#EP=Yor2Ttfws7My*IfVX#9^!-@DCKJ&208VSKavl|WS0}*gy z$o~df#-B&s&^8KqaK71T(2>p=_*9-?jKzak0b*u6MhZg^z&x7_s}U6-470k$WL=?C zRwh8&h=3Y0-1@};vp03NxAR+kxQ|dJ@e2%QZ5X4CYsD|CU4qS zr213~4yqRILexlTijZAz?IimnOQ`aFe@DLO^DA zz8!G8k7nHFq{uLC^6*Y}^BdE|pq>JOA!^8+aMMGNQXp7~^UryGN#8ExILQ!58#?cQ zpjYu&gn~mLm7Hi{=EI!?Gu*?iC6W9bO0J7=5R~MRFgHDn^N($B1YVXx{F?~ z*kh3;jN2+LBQgs+ADkH3zwSPu0fEOb$$@Bp^JK z|H!A9yy{R24iBj*3L&cidd;t3kY@L}Oj5HOX@4 z?C!X$5zGlgp(q>?8;<|=2(nwnx%a6?`i7^V_yi=ufN}UWLfrhWDVi9dqds&Y9E=yZ zN+|bTWyH)@Hi)R0b#)*KBt=V=!00A{vRcwE{nXr@{U9Vu(a^DHB~VkKQZB2XE{jmk zi>ULqcvdB`_Qk9oL(TJ4T`LjS^#`^^nSvU!!9#@Vx@uEKNk^9_C>&s+AoeYj35#{q zHmOFTI;gMrb&}APBiLXZiv(T~5xk$#i!R#s4n_Bd)F76L1f60L7_-jJTOgrKd-ztF za_!eFy7i%nD2{YBi=AckM-n8Ev$ih8o9aV$xY7Kmpy3Wp7#1Otjxzdan#iW%(Xqw# z88m;IR>2u)uo++n3wRazK}YDuU3r0v1lU7k)O!5wddZ$|K!=Z3Te=Xs1+7jmlttGP z97FB`HM-X&+>2gn9!K4S2ABtr++dI^NlO3`P|pT_y!Qw`6%$NAhJlfD`8M7zf1D9E zTp?x-Y|EpqMpV(Cftj}q4DQs#?nIn5>R*bj5M4Cub;3lpJtg&{C`_V0s5aK!C?dqF zBTk|q3`DdVBdN|xswqSFs~qOqf!SUu7jE>#Bt~@n-B2dA(#Pf%M5~+m#;lT>P-AHQ zjqsI2iF?%@=lM^@2NgScP|gh~wgi;QMIj^m6m<|TOQJNnuDF*3fXKh3@jBFIBFe@q z6*P)U;F9J$YmgBjmpx)pu1dwSeY8joMqau%N|Z`A;!^wvAjh3VrNek*omCvb*wtvx zIOMpH-&90hVHHrUNG907w4n72&zaCq6o8pnNr9>M2k)g3{%vtaAIsiOwFCw&~FL15=wX;h|}U~6EB zF;Qn@mRN#semR4z3$1|(1g&6m4waa)e#j&=iCI^v^|f1OPb@LPLXGP?&=9k|iNVgB zhP7oD4(`l`vnOrw9vTR6mAOO(a67E~dut*Bq(zhnXG)$QZ{ZS0V7Q^A%|>LSa(12@5DQtZUPUdlaIMe8m37@ysTqS1oA1DA=SD z^I%?ts^j_xU7+K|Bh^JEc+jSYXp&lD(;9z+PqCX(=6Xek1PtiRA_->jBj#-Sj@cpD zwRCC;;r{mGXw(J{r->aPKKJoxjgMC;(5D*L)(mIzuO6Dz*DbV;hW+X!*|t`h+cQ5< z;ngVm1>x)Us6o6%n8+&V@Id2m~ z`xmoiKz+#2OO?=M>r3xbmna{_*77PFvMRMTxxUgfb_{}ENctTbIAi-^a^RB?mDDJs zSy2eXhSA4ndpRx$E@HdTxzWcoZdV8Tzy+BY=0tf9^peX-sztq@!T&{=I^EH~bHxg( z_zckiwT!nCTtQZq3evORu2P+mWbJl9@u<1W0gm{?`D8)#f9-02E)4%fff0p;%MWeK z>cD3gCaT%Yve1JzFvAYK7_e3vT3GyHAzpp#M8)+J?}J<~L~OOrRxiC8iaM+EQ&7(8 z@GOI?J_%5sQl41gW-(-FH6f-rz6YeCT^k0# zY)_4vl42A9UAO*zjxFZcw?sa5IiQqda(V7vBs5dh2?QjK0b{RZE25(v4m{)(7PvUf zhrCV;ti>1}0)mG9!}B2j;N?1?+>A6fU#Y+(c&rCvJ}0Gv0x;4bO%eoq8}oE?^4}c? zi&~b!32TT`P^%~ytkk;bl<3%ajp%jJ8c)UL7YB&V$g*94ayetfdSTUQDsK)3GYYhe zieTUx5{U|i7wp?<{7o|F2q>o7H4F_%*J>E+xNf#NP;`F&I*mF23E*Z)f$}*yPwUglFA3gI+#ZgwL(4R}SIiA3z}X|^$|o9F>zc^;+0-ft%&sV@;qO?2Cl;(%0f*X&v0 zIMKoSTo32eQ%@ab2$9u_abf^IEn(C&4raKDp2ay(VKBUHq}@qir+J|u-$9Lqm}mkE zjMiUr31Xa5OFI=3O{p`J?nbIVbs#hziJ%0-28z!}dG2>?pOm7R#I%_IBo*cZI6RNk zBb+b~nflGd6156OLog^zRg}8=C-GXMn@Q$V4VsX?;LB7ZMR2VaUuL^lGBXhg99AM? zni$PewZ<@DkbA%A(9|s`fVmMGg!nZE%A&FjFWBQZN`FriimrEFu^1~7{#Csx0|gQ_ z=FI{$UGdHW(JyR0cc8lrG5j4uBsiBGCx~J*YibUTe-!H)D9pJuNXpTGi$d#XL}PDc z3N%np0DP4}l!J&TvL_EgX*?~}YQ?yJn|x?u&bljmI(Stt1>H|&iXM2Z;>gbiO{237E5Lcs0FajL9 zM9YjNi9ULvLYzip6Cz!n1Qk`WuFh6_F;|7`OvQ#t)vIy|#=Q)?^0&$b^Mq*9>=6hG zHQROi4rMh%adAfe5ch9&EO z8Gwiqgc(jyFyn3^$_EIXprc-HOhAl-@Yt?`uuCM3Nb7Mbt}+#>jzOqS)XIq`(ik>v z*%wHG3;Pc;00ynPmcVb!$pnuE!qD$6A;{6c0nj3fqzebNM~Tbd7S2Z!=cOpKQUv5(z#Aks%vw~$d_o8 z%;b-wWUYl7t3TYRDQ-^U4uSDjv-@#-pYmd}@UodI6Oq_J+C-d?LuP}0`Q0ryqvcTJ z4f`#!X=d^OQg9TKqt%ZYF&xb$ehyakPu9{8uMtvF%E?xvOJI(I>&(Fr+XSopziweg z;im~4ong(n@oM@sS!tgRVz>U7It3U8(-?Fm;Ix4~9gl{~b&Ux&vDd~1#I3MlkH~W1 zq00`NWj4zLv^sqO1+;*j^n*9{WorveI>pzOg;FdSTrs4pD-FqZ7`ELI3j89lM)S}0 zEg2@VghU&4bU*@Vt0_XFsZq<0LK|TCwhD$R84-y_jDU4@{yc{tk#LdIDw3*pxap6x zbDYtqNBW``Btq!eqcLcfBN~#1TwE#ULpUM@kCN~2Fh(TD7+?$}-y1EOEdf(ITvabj ziM_jEiyD2ZcWXYO0hqdo48WIyXK>h*2bBrBOQ&2q55*b^GyzAVwqN{tUF*`sCu>eD zfBX-0JdaUvnKm^n_Cq(23PJMIha8-&0`gPwgs6OyVuHt}PG!wvM9}Wq%0~Y~?LEP5&(2D#B_ag52j$FqJq6_@Uo0fj>&OP(e zG+I`#-dNw9p1!no`qJj=)f=ZTZFMTS!^r3x187iK)T|7!r^0Jw+?ORph5!d>oF%6q zhE`ucJ^+2$6~|~$hhOkaW{2vE+3eB>d{mB52w4?uD#Q~e!7ZJpjO^@I2Vo9Xy@VMo zDgGF|c~)>^hJDMCx$;WiI!EkX{+&tIYv5ZT*e_~wxDL9h5;x-vW2m{=N;+)5nkRD7 z3l8%k9U->lU~AW)3r`FnP}cL+Fi2PS+TK*BL=<%&smcZs;)yz1-PaoZkq%3up-S=; zUQZJ9+6`|}Got)!Jl%{LdHv1@Bb}$Fz`?y+iCQQj@dyt5GGhs=h*HJjUG=nhGceGHkDz{ZdwQ;cY01~-IQbN1u#Na_4 zG}eDmUQ+|29#eX_`W71@CVKzPyAJQ(F-+1nN`Z6Vt@Y>X)f*?zUw`68rs$d)ey>57}#FH>lApxJe6s9=K9j?d7XkaHj7dg zNXin4nqBRM#SX@V>6U;=1dmo9o1_J6>SUtH3Hoe#-(rhF#eCoVl%}lHS1}P{#S0>1 z%#IpXj)^&Hy#?B5RX}AFS<#W|**ayZH;;xt&Ee33brh#6yzg7l!4V?niTDS+pt0G9 z#&*S?0K+K%Y+oyrP#KMq?qCXPEM!^S<~>>9m@A&pgu}tBh_C0{lld3Gn{0VIvCc<4 zng6bWx(rN$-Uj)1z^Wf6c|#@k&9}YpE+uKUBN_wRxffp1H3bMFucQqM#!z=@WHMbL z#OMQ-Vbnt}O$#>)*me#AP|BLk>w}F08g7_66&pqF!Ko4Doxgp2$L^gY$iDK`Z(KdI zMnvcT&21yqek$eU`Rk8-|HA1@TcF+@N~6v07WV<3NE&M5ImQ9`Pg+i9aT81tkVov0 z%@w5)L@%HS%T#z$UW(|bFuI?L(^&F`q6U=qWOB>d92ydQE>)j7*z!LJy4|mmx-ouN^4_hrR7ZE7Bz!D(!FAdy-*sd%th^KhQi~8KLOT_ z@J869B^`OCG&m;FRXwLMCfj&HaZKSBaTBsDrHf_!kQ!Adi%rk`b z--Y;dJ0*I776fP=d%$5@7sLdXPL_j+#U_k(?0~|5CK!{35 zvWK~~c!W0qE7atM>P$}gM`YI>2lwCo<^e=&ne5uhM8CScI9~0iQjQ7^jG`Vtz0 z<`ViSpzJpUu@u5E50{|*CZH3HVq{hHf_}rNcOYb}on@~^p;W-)`SuW16v$FWXhJge z=*!)6`ldH$^4gN7uJ%Gb+?nBIL*XzEh`*?om!Zf&VTo#d{t8XH^{S(o*;?!$m(Wow zDY=#ec@VcXN#_Rf_X=eYSot^TJuUL%1Z4pD+~uaX%%0?3d-*&O9ozda;Rs9l$eV9E z@vm=LUT`rtwQ;@g7vpFdxLNYh_!5&YM!7&Q8HPY50(Xlv)3|tGBCce>Ywbkv0f@S? zQ>)bV72Vg()DCmq1v(W5inLoj$ELpa2Q<@I**Nru9W@!Ht2L zM`q4#4c068?Fq$W**G}=4%wBV@Q^#zINp))Bs!5o!9+Vwr8TOy-sXRSe|Cv`i9}rr z`WbJ8__(a7jEIf`t4|Cb3f6kTiLiPibQ=|g_%<&_I))waH!VhBS{pZ7vk^KZ7QGRv z+qlRjjL0vT%ge`lU_m?wQ00Hx1B}~tsEe3VS~>LzU|3JH+s@k1um{+rBNb%?=+*)WHP{&yS~4+3b?0 zAQ}|2Z_<$`(S((UMWt>O|y|U?qb76+oQE}6l2Z_{V<0XOm!Aw z$Bi*T$P!QhlO+i+5boguHZP1LzO~_rQ2m|-voi;_a9Ow7HfF@SN&{8}=Qa?#Q3FI@ z0F7{av6nW+B!g*#{?fNJt#6C_#FR0~7Y;slZF-)cPnwpaB}rM|3wmUrwaPh9LQT{> zM}K;OEp;5#Ect~At)t@06t>aG9u82g>}v4{>X|ioxIh+NElY&gBH&7d8ky_k#+E^d zfm=jGH@#(sBiwlL)zaYjfw6VO|1Zk%gF8R+raggSq;LL{P}9*j2C49ol*UU*2R>BnB_KFL7RxYP~M>8gBaT|V-z#@^lY2bg;B&boq zRDrO58ED(TQ|zBo z${s?g_#szIV8?%InC%DKJ@s6~)fvnIGVkaUh9SHvLC&{fQ`A@u}7B33-G0*|w znEi$N7L;Gqu?QHX7{OOnIIj!NC|1=nk?DEs^CC7(8Zu*>)~Pjc6PwY*F-BO!o-`v` z;bueAZ!O4Mz+$+w`8Zk$-*v~q$?h5Ryvr}XLL^7`E@-0iU$i{@wta2CGxD7_Gnj)H zs&tZuM3RaT%9fd$5OdXyQ4He&=uLB3Q<5O-4*u^Ooih()%YI93}QIB+^87 z&<}-S64G;#5H$Qy)|+8+>koXYn~~8bLy68O;)J_oB@T;A)tEJ!&r%sfL;*69=w_K2 zCHR(fGdLrOTOFAEH_xecHe%GM$6{f<&76#CE|DQoliV|+GTlRa^J1HzjWrxsLUiTYMC9K zsLI7XaG`lCOx7p<7BLw5H>yfVh^39JQJN<8>wsXm>PcNbe`4Ohzy}s~gf56yu53|- zg-EON6t`HXWK&c9!i2+1FvKFnyng-0+SU}47UF`))3{RC(soZioVd7pyH*v}C(p5SIhJW$+>`Qi6BHgu5$w4JnRv zvah{!hlTP*w?GhJ7$mg&Om!naM5`}3iCz!mVv)BH(Z0LiG>SC6_WztN7svlawv2Ym z@dG=*bmmH7fU9bFB?)vAAB>NPslMczA@1UKAV`h2AtSaNi4G17=0tOl&G5O|HME8G ztwbuPDAt*rzkne`_uszsr$=v2H2vn^y>jC9YtEp;GmYHhD-&*&i$MO>FW+`#DHwqh z=dS+7*I&T}MPr;YSm~>{n_%m^6Z+N!`;f6UqzHsb^4kt^T%=axcmw(?_A}a-WQe7` z6rl+dahQraH8?a8Y;|XLNpKh}hEd$KG;rCwTFvk}N-^3~ibp{+Jv_`+NZGuu zwJpS@E$8#u6H9>zkxp2lLvvBK?9I>;9FEydw&@_kF)!#27LjP^B$CLMI(X95Yf3VU zI%(xz-xArR56gGiB~+FaS z&Ezf(%?S*W%7CaK5Nh7`!!UQ|*+bKuH=(wZ2Fb(4F-6m4dE zOH_@^LO<*O^#=(7_XqD1tGBYS^Z2g>{eWqDyjDG=)M`~!FFIeu_1 zZ@qf;1__G9C>-idaV2@`Igi3ALK}F0?74Z?ya3 z-=Ne5&my(YdTXl40gWL!FjVaULso-p&WBptjRZ8(C2-y8BSbu84Dn&p&O6wbE z>iLYZg0EHfmH3B~atX*32yub6|NOQ34kluC@g_s2wuZ9X}d6m9}=)Zne)d zyh~(nD=5IR(bQ;=fcOSS^id3d>(w?6i|M)CftE?c8Z@=ve3v?d(Dg;W(h^RC?ExGv zz;R4Tp(qWyu30uJCE{9A9BCq-!9eZT{`z$27Bcdt|N2 z<{WhUg4Xhm$?=1W_uaDlC-$bDEdWh9erV@|Z`k%ZsH<1D9$&rm`08c7F)VG(RUM&tzjfXBi?_-nG3!a))o z>w9-bw0eA86f9(k;uBOulqUiJgT@7*r zlWUtFdwzZO>Wu)b*Be+c(annV<+E2G`QF9He`fgucP!1IxGC#fQy-2_rx;pnyBuCx z_}n`WeDv_%_y5D$)vGsZm^oxkYw^&=gNEX7igX|l*VYM5r+r9fkQthzK_TBN!NlSd zs!qYJa27M0VuF`ptv0p+5AF2SZ3o+15Z+hQB)J>H9xF2Sa}#848JbT)`V)d#lyPfH zl+5JI0$Q97&>Un!euNu;-1D&_(!v7!;ex^qk2)M>&()Xl)UlhP^Gw9$ob7-IYZL>y z2{EgrXo{UEfcd;eiqIE$-x329xI5FTbM>4-{$b$Mqy%S81KdvBp!k=n_<+EfXmENaX->Hz>F(qg$F7jS$(VM&+vIac*(8Q6F?-?!)AUr4SNL(Ktx;MZw4OX zoyq@EIJ$Si)&>9MhZ~Q)aH(TpT5{{(ny#uT zBdz?%_b&bfj&SwLjbI>i*V~9nhRWfkg|Gd}ZSVWrSF5MbLk>nFS})wCAUfJP{Hmg- z2Q^wVGvncv1~Nf)5w>B5EY|uBsrzD}tZyPv#DPi71CivRoE=68m5F8q)J}FNVx|Diss^K9?Tx+NGtcDOg6fNCJ>xiNN7VUffb+o|Mg> zq><~dIy3_xlnRq#I!;dZoQltpeuJCRnl3;KZT(b>11Uur?8u)`Pllb&0Io2wb^+Og zs7~nClZDh@tO(Ydvs|Dw(Ubi#!_o0d!^O>5f*E@56jsw1=F#4?y6Z8^EN`N#_wT~X#>Le-%9`nFeH-B+Tg1x7!n91S*CuijX_ za-+R}bl+lnJ#l$q^2ECj-uDk@J5cgUJ)`%q;c#<8I2b}x%BCwST6s0!9c3NDn&7OG zhaWc}BH?FVx$?}*8`httVWc!In<8{VQA2EfWe|q3uHALnmnBa~cb^i9Sn`DeBNdHS zKA1c8g8jC8H=JGg(uLYhPXTg}2vmyDk2LZmxk}~*mow_hN34i~er9uSv0zr=j#(%_ z<9E@ioQ5am#SC0^!(?a6b6rZT-mLmXvLpw@_z{(fE%YTb7~VSvkEA`D-*mPmW0aI4 z)zP0KWHL-uW9;N@s`5WoC2@cq3)vJHQNbfA8vD;g!V5hKKq<`v8FqZ=2yMgtSu<+b z53VV>J&NQCUYKNdRxM{04q>_kA#*{P%M+@C1Mj$1`;J3#RUGt!_S|ZXm@r2>FE`5j z=6)UGW6xj6=45~YVrPKW{28|^*k$L-w#qZF)p<_?Of=#d9FfnyaPiO2Yy`B`fIocK z{zu-n4=dkuaM$sJyUg#p9itq59NxV^M5|Y~T*uA$M%%B}Lq`*1v&P~tFYxl>4iYI7 zQ@2-E2|~jd`MFbq)h1yONAtdlT4Q zxoU={${;!$AGXG3Hj#pt!hm?Z7i6Q{zTv3D@HC51;ce&ga;hkMGXwM*BkY1v&x_&O zT2oMKjNZ1o$-fRCN}AH`r{YvKm`wDpsl+U+lnVETg94EYJy1pmX+!~)eCy6k)169^ zGQck`C%SGl%*%%bGbNldER$fN_`Hu?3%13g+K>j$ z(fLl$KDl;%_1YG!CybHLwvWP1cqkvZV`;m`)BO3FD=%H`OzCQe_3x}fVc7YpyIVf@ z!UYm~!@(C+(h-F@F}yYXea6R%(U(vL2mIDhTP(!zsp%ropS z{rJ*}bJq%}dk*e;uzu6nmE^JSudiO&I(}%^{kJXMfBRB|eD uRing#_H7@MX19J zo9Xz$ogaMD-bDIi-@j1asWcaKMU%d3( zh3j3;i6hK^`ktG?0y7c)_1P<5`r+l}1%B|&dmp%C&oO|!va$8-xvO7#>C&@>1Fqr4 z6UhU&?|Gov-1PBpy}rIVU0$42XgeoF^z7NIPrY=Bym=u=mKP@pTNB(4SoZ8{-*hn6F9eEYS5+&i{41R1Rl@${0w`(M zMYUhVjt6#)iUoqvwCNhVSSSHx&ooVdZon@;(r3MnmLDY3*Ko+y}58 zB@Q`48i|gNQVJk)MZh<^qyax}jj-p+oOjQCURwPwAjh5BJ=`twPw7xANB1s%_8t4@ z>D0>lmVEXH7YDipcFtUfNA^K^eDzXtgr2qq*;-zhJn`;B_utl8-Mr_}u6qvcBBE0l zt|u!=L?_N)CB9brKD@N!!8^03eCCynN8Y~wgUx*+y64dDdk*dXgS!tr_PzDTzQ0~Z z(n3xaI=pA01^CpDF0WkKvK@>@#Y1n||HwP`o0qb3YNg%9o7`=K}Qefs5%|M0Ezs}(CBhSG0p_2_rkKK9oA z58V||J-oE=r&f3bxmg=RD(8`sq z_08!+cP(S94;|ijYW@1D_3ObG5)siuckTbeyYuk{zw^yknP_cu`su^_(xd0ruYd7{ z^=#-0M4{h*`+?ST|IRmGrCNj2w9Rrq_I)A=);H=KT5B}`{rtBrub7dtqKbu$EF;ZL zp9yTlL=o+^u(O8>J?i#UpDhV(T^9MTBo0k3lESMKBQ?(WutBVI=(YeyeclRSJQu`G z05e{&Z=cn79wt;f%=9Ii`Fe~&4#yi!Br{{sTlgY%O-6a+dCA$H#A5)!s2#YGjeY{S zJLurr5AT|HmF{b6*E_+;AnwvlhZ51lZ`pg|7jNF~bntUOxY%9ct9@cZXJ!EfVPfiR zb0;ENUfdChmBUL5Cx7L(t{248dOazi>;}3u5`?JC^_EeRnJ`Oh~9~ z#i3-g+G3JN!bE@YGY3Eat^-3pqm&Q6X>U6^D*EL}?hgL<-@fz*?>aaR`GGt3eC1bf zKem4-55uwYm0!8z(|31JMLT|EY2oYt_Kn?Y-SVNs`@Z_VH@s(gr!ub^(~p1a+_Pt| zq{@Hs-rLJGl*rk5cxmAaM{h|rKlA+BldBhnjxOH?A<2{({JXDK5CoaEHpada4Hm2D4|WKGdm}*#B*|+!eDPG>Sa(cE{)bpi+7F zypLM!&6j+XZYEk@;Nu5&p7_O^AAW0&{b!aZUb_7FOP2?tD^occ;z6fWgCJ%d?!Rq! zsWXyX1CEsIeq26~Ryv+-$KQ zpojxp$D4c8xB=mB?r`Q*Kgs~&m=+3Ww9ks0^qMel&jb;XC!|{x!xXxSI))0w3Cb{u zU|SsewO~(`JH5_g_V~2Z+^87LIb2|9UP|t!sI7j6CsYshjB9yoOs?|oo9413>s!;) zmxtEpFxH|^yyd3DyLO}wkM3PKy!$8oX2q9ZyY}#RFVGbWn8nUEQYfzJog- zd0S4^o_rnBVnUC+ZC}?5o;-i;sUKbb+WG5;mv+2)_rim3+*1;yP0<~Gqda)#l`Bub zy78@xTgwZ4cxmC2Z{L5o>Gb~F_k8TgzCV8PqJ!?iWGXLT*A1jc_wD@Tzgljqt#3{L z_y?Dser1D*4lgY{aQp5LzNr(b2+`CR8wUA&;@s7zU)fmMxUq6&>&Vi=19vR7(3Te` zfBJK`{KntDOrRIzWo>g>)(e$fA)>Fo?~c|Bu5C{L_@6JFTEBK`{rcfO3&)msKJ>Qb zw%T9(((S+eUw*i@HPvz9wHxy9|NfP)zV8j_(SPd=H$C*WmYRn2^<)ynbw(# z`jdL(sZ}aSz(zJO^i;n1uDKP_i4j3-ux1Dp=rYGZ6gINEW$qWLpR@!5%>xyqhmIKA zbrwXrVij*rh^By0n28o<$^r&`wlJ&wF|743jVeb=uXjZa4cQ4K&h;#MVnln<^v?j- zqJ)ud+e!TpwdO{fv6;aib0&iLbsG6X>Ih7G#ynK1bT#KrnbViHX3g3~h>q=DeE2Qf zp74$JdvWt4|8(AYnQ1O6KUY5d_`#iFu!V%^$nG7-5AC|=V0K-rS8trKoifCSmnQ8Z z91%VI{MzHImx^y)xw1ua?azO7`H{Epf8?&-3g~Dkw7w<(%Qw%Sc>QXrTF%#g>W7y; z|E@zHJCa%Y$+s^*^`pz{B?~u@fSKxKmx!474}PZeb&+RYx$@!X&aG`t3wEb3Tz}@} zjmN&X{*_<8Jvm?|n%GAq`V`Qk-(7q3yX&k2a%%nh(`PoGK6B+yfBxnJafk4|x9q;> z=G`aGT^k>@6X&izxq9iDmp3fmcwn)32>#%kZ))A!%Es1jeC^DO8(T#hUfkF^arWvn zXEy%NU%LI+a>!R7*|YGmyOtmQ&Us&9o6@PZYyaU}=l+kMzZGkK;polJtzSEF?&_!S zKA0wOYn#(wJ9*~(=8d9|R@01Gn$JW$DPC#kzr3P3e0M>}pAP_O+`f zR9 zdBMQa8LJFEh|!ZJS7QMdV}1N@27Hq*@N}JcO87E()2KKsIj$5$`uTTT@_Au9VnbL|k=ba?)We>(r{xvSPC>kRS}|8%~6S$=tO za;%&`-Un80>o)?KkM3W*r#UlW^~%rN_7MW7oeKaanFy=WE@rG*fN zzh1O&m$dx2ce{()b|QoM-)H~YTHg`kxH8dwx8#+!pSYY{yMF&S&Rw`M;`7CW!R_tN zno%Bpe*Kx(u3)Ffn{4Om)f=CE;Q|qreVasxNN@LzNofD0sI+?3eJAJ?6Uj$Tcihtb zx9=IJxczFx0qm{MI=~FLVO!W^ePJBW( z6~q1Gr}D;`BYPJA;=OmI=AU`~{L^PP((6sE-97E=e6>7u*YbhI9r)(RZ@n?UF|x9; z`Rv&ZfTyQ-0X@~Gs3Vw0@kdJ$&8fvX?tK`{5!$7xy$Fszz@Ou{CIot{P>FEiD(& z@lEiK=TKM_Zv1bs!W(C$#FRe==|ojxuUZ-DepiH<#Um& zR@E4$M9{kpz#P8NV+$W97>GD^N?JIRSn+*uAXRzNnmH$mjq^13gd3Uae;u;oZ|*-y z)#m@Q;Of8&d<+*tbo8e*BYfn07w-GUx%C?ZUs*+|yXgohdea_O~j82-HP0hsGww`h?>|*Kn z*hK{A^DgsbuZ8GxXU7%IE*Aa92(j<*J4Jhr@lt)J<16;x2()Spkk$rMeKKbzbA4-i z`ci+206*pM?j8UAUuZhHcK!I@pMBz`%Y9L=#mo){vNOv1maJaAv3m8!`c{U!zBPUL zJL^YJyz>5kcFsCLf(-b?N|2 z&)W6rzyJG}+w^8BD;rxM{^qMiTQ!ERdqT{TuWW3#?_+%E$Ua-3dZ2x$g6NOp>xQ-# z0&jLn^Rs;Ybr3Fx!pTXj^ z=;od{A#38ZVeQ~Zc8p)K*~C<69N<_P2vUaXj!y?mYv-cO0N6#Qtu0Yxq-Ewzu*Ef% zeah+%lNF#IiyZ`8l|dypo1aJsKZA<+?*t0d33Gm8xAlV5$PyFJ|0YJaoLswp|399) z|C{GdU)(e<%-95DfhWzDg@>PCKYHSocb|Cq_}5=e6Wry6$rG426%lg(-wZg2aXzHd@R5Zt3uZLkAx!6T?PDf~^ zrvah&VC&PPF`(?IEBI6E*H4`7Od^TsEEtywl(i%>|wb`q-R-Et= z2m#tb6AKRC{t~6Goe+pZR#tSqHa8c&5`*2+u@Df%L`0M76ND%wW9KiNvY>sFCcqZF zIo#oZPZ+8TuuPc+de(l~VfW2juP|Z4-jY!%c{xaVc{n_KE7io;FDP*n<0pSIfV_`} z5XrpLK_WW2);V)aS!M25{(BY(Kl<(U`@i}6iSyS@4F>!;`Q0|goUg22*}Cr^&L&^@ z;2ZZm@vgEPB5sQ#TEEc&y}U5NE$%*Yj3HzP>!Bi4WQj^rDEKhx`;3_y5MPIF@b8_s zH-tM8G4q7SA`=TsYzIZbfd_(#GUeJ%d&@Y6qYkcf_FxP3GL03ZNKL_t*Cw0Q5Wd5!arzJCD_ z>Lc1Nh}Xq3%WDyl?$oRQPajCqvuaXE)_M;oroOYE$Q(A()$_u5m|e9y3|i)9*ms*! zUOBFgzWPZHG8XY2Em(^qde)6smI)z802e{%zP9m#JXD>Lw`4=0hYzsa;}&~^#oXCk zXoZxz4^d*KjEw9kbd5!0v@^s1`vajU5uIuP7rV_js-mE{1t_ekyi#fY^}KHl&j{=5 zJ;Y4>(QmK4=j&(R^Yzz`{cZj4k?-Xl3PZ&{$;5Hc2L_l?K<4&>#n()PX!Yui_qV?A z&OHymO}F-uvvB=-2m0|tJ4=9K!la!Hr4=xc8E}+RKeJ}vAg!>N+ZDlbP}vwX?Mz78 z#Na(Q?*eNWtW*~q6NyY`eft|}adi`tqrKC1$~1RwofFapuWp|6*#4cU>3G3R?SdSY zBe~6zfchU;o{-rl9B&s^$y zmlP*+&me~?_J8|a1AJdWv$*K=iS_*5XZdEq3kH&)4rCSZuM0nkS&3!cR^todbFA#aD zkq3Ag#B)X{m8EA2aq?CV)Ex}rzEcVV7hi0ENE{cD&Xaa=9NnJ;83Z{$jO1wfc+^~%=o{kI&m zfBx>}u;oXDh`;=5HfPI=lh3{LK;F@#KbD(At7n$`ZrhE*X-a>1X~&-&J=E5JrlK@r z8vS+br(fP^AOFF-4j$RFLu=MBp%8F?|C`CoQO>a3b7=R0g-JESLbw(5Q}4_Nvd{oZ z!v#D9Gcmz8C~7&dIQir|4}_3f74)II_O}Q;X)jcuR#@f}%F4!O`zGS=-*w=jyY{z| zT5Oh%BY*LQEdL)oyste_=8G?^gRL@L>ZhsPIY^1kLM3pQt`(Y1s6ub8x|`T3jwzez zg_sjN1eJ@vBWWV595-_j%Jm{0dSPcjmFHZ81CFyS^zl6d$^I8FAmX6tgN>QIO-G1( zo6LJDQ<6Vug9B>sl>Jg-{$@z~T1;H}4Q!aH;jds!XZgQe(6Qt=e+*aiqk`^|+bTrr zYPCkEw%;nwB}5W4lEE>rL9X^y1%V;|0mb3cOkUp_`#j`-!k$_fXbNB%Y&<- zLl8hfYtt{kdgY1L@XFWn!sO52bBGA+0n_8Fm)gCi58ipx<3D@w$kGmiXl;3cpLqAd z{{EU0&j)ko=e3xD-Xx3w>CtX|nVaqc?Jn#0*x*vk@u(#6SN zy|LPYej-p3+{=+WoTd&H`t@Tc!SkUm2}M8AIW%(G{&KK6V*MD){l zA9~L+COOS#`KJ;8^33HldF>v``fS{oBPxCIgfEEZJBwsQy1hpC@iZ~PfVD%7aLdj0 zi(M|5g&I;;ELR3=r<>lSua9o_B_RB?l4}Kyg}?O@>;|p^o7a+*9m5;o_SSwr5O9QL3aH9G(YDe~{Q9 zshW(|fDi+NnIYB&tV_N5KgQdAaUw)&v&!dwaOs(!@62i1^4WJRKl9qF6-993F7zCu zu5FcS9zrTmgdTh0!hN^yF2_KZa%}(3NABA9S&QXG*0<>Km5c4cX&=1vrh5+UI`R6o za`Mixy*ux_efNQ(4-Us%dRspE_T>lfyy?X2*G^yD6q0)n?H>Bl#fMM5u7F1PP;5qJ zhnzIkug?mzX?rS=v3;ZkjZ$D1tc*X6PAto^~evs{&f(4IbXd2Mrg zU~zJ6d1pKQBBIYce}3h17VKmDcYf;b4Dd70pI_P7BIf`7g|*|i?7~k;99W$E#d~l2 zwUcL>j}L?4Ci$Zm*8lKm#}iw5_UzS?7qvUySks~Yy<-)BR*y*Tz2H-W<(#uR`1u?J zR4=#8LR5}ei)(;tQ(D{7SSH6QeiO&M7sDv}##HrvYE`fsvBTliVH$^`vzcWOB2brU%&>s8#x*kbNN>H~1RXcUz+9$rf_W7U5Yjs5Qz#V%Y zXm<0rh|Q^t54@rCm0c_=SGFGe?i$5Ft}SYc%~+DFuMiRa(F^O#i<5Ts@yMQqc2RGe&@Ubt>*N|(n5QD%#*7ZA1xbRM2O_W-#q)C|1lg8dSuVypZ&tEzjpG+Fk3V9 ztDrr#dhvffywCRic9OI!IeqH7X2EU4C9(N}&$V#c#zHb=;Lxj<9wyWcY^z(FhOMVv zty6dt%Ep%!C)=vW(4s*}uO0Vj4C=-iyzhHWSsMp6GOxoLXg~!-5yBbmOKh&KJMn1vh>w9@*ybY!6CRE?cT{BP7U``v z&f)lTxMUf%s3-L_|2u!DFKvSO!)}}M-Q9VXY>7Vl?R=!Pvzsr!b|t;>y}USi!e$%A-!pySxpQCo@wQgByd2xNtH0#G&GOjy)*t)cI{7X> z8W#$SuZ8G~Ke+JN_vRglRF3RfSYDWbV*P~W)P?K6`8O}WxUog$q!Ke>O|cAs|0)A! zH<%uM{`{lg$r;P>^2Ha{fA=3>Q-D<62X{8etZZyP`knKndoP*z{N{~+_jfNRfcM_A z1YamjVLy}5+UE3^{__vt`PG$oes$${zVT{{(z9o;bSDoIN%xEny8(6~D04myE}BIz zodEQ)ti>y6_))r@c*W|On;}yxh&q-k`63`2!X{A#U(1fxzx9c}!v|iFAvG-EGqY6( z2S9qP4e5+9B52Fn4y@}4qYH>HTntwYeoDz21}=DKE+ZTI=)>{=XwGb2E+A{{zr&R@ zcbeuycxSmUGRr;S{#)|ZIP!@@a~=Ux&S63HH7Pou7gQWquikj#rMXx(HM(PyngLB|L)btzQ3-o8p21P2TShjvF}}Y*MIqO_na5+=-jp6 z{>E$TTY2hwba^Lf=Bh1E{qWMG-(6oBKEP#dYx?MS*M9l0fBd4ZnN>qLLnq-|L+RBt zAz}K=^XGr&|9+|6JJXhD&tCoYubuh$w_Zp8S3rH?-Fbry5&infGbMdmchM6O$?5g$ zA3uFA)%^XtmLIBVb7{5RaVBo}i^3wJv!S@X&i;<(rUm#uS&lon)OWe5$zitz-@&b# zxi)zXMxCM z=y`zc_N!1kz`*_m0h(w=3}ty?GUrYi{OAN%NQD~d$-r(bHvi^~g>swQq_6bAvbwQ#>f-eiuV1r8HQ#%{^)qc> zd;R){Pn|Qo3yW7binq4C26bJ0^~%<7e*KlB`xo!OZRyzZ&I1cOYDZVLPF=Wu;@q|J z5sVMhx&1rrJ3JMO*j{Xv$C=EyWf26XogPjm^J4d-a(!8}^+IB2XWu13M?rtzZA0Z@z}KtZz&#onu9MG3*{2O78POc>KEBX8 z29}4Af?ACt6`04D7oYyQ&e!SM^0^l;ef~w?RGBy2&Mc;@LPYfNTldcSdHOH?c;lnr zUJp#0d-jQ((H%Vljl-BU`;&d^pBuC!{zV;eW(@K|Z7^LRee^B+K7V&!*?RxqzxwRC zt4WASboW9iby~x=a=Lx^aQo&gYzMRU}vF)ojg| zsaI?#J;qSJiONO+dEd#%_Ejl;8wiyOvB^Hf2E2=JL*&eYPBuq_+;P{9RM|WE(re)ZW3E__UIWddC#G)IZq%MC~`R7PJ zE8bGmX!+eiCZ3`e+loxNgaPPo1XX@iTEy6Abb$0vOfa}YH}@1@I38x4|b}zN@=U>>UG3Nhjf9q4O!cO?@wj^51QW@(62Mo@U6=BS<-M%;k|)`GW&olCqY(4y@qzrDFF&zFjO zeIZRRphuk5Jp9vKH~q}!UOK)tK1y9nr|yZYScF|&sZHyGNJ#hZ%v<=`|YvM zy`%S$ccMX$80?Z+gR~WUD8gzU)HM^SCW!*$>&yy=LR9VC1$TXTaZeR2|vX1?_mD}tV9IpaP6eI&v1Pnq4kO+w!SezW&zw<+f_x;)baO?BGcKD&U zXD_(2vH6*Azur~%x|PHFi^4}p>9W7?ec$X1bqmH^i(evtV6%ZjRbj$wx%U?xX4^} zV^0hQN3}Ez%V1aY7YPJKA73SK4tQfO(2auhEA_g7^c|;R1dSiWtaAI4FI@VQA8yQ~ zVSP)=1kR0TjihF6li}{bGLd}l2bWH+Z5p<&IQ`sbua{IW`wSl-kW5FVGYkfnH4H9$ z@1~%j47?xe9*uNkQ_AL|sPdt#ZA~ls*_CSCtY(gxf@dmI0`|$NW|qJQIb#fA+{|8I zVb%k*^mDJkAY^lQM!UW_Ma#o_d}ZR4iD|a)U8WcUX7ViEXMU+PH)!b)dm*5bYR9d!EWefH4+VmDy5phCAKjx z*mXTZtMu-|I8}zME~-JtjrEoWfwtO;JGud}7stWWm+ejQEQI7!r|4l@CSRwR0`v4TO7rWD$l{|sRH zE@v75Swl2LWFz|`OQTMX8f+q{Maw0FvU+uEwSUNBpCGQ**YftGtPB(zC*7W&IDf5; z0EVFy42GR{)=giO61R`gHi;2Co=MtScVu7_i$mP$IEJM8G-R9(u%)m>+LL04XywXQ zeX&&_W5jX9wUdc{q?9x?)F?X<6cn<)iuKX8R7cY6c;9`&T5?5 zmERnPYplR<*yf4I;jnv+qQMj$U{eVr0KprTOc|+==d-0&JZrnl%h{BzY;69!zkB7y zjZMeJrGq9Dok~aHV$>pcOs)Ve9xTCp;l&h4PwkbNMW~{!1_cN=Z5YZ6i77!E*y6lS z;e0qYK!_~UTQ^3zA}~*sXA%5>$+pPFU7!S`f~!)>5SH>8rY0Ba0*8O?aeFhi1lHu# z&#Z!*Z=-4ngRH(s40x-aB=_+ z>whx}h!E+>2@%b;GzdNkVlN_-a!iW%J2$S|7PwL8i%2t+uzc}__4oegAD&vf-m0<0$cI$$#TW|HwDOoaXeF-RM_=HD)nO5i4(F$N3E)jM)GN9foH3YJ}LA;~HCm-3$zgk%GkH5ko z_{xhw*v&v-PZDBT+1Pq=_2QE&7gp+PEoiV^H8g#oQ|FizB#=HKU8vT#`bvJ4A@pcy@S`u0w-?AA==1Gib#520j@~2o84?HqG0Ke5se=PDAHjD_Jq{k8=<*1Uv|E zdX5&OZ|be?+~>-fEwiH>aJrmfbUZ|;8)OgD{zDRD`Qj0G*a$|&TlFd^i@?xkyb^X~ zmIv|l7TzHkYdf?rOJK#JTpr%D(0+dCnOF72i2|fgG85G)2?QwU0*UOcJ2RecLvb%f zil>!0alc8!g0j3YNhgJ_ZBC!Npf7GZ+0E&LoFzxXfG`Y_YJT#5;;lm4POy@^pLY}5 zC#xOYs_4$RskuF6@yBDst{2<;kLdV1*#X2>6R#6+8t&xVTrCb@h*C* zhm#wF3_e)QeIt*&K}1}yOJ}Tscx}PPr_^D;M2j&b31H*Gh=iK3m#%G@OdHjopiTbu z19lD%bmexJBfnZwCHi|JB4(aW1v5rQnz?l|s{mN%-#Q&k=WDxFZY)$lNac4hFf~xH zX$5DFkssXw?nECjXRklA^$24mjs>~=C!Zg9*ibZrB3fl3nY5o_LQ5j?X08S!+gwQV zw<*wpG*xxqn?;&Y2)ozg1ro+)$y#eU1~|wVwE`BD5=O11uG5pG$AhThiW2Z|0PJW9 z;bzfAv#dj?BA&gCErND)gueud!LUfy>oi>p#odyGb)nHV{Fwr91)9Ae z!8l^z6`g~_XKzvI7&ugiC@>DlAMk?64+-M{V-_X?R+ZpU0;e6;FvT$`-y;$wN=K-$ z0fkAK-KJ|ZU6b~i=vg^r7980M$x^odn21^I)j^13K&%Cj3m3*clcQH$msQKzIsO}iD zP79CMmHzek=X~GwP6B))RR_X&4)g6}1ry4B7p+U^DDXcuP!EuFWm>7sX!<~3O=Yy9X z3*2%&vB<;6*!QHst^#|}P~9mox^+@QGKJyjSf7rtqGJsYNWGAK3%~(#93X$5<^pJr z0pl1ORQ9pkV(nNo1a}i9^j$$ph4Yio(ikf83-8flcBK`?xF2CM8py33FU5U=Lw{Etq^{g-aYnzyfR} zkNrudf#D{R?6~Po2a!gjx>Jm=jMe8Zf^wOM4IcJ@Z?mhWZI%4gX_A|Pwum4#Eu#H| zv75_NyFDGP1tD(_Cg|aOj7oydL^Hm!#G#X~J!p zUSOEK;gC5-ZMGlLglkJ@F%G6wBdZsUvt?8OVEb2)R(Z$tnaq0hLML>jUN|2hg>qT%#ZqB&e5{VfC^b59Cp_Y}uTm?L4W} zf#Q(bK6DREG)bX6SEwR^ykOmhibrCz8u0p>h5Wi>aH5Gi?LG>O_rvaY!B7x#rKX^! zb}m;dd<<-{M3dGG^Ge*yc+p{GryWkrJ6{;~GqnMAUr4A&WCy;3M{a8Y5|_g9~%RjXknky--gC zKEP^y$No{M%20>b517K2>QLtll-XJ{?2%xv@9YOl9;#q?A69pjM%LMTGxj{xs{;g% z2|TFFD;-a*YzZD!V+-^5oY>MGP;!S0{>CLD?f63jUs56L_FqKAGnr0Erco4MKtg)5 zE&AC--7-sD=G#d_ox9ZyM9z6cIET?4sM6Rmn+O{FRV$3#LC4bc==RH{*@kNy{?vt~ z6}DUvHDWfVK8AB35?{th9ChQDmhStItbce}uv(#6&XbZP=Aq@7DHs69RVf;0Hr!)f zwb2`+uQ^J~ebhn0iim;xzMud=vWO!eWh`0-7MYT>LUq52oq7`{l92q+DX(pZsJ^$D zP}^A~gvXoJNit5Qka%E&7<|&xSURRF*^4-oW;NW6=-A}$B=ApSO;BuT00u|2S+Kok ztCNqj!DXiDRL2F(q#+x8qGcEJ`DEf4in=Ok5JvIW72A$LSI>qC7tw75U|*j^ku`vkdBzTAGS)0(!4;ZC4w zHZbooVJ%TiyhE{sHHQX$;I6%66$y=Y=NKa}8>9-9Wu)!gx8QyVX@X6FLB#kauE{;R% z_TM-IA&`HJTO0`xO9p&H_b7o}jYRbW`HKRX6^=;6d60>bLp?Dsam^IotzvS?GTBkl z<0>mP)n0CeT>lUrWY(!APnae=S=bR&DM`B-H8BeVn>~qqY?X*|0ei_}xKp2&2bx_h zJ=8M}>qut{uE2bDOKQa?@Dp^m^?aZh`TPfl6{!ow;($+U;L-al_xPd)}62_LT@OsgQBpofRI&sOp&yG;F&^ldt(txO&j+y zl0dfTojeiH={{_5B|CK;kxxcSj87bxoG=#>;HTEIu1v-Kd9qj1{H?@0e2=+<@YH0# zEn0cFqXcL9Zwyde=L~t~h!&VWLxt5HRy=h^hzS-q4-#Pxmn0jKFG2ErLJAuU!RVOs z%}VrT1O!0Ah|MM-U};8fM6sX)@?jrSe?!IHhc@m(Ha%%vpcU20VzNhBg@}nt-du;M z5wcGu#Z6QFod_QQZ2kwpj&xT{J7!Y=&|tyAFCx^KZk~W)gkag$V|)SAr0YALgVbfXT?Z4rM4`4I~U5llbISSQiN9Jj-79%4NuZ>K|cL&F=iYm5dV<6*JN!Ke+EzF@+I|UM#Ls7 zYy=8-Aar$6M6t5k=5a(+9Uu)V_U#d^?*pcP18U+A@d79P_>ldnP@Px`QF(#G1h50} zCn?Az!yNLxji$Ksp1Zg}7pbjbY}zR}8%rED=R@wy8-WpPbSbt_gp^{Gp+=^ox(z_o zzdRRx)}`$dY1ZzCi~3FN)NMSQ0?z_b>7Pv+b}RL7RmNez?Wmcw0c|#?F60XdGB%I>RUr$-k&YouswEhC;lh zIb@{h<&cT-zM!T1;?T!3>_W5Q z@kf>#2#0wS1umJU7T^=EU!TYV+IY~;K#g6uEyd^>0$cl>W9cY037I%|H5i1?fDq(A zYH6}w+DG}Fa-n^9h&A%v4tB5z<=vxVh0FQarM}qA8_72K%ncSz%dyo-Q_p7RID{y4 zYg;=8`9&$NF!^)8wIG>H2g~2s_YMVIfyVDkl zlx;FBG}SqM(fp9uvQDQYQ<-|0*fY8q;{@?{XkOE1diFY*n}1e5*j=YB@dkVeIfyg~ z&+_j{ZV9HeM->&Q_0lWrgCZid zWLuG(#Fbuo%_X5R@y?oC)|)H}I9N(#MkpgQwc{|vI!=*6B2gahNq%G;+MaH?6$g@} zjFKs^(4PQGgY5(nuRZ9oM zRZ+bV%qu&GDF0+kT@kR3nao|qRAs}M#~{HZi@D@EWt?G&SO|O3T>Fw(4(Y%eV~Pz) z_Hja_DGrHsEZjD3QPD?&Bp3;fcEJR)V7JCn(wKxql1YPcB8UJr;63{x=w<+=c?ik> zqw7e`i{*69nkPSiiK!&l))*qIYsQa=JLCc47+;Y_ps;a~CCXwZOihFeU0t^NfM#!% zVBs~J3^P)WG4!M!2Z3!ybVj^A9n*i2 zv!-m>iBIAOEW8U~%Z`RQcA-%2;L%KJtJo+Q<#y*2&FNcs$?Dn6cKBBJA>t7?#ia< znG4Lz4lp!6zyUBd$UUtd0FjvOAraJAcbX1uH!g0@4gRvkq}w!3z_ut?_Q*PByhj0B zKqJOnQT#&&j?9?qVM_65;&*%-4*J>C3!#g{^Hg2i@M~*?_#)buTDk z{31LHEU?2Qq08)L)V&B60W$xScmUuzrubq?DuGI;s*gyv%wknxE68*Yy+C_WC!PyN zgl&{X0MmAyxl0LnsW*u)JO(A(O_@Bg@34B-*6n(DVgdXQRgr=Mx1*)=ie~c!v>gu+ zq{t9GRYxdxxpEPW3Js>loIfjroi(WFEkQn#Pi;d1saP!^m5taBC7M&lGiNoX<)84J%ssrf0jN=1P~IULN^BSU%!>^@98MKku(6p9<{!vFFpNIdV(ya$Li4BptS z-=&N-18V=3Sk^kb0vSdaY%i84DJ@>1>|^H!%EQ4cBkCZttS&KwS+fiP@lZlkV&AJ= zBbZAxE$$_XO_#*WsyeQvJc@`v0_c5DGLubsU@U$3vzHN05$?W4!?6ZYRvOp(}ys)y@&P!>SMBXfv3wl*V->lH9s)l7*nN+*T z)D~%mOXfibMa<$9KC9G~d6d#%(^l==lyg3vBVZ>Ox^pr`Qhv(-l5tJe4EP6J(Yh<9 z%;uwQZ7!`A0inO^kv{_u9A9GQQb2__BbR_bl#Wa%7!YCfC7#d)$hSO@b zuCBI@(CVPXp~=20R^aHIl_OQ-ygkKD#G*JRVqx9-6ojDgTmH5-ud;s|0VI;Ar4UA? z8j`kPc`XZ=!ZL@ut7{szBo3`(z87di^5R9TV9?HgTZ1*l$i(WXDri%GnD%VFjqU3r;r9_bM#}El-`wfEQw9ao6 zv%FmedDEED?Z6qUJ8nR#Xc3g!B9sVI?xiZ}sP=dH@L9-MRVgtKZM~pHOb3DhOV+%y zAN^Z9vzr~J;Ip>f?x2VC^D6-UM6^@*DWs^~YirH4KC#0SZ<0tXvW-H`Zn80S89DaZ zMN0+dUPp&doQk0ngL1CX7#u_mchDGgZ(32zD1RQqid1U8?T{hB8{W0|epj4GuWhS8 zIOfm>Q(p{yCY;V1Sm@P)QTay9tQnDtYW{~4WXQ}spj)q{teN#=CCVMkkduDS&DqsA z$QBfweRYXT)c~JT_aan9H{T6JV2{7Dh6&_a%{WB?hACae--2m6(82NOcgZAj01;xd zCMc;0%Qf^wk*Cx$Rw$DDXe_Q0Lt(taum;l*YFMk)5cRetYPE={3#v*gMyaZRN>Zjw z=zmiiF|X>Kv)UX38|x&Wwz6HOzf)Yn#EjJlC1$rWeUUsW*@CPE>f}L`%kIM02F6~6 zY+JeG_60$bXj50@o}J@N8HJ$OmWs6?ZuF0P=P{?BcuHSS-_oBfTl&-Gu6nHH_8nL= zgzTmR{8Hu|px;ux2m|^ZV5hV<`V-{7nDQ`S}b_7f` zInk`I9RuK54gmNG9KVW()KtoRVHc?=qS|n-aiba`a|C1@DOVN+lW0y zP#kV7@*#s%bbDg{EJdL!L@{x3lw+$P>4qUr^s0tKyXUx5tfOh>gocqJ1i)#bCu~s* z30ky``9lEM`an|2Pk0QHzo!)HQs)3!c5TYy zWFYTYUA7rTXL#rCv)|l(_aB-ub8yDYdb3e)Hr74y==vug9UdQ5+BzV|4xrkGOgYOW zsMSit8tuP57@4~hsW=gwe!_yQFZ@=ha@3(m_VxB%{iACf()`PV7hN?k5|f-*Oo782 zN+bmkF%tI>yoi_$;-ek>WYL))TKtiJ+oh+^BDCU;+fVz*`%OwSb}jL{I46sA#Y%yO z0Koh9?6lBGA^`HHCt~u_s15+I#XG`V@vEyy{sR{YXA#5B7A%gBVOV5TOVIY1{eL>N9pD^YKZ6zn|` zJX2Hp!SmcbWXF@Z!vLFP@tZ}C9ho5SGApU|On%s4!gwu`_DQ&9`GkP2Z7rZy%e6%U z=t|%TOgj+#STiuBZ^*os$A9kWh-gMdhrv$%O0_Fqr7v>L$y}$1fmsyq)R~eCx3YIB zYSQEB?|t7nr=K-<-~EeFm)^SklJ9)Oxx=>o8;+Ul)!VqQvSmT=zha2hu4n!!cF`!y z=_j6I8!TFQc65YMNF&p@FKe)(%}pt^$TN`~I7JE6bMzJ$5mlMkI;9B>aPhh4UGR~A z%SqidIt+l}m;>?cPXvU76k}Wog1KjVUW`Mm$Ry<4b#Qj7Sz-8EVACxmEO7tIW(WkR z0k&ce0K=}H(jd9t5(6N@w4J*97tGz`eFt>)bW6kD0|$B!8~}i=&B>QmJv(;onk~;Y zBZrewAOL`w|8!{o0tZ1#eeKxOU%6xQxu$U1dBMalF)sk1f58ERXC0OUES8ZaYeufU zH)2G92)n&!)|)_imYJT73&G-+0`9LuVg8C6Zgl8}+rL!`IyV+C;-DX#^MkSK$^G?_%giq!zoy zOPYXQ*c=O-nK3ejCfy~4txFVaK=yNF793bt;uN^FLo5jiB+Vb)DfwE3OS+j6aT3`= z!y4jOU5O~m&)n=VwMZ6QwIC^wj*41flSel*3dqI;svoBQ4s0dz_0octprmPgoQZ#@ zc!^Cn)gk)&p&5f$e(~$`4>_XPLnx=e_mtD$d&=U={{1`GFG*fq`K52zKUd#>&(d3O zQc^p~h=Rp2^)P8DV7DxoGI!RT51;Wt`{b+N`TFqah<#FT)ZL%2G`N!h)@{uHfuj9y+cclQ28F9-mu@4K6CfK-M*keVbNjhLy7k@>d<(9#QZ*v2p;Y+x@+jSMxk+VQX7u5)J@e zJ>9z>JFD-&85mN{BC&3nsx$TU8c|Iojzl~zJq4|{+oBc@Y|;?rYK9v;UPxIF?N&B8)*W9r5Pn{ zVxTy^ggg!d_6Xp&9XZZ%&L4xpki-wMaYP1ZlBQ^@Y=03K0ja{vAVP{vsiZX+6t!zqK*p0&?o`chvRz3A z4s1L0ZHb_-r}t;qEFYTDPGO{6eBQs0jE}7R%`fc3)8BiFuSGZ9dK0gVbzB@~1UkM` zg#ujqdV0?|ae;lZ^ry>*$3`t0U%mPpM;-RIzMkGl&J|zNuA>ghac@8)S1Bp^Flaoe zhL0uNCPDn4>jhJE(Va1r zu`|ifSYoHZ*g8LFg2+@8V;lbc^gn;1LoXQ0m0$X%IFgFO0Z!rIb~W-TPs!S1jUt#i z9Rf}`ziz|&`G0@(`ImhD{7XJR|L@T}KZnyG^JJqk^@`asimtbvaz; z$qjf^P}~H}n?1)iA08W-bMzrSvv%MAT?aRsP1e-V&}hCW42V>)!n4yQh=EE}K?qVo zsm6Q3aaWHjQ^@=ml@pm-G&LACS)}0$Vpwbu;IBXX&JN8t0Cer#{nwv)SAyx_S%-EG z;Y}YuN+ynhP3f*Q4a5+LI*05UGo4-AGb1!p3SWqJ+Gkgr@~uBOCiElX^#0yCSDn0F zs1BX|*8T-=G5v0L>FViz^C#bKjTP2BEE5-k2nY&`D99wBiX*F}6%p5iHyk4V1r-jZ zhVga0JCKahNrG@H-CcdlQu z^wyh(Cr0{q?wxF5uSja>c1 zYg`ZdTm50Xl8z0DCK+^G&fZS;$kqW&J*XMeWP{8jkFVP_It&1u?~Mx^ycEZ_754GsXbR7lb#7Pn2DJLa*`JUyZd{GvR-!DXE*sx;tog{~W_Gzpyk$|R2prrrl zIH57)NfG&vSx7lC*ovJ>a@7oZ+B|t^H#lL$!Yyr@XcPEHc-;~rkih4xbJ;I!wYmH52LMAe22XqM`*OVI?z{i_ z=lm-G43Cf8Amoz>P`u=;OR{-FM1de;hF>`0d%{xRe zJU+7Yr_1|#dQU%L!MuI<>+S8k`p4I-zVDv-haNHikRyj?3?_Y@apEcS4?O|^R^NBe z>ig~nfYVP{Ff^b}z20nG{iADwNk0Ca@0mM$|Hv1BbsN^N`1Q{N!l$2j>b!mT2ZVZ~ zzT($EU%lpT8#bB_2>|9FcI5m+kKA*{AOH@J4_|-FvQ1+nK(Oe{4=E23*FC;&#jk!2 z&bdVLhh`3*apI{6(AVo6;n2X);*XpceqVI?r2sf@zd6Uh>lhn(UHOCW)|>UC4u9Li z6BhvDrqSVxuDDDM1W{j4@4VS_k3agDU3zy30y;c4dgr}&tzL7tp#llQGCnT;=y^s9 zaK$fIKC_QT$G{L$~}>+3^+-~R5dRrlUC zJmwZ_V|c*K3vB2)a7HqMqu1Uy_Wd=xzGoHyOinbPziq>76V3guIMLSWJ#gTUw`~ZJ z2G2S)g|KAJ$oKCH3eZ?P_UO5{%=^ItJDutau5_X)0ZF2Rq>4<__i)Pc(w$FFPBe`k zCYA^x#1_2v@pfx!001BWNklCUI4vn&*`6(--!i(jvQyIUZvKQJmt|3!cA z?#IneV2v!fcla9Kwgd2o6^~8t@11?waXam;uB)r3d&Yvdj4ZjAM*Cv{Rbe7Q+Cuia$=6Uo>NmSuN5_x5(8jw}6UdM1V1bSi@ zYi@&yd}xWilrGAE1q}ciDIyzr&oA#4e2~lT-Z~7kl|IC^U0#}md5JGc@!CEwXd9vi zfw|Wbz%~d$|EDVw5>P%oDX`pcYV=(2V@XFQNKB=nqN#|S4=X_jePa$uZLYW zvh;i0JC8a21f8(1zVDvZ_uaGd`~NXC5Pjad4UewA@1FUG9JzSWzveJcKOrs|UVPQp zG+kqGUEkNfNyEmrZ98dfJ8f(yjcwa$Y};1jG;VC$Nn^d|_n&z`o;i2!%)MvrwI7|m z3ds0xk_!<9{}h`8C)K)%x`SA7=QrWUKkz`7lhQy8U$^_xb61 z`KJYT8-$RLfRCpBzl*&sm+0!=r}aC=j@^c++PBpLHo>N&zPup6kqcbK0>rJE=zO^4 zofTI52Yl&NJ?9})7-sv$EVysCG}=cmnfBa|6MjA0h4s%R_1iFod0DRketK`ScZm^{ zAf4;kJI)wHefQ^LS^MvpYH@|8FB6gkl$pb!`gWXlpR7|?~>fbCFJIX*CCsJ_ry>vZ4PXOj)u}pz%f(k}zP@4UT)Bk!f z!5oHFv#^R#=LG4={ymw?g5zUYkPGWkW zcD9w8U32BbeZTXD7_PgYw_A(G8f|P1fAMD1D`KQlHn4(J-F`jE-;o||Ib1!hUFr0i zxEswcc5OHK)%IKbLG1(nLmh_;zozfe__)|zr*vDm87%HMZasR{k##^xAR-j8^?ou@ zgsy}fuxNQ-_A=GD^;T|H!Px^1R#VT-si6Yv44g0@@qu2IWCpVO)1#dkr3Tl1n8OR+K6zF)D|~>p^suJ0p9`Amz5iyZ zK@RzdxU)yHkwnS|6os+m%8%`|GAOpp9Q}5{*5SM8rc;Dl$3$Y}cM@v2ZXG6bp5NG- zbK9P{{MZB-C{Q96{M@JX$ZRaNY{n_FvcGku8f-D??1wKm@Q^{!1OOD2^ptG2A5 zYv2(g9tu=7P}qYR{9E-R<9sON@Bfrz=k&HZXu?)O2HE+TMzK74%#MpPC+;1N0Gx6v zN`=46o*@%t#x=O0J`?Hqix2w@6waHw0r|%)6^}X<07xcdE1m04%+?0GYT7P9}KGHN|-$GoXlcws+~z6veeGT zEw?{8dS+deLQLmz-*~JLAdyI051qxWjQY=+`s*&A0{3Qx%Z^^-L%A=H3N8Cty~q0h zaY$g(WrX4p{lX?%arZew%2$MjiGTo4>7``5;$-j>P<=X{{?tbN&GEY|jnrdKhMN+o zt?M4CL6pdp#wBZEP>17zQ~cJf^_d9AXtM`d?0smQYsG%5`F8Dv%t6}y0hn)aj)GW! zl*4*2yng8@-?!AZUL{WJxliCwm0>}EI7JO#o=;Vo$g4e(aJjcTRQs)PPJeY8dpS-7 zT?+gSsRtgA5(jJ#yrE-3eKU`x?yAD;d4&dNbl@5`miz(Y<|VM20j#xl=5!_FJq;|t z(L|2vt%o+-IM7^IQcjkI+||{8H{}$#ZPG1(t>H0_j*D9+P@EiG4oP8@#}nLL*WPV6 z3)gz&!@Q~-u0COu+L-C>;lxB__MtB6&NRrJCja~1?7WwbJ$GVUi7Fk&H*E?1{*tjHnt^%s4G(@D$ zxnO1>?Fto0aAS0BYCl!+Mc0ciQ`>0>S#&Aw;TZC<5PTVfrFYRC6jC#2x(wWD`1JCq zDSp@_ma?x!*q%)e1Du~%K>HM~6bj5>11+w`Yj-xK_j$t`JCcE zumk`d!tn1JAb(&xa9HQl>Hm39>sSA8(e&PI@84riBKzlcKT=J{=Vbxgs3w@`_g2Gq zK&2(ZP2<@r5&$zesI{-(IGr3m<#2m^_($lnoG@mkPjBG_&Wp1Stc`aDF7Un^$==fGA{KzlYSgyV!R6|t zb5(9fPjg!e?-B#Q>i*MXpyXy2kbVXC$RAGw<9tp>TIs#opIHYB|JC5TJiX1}gB)`p z^j{qtm1~zCv(gb4*O2_y7zKRp%{D?U#47G$n)i~XIWF|DLC2R@2*W;y-@_j8x%U=g znBE>M%yE%olTd7;Ip%y4Yrj2%`_FPM)-UB)|BzhX*os1}2}(1th6!)L=&=;oaH`>@ zxpMaOyV}g<^8_IO`+KWK!tCFZO>z0NlN_goi07KrllN{fzsn6D=*jh`13qkz^l4J$ ze*{&&ZsTIccS+OGSqT=U=g0eWir;b}3>E{lE``av9lzE+9)13L+qBHxNF2Li+iG)i z)Cn>(m!71j-kcV)85Alt><9?((92qKKoEOmM%1!9GhxIOMnQ2TWbM#vl>9p#PR+XE zHZ5g3>$J<2Qv6_G7tUd&MP>!-WfEs_eXR>QZb#u53HzhG4MHvaT~xnb-^ot)yAr$z zzD(Lu-ClANbJqQHT~>zGv>ERuv(KSMA)EK}4W#~S|C?xRSw>9=Wbc1U)AiFRP-Pwa zd|L)E+^pXmFOLLa<9YiR9=95g?{e!D4~aYnub(XR744a`0Ste~_Hj_#mf1boAvPp8IekA=9@;Nsu? z@ia+KTUCua^$AFRhkKQmop>Nj2L9K|s9u{bfbRhv3E1&LGdEYBiklD1kGV~jHV|N; zP*C=-zZongxXP5#xgoxj6JrzN@tEI?fVO7`2p@9EeJua3b&IT5UxB*irM1@MFGL;3x8o4wMLGZ`!SZ=! zh2Vbt^65Qh-jmq#{!omi>x+YY!Q*tCllxvWT`P$=iLdkMP7u5j8fXxm!~NL;fMXVY z`sg+S_8GSsq0hiClvKE6OPh~xwsxy(RuoQ9nK?m%raa4+I1at=#wjPPC0^f3TLo0P z_vTwceOI54R?;yFcgM~4J+BUpt(;W3?}xO*Bt{%Q61f{Y4Lt6ii+7el)Y;ULbphn> z^UcpWb1ojQ_icPEqJ)Q}qS#$va`ABgnxogt_h3&uGs$e%cG=8n!22^(nXgldGbT?x z17)9(h3Q0;!v{W>%0!B|{1-C^x*>VbJV|?=8|@x4eta+Tj83?U zlO+i&whO!PUC2}0cmBKJemKRX!kT5#9g;CMMh6Sq4VQJXiyZZ3^Aqu@_wV%F*Yco_))TD|-GJEr&k>nCPVm6s8L{_~=x$xMyOhB%WH z2txDyDWlTYU0%g-I^g#Vz2b6?)9(?^n3&*iK-ZaS-!`T&S?7TBj7E}ADMAt9&b4UC z>15^Syv_0BHUx^}qf>8-o%eQhFAfo5g8%dct(@7wqipLdjzAY*Pg8DA_9HiS{7 z3h<|2d}cYEYP&8Qn;XOri3S6Zb8nn5L+c@4&(A8u?eQ^fLkk2tnY42IHrWJ*+*LDT zI}G+cf?YH7o`8SZ?Rui(bE2A>Af+bf5;!c>-b~l?E!OLbvR`CSdX{?!8FhcOGkaxl zXv=*I+cf8|;`ev^FM~)nQIq5$Um93R;8IN8)t5|SAslN0C;1P2%n5GMhd4OMk0DZR z;Mg@CV7^$hx10NVI;xMA!3$dJwC4v|cHhYkp~t5AZe= z1b2M?qd-xD_d^&5Q`O+hR4*cq)1z%FwUa^da1pxgT14w?6Ql-lD*6nX>R!abpe!f@=5co7fgHNnL}}~ybu3Z#)U=A7II{fpTuHrd6g;Z zK-am|zpc31PKVa>dzo1YbNV?RZ!`ko*evlo2Rgti|D7?kAb~q*zx51OIOOY!I*kAe zi~o8tB6DGhD{bcDckh)Tls2LoOBE@F;}yaI0Vjw&q8Ji?_Y=-G^5KV+e&BJvamZ{jje!JNQ+U;PEr8?_{Qxz)_6F zl$YK3I*aX_` zgbx245AudK-EJyP&vLY`F#EvHVqV5$LdVf${+0hFHYV8m#ar~mLZR%b4=nsP81Q^1 zK6(~@&Z;oC0Ay;OeQ1#1%9^JA{F=B0NL_{r#r$J2(tdZ;Rtga`2|WT}gxQd~@qjO# z1bo)5#&gEE`RA_dRf3x>`8V6xUuPcB3dmk%yl=nizB&wL1b!od)zZy z6CY_q7%{MOTeR`LecnzYSL`w6&rbOmiq@;WRz-(r)$qI@<@sb8TXOo+hxbw2>pMra zX$QPuX5QP}<68C6M~!TN(7zMTJut6&Fed(l8nWpB;{>;QAft;`gPX=0wwP0Y(`+57 zaId=t$uEz3Akf<;Eh8I3^n~;>d8c>MvVA)!Mxavs=-S*%xOv>9RDV&T15_C(_Sj0m z`iCfmWTm{OsA85YCj&q7=v_Wa=5ml3G7D#a6-AxsII*z{>a>H3P}lS9)=BU);!CX0 z*&5cHtz#oK2>&s*H)ihlEoWBKzl-(G-PO#Rms>8ro4~#?g92K{F-9_n8`~A>5d8~6 z3_Z;P6|gVWpx}II^WY9bTdL-jR~2)TX_e(u+N&?z;GpH=);eNxA_;NZgFglnFPLKr z)mq?dSu%`G=u`3EjIF>|OX6^a_I}q7uR4IRyIm($u|BMB+vgX1KQ8RSK@by3Nq}>4 z?HKKh1d6wzU3uGwOGX3(Q||Joq7Y~Z>CUT+r`HY*5#l5ym179MTXHsji*oyB`o4VS8`ta-we&Pvb>XGtfwJdNU#AqbSJrHa}soFVILr;GiiAnq4!Z>zW(627l*^{dEW_ zqaYH9qF$~fx;XLbR2@#U=5t~ugt8@%yb(m#$G;%s+jF1h?!7z5V4Lie_^j%ea~fmc z1M??nh&Nx8SB~Cej%z_`ZBRchQ)TsQ&Ft11D{KmIDX!7^&nm1=leO3N4brFg-j9&P z^4+bXGi*Zx9R;*~Ad!)HadAS{ z=zt5lldmRQ!B0ql`l+2$SyyS&+&pf;z1Dp9&xi3FRP=hR@8`FVe+K8r03~BzjGgYh zOy0kf%UAGyF)Hk+MXe*i)o%Tiz7sR&woihn!#`(>!lgEu%Y8g$z~xu~ZAJpB0PH^Z0aZG{k}p zgMtEN7_vF!`2WO2esLP8fgEa?5Bak~OBgkgjX@=d@H!gpvP>>7zm@$!5JdIc%+Qr& znA6O9>!#$7A>JyXDUibe3cN|JU_4esA?&jEUos{x!DkUiq3e6e?&n9o+SKNOnLWFa z60Z1nfKQ1iQCV|ZHCVI$bM8Y@JX>wo4*wYVgHR}Y$q^~(f7v4sT3*abhCRuuRLI!e zqB?yGONj$yTdvu&lNDugUq%14^EDu4BR_}=c8U~D0gblxZ>+WU^Lo;cm66lmTUYV1 z&vFU{$YF@A?aRD;ZLF>fcwV%)rSxbFn$w}Ln4+EM;NxcV;h1B!JJG-CAbP$gF7h16 z`*RH*I5rk;KW?9@5rzwuAs5!cXsm6a?p3A7&aZ&B5)O;V8J3*0X{+h{Fy>24iVohH zoB)fpCeqSt^HZya{5TJXEIl1|t7XLb2wJJUq1@qXL?>e_#iPOf`{Qy2V9D>!tyEhE zv6%r5L~pYquYjo{$Q!>uwN~%F`Bx^wZ_5h%?@tW^-sfj0F%rR9;r9Bo=_0T=;NZ+y zF2fB^WJfrH&_i@{St{cG7W*Ceo$hEeu?6uTfQ|8Dqk3FU`D0K3`ay1Lwf2XTCSFYW zO{sU3J%Wg-EuE2^zqY;y!aWu7n!z1JQDblY?Q^N*p-vQpDx}jqGL^t02$77x*r2#Iel$q00nlc z`NqQ|c>{>>E`-|PeSjrJWsK5ESwCoF!^{sD&}>lDCAKkm)CH!YeR(hW6O0HismTX<2J za&cn{xIvB^duLz)+}ri^9Y!u5=HGI@h;F@~9$@MDo?I_q16Qq| z$Eij4_rTm$Wa`TS(=)V(&Qb`Hk$&-sDkm<>LaFh)R>5y~x&Lk(YqC3$+U$Mu4&rVM^9j&_iI9U&Us*YD{Ls{y-4KsA;=J|0rR zomoqg)bybc^Ty0hU})!Z(-~vPni1!)D!Gts-dSFg7fSTqua~OD3u$~!sF#kFTjnX) z@csiL<_R?@4N_5EGKe-dvD11scNrQTATVUzqX2)F>b`tL_2nZoW?>NKybAxM zJfSGh;P^L0DPGFFTBAY=jq!x5b|J@I|Ce>R>oR$^|9`O>0Mf`OCjA$`d0#rM{iuJ1 zn!A1~o2;G6ZvLz|_(THHzpVGtzn{nt7@v4VV=R4)EG#j3w13ZSz4~xua*bU>rx3vV z^87na{PX8Sfgd*>D0H*qYJ8HHm(op1t!QmX8Kw>hW>hY%3w$!SB6eICsbb}*^;}2q zWad`49fyXKuN_rn433uLIz|T+Lg_qRj3>%kVdi+B_9rOo5IWIk~Epdd?HkrS!FPdFK;TU`GXZ0g3VGowy(*Mf~dm za`;6U; z23s##iP`Jr(bLyIsteCy1eOn>==se)0USwdEJGF(JXqfJg2BWGFWPqN$-U}0u_Sg+ zsZ8KT^?Y@^aYL{*-{GiOkjeJQyj7#=+Oev*@J8tQ18C*){}FLb7*@+?Ef1$eas4s} zyAbSVZ9nm=t+CK#^iI^h_uXIRcuVLlA3tuk{wIRtPtZvB2Ux%`C4j_(H>JNqvj5Z6 z;)!6wfSbFmdD3vC=(+h3{8(spWglRg<4Dm3=kEwZKi?u0tKo^FuIk{K+8>8LJ%4Cs zK%8CB3@oj8K93z`Z=(j+y1JwLgexX`*KOTQp%Eey)nMb~@y8P!>~MgQ&RZF3VD^?9 zfr@8!isO(mTsDlel4+q%`(vn#QrLeN%cqNJTD0vFBo~9e*GMANl->GlfjNqx0TLv0`!<~w{Aq<`eDF~N>lx(Q@5TR z{;ro1*~m%L(&Js(9;Jhe&=JY0-u=65=MCxhrwVkgVUvo=j`1JUyQ2F>St={+oQ~(O zinK5N>W>>fHyDtDrBOmU3L%#>CW@;|}NpYntOxm?9b zz{T2%i>lx1Qm!hIo0!>zfbLxD8Rw!1O*V}q{mJ-H+M?+wJL;Yl*>V*+q~fVU5?wF; z(~H4IdhVMZa~#+CK3qRYB=2Gvj9GB)9?CWx;{ItiF3H29>prywjTKb zf6-AO`%CH*-s8UQKe=@um5Iisb#CeWr~USPq?!WSaVLK4Xsp#;gE+$=ZpsqdBs_ZG z$kmUqr@tb*>OHtdQOAGIdRljXwdw=5bFANE{TT96W(-~J>t)K4&*OoFE~hJ2wXzam z#`_y7!_kA02R{|{*hu&|EU+=vc$&2un_u{zR~hlLXVGVAeOxad7Ac#hJp3mGUG+Ku zcad^jw0vneK>(u>aM>Z)ak_xgtf(cxYj=(DF0wQk%K$uQ*aSco+}UBXcYj~o;s-?E zeYVlz)&H`oJzcKolj_snZakrfkGakpK;R()M5ze3!GD{!Vm_5O#^fx5sbeIw{;x^qsJy9D0tU z(WTTPqe@N0(4Fsu>x88|;UUGGi|ylX)Dmw^S5gWL>1o~P2=J3aKY$VeY6yKrc5ZzA z`ztgVA3rD12x!AYl+VsiviYnL!Ps$Qc5Zfl;UScX#gp=+9@2cDNDGbym2KJ}#@Qwo z;+ld&gnq+dfg`Wf6Vo-U-0~m`BRtm(fN%mV-fIbe5*>CQ3*EM>eL*H#kf0=cGhg+qzYi`#=eRU zu4wYsoJQokAQ0TEGwot|)^V~flv~}*^=sWwdYS3R)AKof*5RR#g_&XX3%y_{sU6P6 z)&5uboAmfOEwW6zdfPHi%gK~y#E>F2pq_oSg~%V&X%#ND_G7-F%5VW2JIjWGN3v8y zWzw$4YaLWq8^0&T=DY}Dfo5>;NQ?H1`~VZr8G6N<&lybUthqY=IvfP*z$YW%gA{l3 zjF{+!wKawMQL{2i47pN6u8~Lf32nv?!;hK*nib> z^tvq;^%5u03=$!Z$5)~_Fk=5-nK3|kMSeTH<~lIcELr#5mS#3r&7Sl2zTZkhGbRX% zIO5t`M+yWI{SpMH$eR)#SP2w_z=iQAwhsw7GQuRj9OC?4Vtym`Z|aehL~NR{AW8lz z4;ku4$l-%QN>Wz{bcJDoTkfq)nXb=eb7P!)DgWUqktCMv8gS@U6NofW*Mh`l2=fUV zyf*}~JJ6U75B~uPa4LSZM*9POo;h~|Kw?WvA)Bv$V7(U~fCk<%NOO^No`|P{0o66P z4;?BVprH;oEoAGr`hlR2;|HnMgR7rzU^J7jn;Gfrw5nZo~K5`v7aA?l6t#> zI}o7C%Wk+p;qvymUVKQAtl)we*ai!91QD3sNu_m-D*B}yaZcD)B{vsyjp1cXaCMOJ zLQy&|r?Bj`ON(G5^>F437F3~aZ5JocEW2dC)1)cNo=I}^`k^O-J^MNL=>PQQ)!$tu#Jk{rivtA+^Q z=lsj*?Y$&TY6s+>FwcHMP!OwJrC>M^9CAhBU@AX;^O>^PA{}^TYEPIhZo`)1 z<@n#XJP+F2GVJY+8)@u)slBM`>nH_wR;_o%autPk^NC;?M#pT=wNm+&m36pfEo)MB zQVpBsP^r&^9j8$&2l_l^;Pa*hx=88aE3t+k!KM=>MQn6eMMegcgidv_5B;_R?6 z{jFNQBx<5(fCcjw4rH&#oJzsWlU1nlBka>zvP2*H);;5nTAJMLF{Q;JB32J4Xf;0a%s>RvMO!Vr-)50n0Ab;x8J>bS=WNz=Au z^G(p_j5bz{lL{eNH{;scrifk#^FElFw5Qc)9WWC`>41}SOJQ4eH#-a+QWQtsY~uNQz^^Jit(qCzgDbWw{Ntm-jeZzOXH}@GTT_?!VmE0?T5YcH4pr? z7w5ZF)Q3^tb%Ckngx`SfnhASSDetV%1a1Yn{Gc&D(FcE!SN>KS5qO2#1)+`khZ<;`nOQ z^#~T=;JfD*2u1Q6Ki@m^WPLQtdP9yJ=Fx0icUhLjKAY}F{1mY=eg;orM?K_XI9m#_ zaLv>JHLa<>Vza@Ed1_iP> zkndv%b0VP*(GcI{inJaiEKR48|4dG9qUUmW;b+El*=9Y<`p4v&L$}gJt%!`{81jco zS4zlufIh+Rp_rDx1{G{;=z~3OA;bi8EP<{p}`V6dE5GG9T0}#ubR39t=a?rcygHb&Wc9h5E(^p)i8vt{$rp#rO=YH__3td>es67d?!* zKS4L_jU~Sl>`E@m2{M|*pdtr+vHGAgB6T4%+{)Yv&ut8|4L+FR<~fgfR-M7-d3z8) zA3*5N2=yOFLnVK0I_{0%ceu~Q3+`K}_(S!v@`4x?2t3I~xX>#l%?4_6weQyD7&L|< zsJ1iYO-CrN66TYNuLonk2jU05#5qG>}b02|ZY+2v0!rPi4KWVUT zNI?5I5f2ZgXvutLe&Eeb0u4HBvj|6!e5dIp*klWS`dT6r;Qrw39K1B};Pse{8LaaV zO2rHk8lxyewJOOzCEk8+FX`bw=}0PDJ_(cV$#}hzYQajQ2wT}^NORGAlt=CiB!9f?`Vt{uxQFg&2RO4$T%UqGcqeq?cM>@2e&eCw& zcodp&p7XjGG1i zY!Iko{6T=%wNdF9ws~TS`3s%*^8{1@uEh&h1xo@7w&d%I_5V?8i-As`!m*RiwIY$n zHrNc6jx40BjAy;?FU_g2Ocq}UQPvOFK+PP@$vs$v0iIWRaifro7L~$Fi$>7J$P){0O#P^jqmT>6t+~xyv-cBT9F)O9k*Uhex8nQsSJ_N5d$%EJ$wotG36em z%K(hQMA@i@vptx!Y90(F?h!OtibZxII4K5esSOTjmmxYi(KK1|+X3c)y3tl>UVISD z*&VRU%VSLbJU$*3P=PU{f15FZU#R^UUec&a$V`(-b4O%>k3cKwl8uV2Cb%GYF(dsw zA^fMTA^54^g-xFQVXzHjg(-!52ZHx2v<)O4I(kC6gyy?6vv?7)^Tezxo=l8oQ8X94 z|576yg^Hrh=lmD&pLz+T+VnZnglTdvqHMU|geg_zl;xE0u%`SZn6RC*283dn=VAVQ z6>*mXc~6M`k|~~t-#HK&BeG!!4F-+n#S=x-_=8zk_39QI3AbomCLGj)Sy(Hlo+xz6 z_vec&xRM`Q3=2EXWhsC;Z?7fNKHGuxHSsA;#@QBTlFW-Fc#_IQ!qVqx7bK0T-)by1 zVV6squ-U#RI+-aoaf3VK%+^td@8eRg3)Q5S@ttY?)ZF1x-1wL1%gN87q@Eh{+qEjX z1`1>MwbyApjhh&?_4KXSy1(ellWoDe!k0_27x{c=2vAR{1@3*k{ocg)B)rEa4Mu^J)8cX_~@m*3;G^Vp*pZ2b~M&SEFr zrBx$NpY++O;*(JqupYljj&&d~Q7K=}{m#RcWnY7C+PnCv%3J-UU1_KHeS%EJq7-d* z%+jqRIA5oADPpyV7+hqXa?wGUAT!*V9U*kX*7;93R-Nz^gDoZ@_A+?LtzOxn1*o=c zzw*V7B*-9mN`)=u)8L_Q79MkMdp4o5^zBz9j7L|2g)4|o!~TrfRvbx4@KMx^nQmkp z5+mBua2f|RF3~hgJ1TA>^`0xPh(4@^tl@5AyF(8l1mB3V?pKlS9ZtF@r6de1Vtu27 zMzm)-Gfcx>6919<84)Y@R>=>3GY{KtRJ7it&ERZO^)PrWgIM^Cy<)W4sq?db$7 zuJ)&}TiBS@B`8|6BC$N)m*xeGT`v47&3T+aG!{}!Lhi@5-U?AY z!51#PrUDX+sLC5x)<-B`L!nW3QS<*x zwZ|kIiL5r`jQx<)T2LOFr95HGiLm>6j0wT}LqQBo7?hXl>!uXb(i{dZRUkdAUCl>_ z-;%8MMeQHTyAuapL@;&3qIn($f|9%@BusW;SYhfQW*mYxp_zqI1pOG|XYh$2~mMCxX_`39`PD7`X0$-*JS z36B#m7k7c}4%Z<=VbwwA1=E;bLUwvx0>Q$_$~&B2i&I8dk;O?l!%(B>OUi_WncLK> zOK#Lez~Q<*A(b^^f;VN}Sip;+kk#VhaYxcaFsDjq4;&j+2Xmn0-nuzyJ&0>@l6B=s> zmt^r=vTd|^o}_WYU}b{kfg&mq6rP0OE2m4HcV~8wsF5H?*iX4E{-h;h3&*UW9N(z& zYnMrv+Ig>hu;UjDb85jq0g)x^2E<=&x#bt-Q+bUKuo@}8v+eFDxD8teo4?_m{&rR$ z?;PoB_Dy>>(i9jNjuR0u^%oR%1`gNl%q!LRm&4KzfXnb%gu!#O2>DR*EtS&Zic7mh zC7wmSefiwr`q=t;B>1xDC=)LVtIpp4mp5KdFo7ga_^ukrQaB=_DyrMoiePRJ!+6w1NogFCE85t0z1JTEer9>63&)E#%6>C2l3 zr6fnmm#HM(Wk|ESGLkdckZoG!`5dIq>>9o9l2}&0cN~(OQw+uEm%G^{oTVB!I8#J= zn4w4zRIT&NpbJH(Vl1iI8D|MWVLEc>!?ID5X$W-_a;TNRFxxv)K^5y<_PuL0IjtEB zDB_T&%Rqb}RaqtHCD4<@$}Cu28K&BpZTkCFYJ5NE&45#R5dD-O<9ZOC275{_qX?YC zOy6}U4&P~NkA-itwD?4G3TUz9>oI~S7K$m$hS4alf)%mI&03}SV3+ILnz-ffx18@U zAoCga{Dm&hl_er1W`;9#uB6!XW%iEXE(y-E z)}r3j8X4gSdx+68MACW*4&R^;dDz!d6$BN&Q3&48(eg|PuK@_xSx>Y$A2P?3NBqtC z-B)k)My>vju{r5RbxNGDqT^(YQYXa+;4N+{DdcWv$Z?_QgS%{-BtAW z)E{oM44)E3RCPy=uv?3#yj)q;B(3k@NcA(}-wU;pGAC`I(B_6Xg;^Mn_h1eAAu}}C z@m2Z6>7%v!mGv`&u+5UwGlhGIh>;vpidt^$=gBh-?@hII()J;g@^NsGxceW?J(XzI zR^380Vmb7Nv{rkgiHhAml_YE5bWu~R^Tc$MheFAL(BbuCjXOD}wK@Ft((k-nIU&d_ zgtEU(;7Y<(=nAD-<2QelqnAUg{#kg#(fe64>dVm|_?L5$ocu3S`eN4e6f&E&_BSLY zQyXet3|77Inl9=ZYRW0ebdw5)Z}J-y#)hBOjO+I3o-z!~WzQ*L7_$!e7UYq7@>|Gc z(zuUNVx!Djgi`fB>16>79AH0p@k0n~jg%0{W|X8APFeaj&%Sd+^Mj( zR?uO1&rZUqXYtNa7s&V=Ru9Rv52n&{{gm3^^I| zo!clS?Qd>kG1u>bQ^-42GqLYkJ1j=Th+=EeC9X-}iM^=pHRm{Oom#(^14||geaHQz z=d{KkG=_N4UNo%0mwl1N!-mFCdqaMI1+hX07NoC;zXft_k!D#Ya2@b!Ko%RD90~C;B!n(;aVSY(Qr*K+ZA9g%MfYx%qk9V4xdw-wmDjx zAf=%`SYE+b*f%PAiLT!RW`GZ#KaNk^xp&FNSrMz>el#us*QjIy^P8U1bFNnP?AQRp z2cpapL>73`3L(Z@9vYs?b@MF)nat9qHuO(UBxwOvdxC%DR-FM3TMFG5HdZZ`WgJX# zgtD}^V?8LqWE!W5k5aFaSIP{rIH)UBLC#JV)!3g4u!8omQYcLmGdrdcHpXFC`lFox z2X0JNp9Kn9B-+D1ly*!W9HlAI`W;PnMAj{fVs;#=93u?xz))^doDGlOQzW{LmM4FN z4jv}t%4z5+3<0(`rs^MVaHxg()N(}|vhY8AT>>AeKz~DM>h(Y}+u1@f*wC%x zQ#L>6r(6o>2514cP*U>|%;VW|m}) zFZg^mEe^Duc^N3B$?KMMs)?+IeJTUkNhHfwifT5_0W9N5=PBa;Lqm%jO>E*y^#gEGb9gN}6XGu1;^4X&1e_h2r(*A{a*qLA!ISdg5l>siNKvDs*x zW4<*D{ymf$8GETQKHgw@69!n$Dy&b7l>_%ScYuX z)CsDWST_DDjwVd_3MiSZ0GV)WHF9>rT}CK=$x8Brv_;2l(+;7_>NH?Ld0Pv~O-O_x zOG9nqQF6JDvOF z$fFkJoMU4TVVum~Y;2DW>9$d@L9${?#V5RXjVa=-I?#Ko5mz)`&pJB0TY0H%2x0+e zJ{DB-upb5~id_0llVzS~l~Sc-qjz$gzaTVFJk%rbU?DOQYdGm(K&YdQ{J}d?(n?+< zMzWy?4g`T))R!p4Zik^WR2VBpPwPU=)rs+S$=B+vzzHavCUn9kzx4Z-b(5T9`keSm z7FW;Lo5r@!W94AQR?Dv?+h`@U1_!?nG{9*ZUgW=>1Aq3vhb+l42AvHGlrlZFwO55uM*n@a6ON-|t7CGqM? zE2jT+eE#-_Rb%p>j<~;}nAt7$t0r|uLD{UtqCOr|(_JPQn)DDBmU&Vl%^P}4H^c6r zNEAf2$G0utilwO-P#LYdo9UNCdd6dPCLS=HB|a8Wu8;ons+s4F zxP{Ansp*>IFW6=IY!2CRE^T>VM+b)_T*_`31`MTthf-lrhF-#%t{XFT{XrF9x2abn zcQ+2P%R+&0q==djlK+dlS85K;FpXAcb-B!<9-i1NQ8e9v3p#OD^8V!z=Ud!BCSMOT z*n>crY%qAplmP4arqCC4LIQKhFByd$TEs}d1I}R_zrjZr+CyIZr;;0!l@x_%*Xw5x zM_Gt$KI}ioRtT&FC3=3!*HI%wB&CjSDvpoC(2KE&3Sxv;d?Na=QQ~tJH7yEE8Ydht zy&Lm-$-PjNeNG(Ew)VBU!qY#RhR3~cM&xR;os%dmNG5tlrHw`=tDKBIbREhx=?ff; zcs!jEO33?iPQulOUS2<9=#6ncnG`-+#$i+GREHBR9wF=pA2@^Ik(6_7p=BJj!$~+< zupEze7G74A)o1w3^Abu z`U8}li#p@i{^Te!eL?$FE|$f;}Np**WqxH zyDU{F1hFk!wM@TFlgPbjGA5sAIFIE*66!pJz+&Qx=_05wc%iFN2BAhGt7nrr4|(Q^ zp(py4f+x=_7^OqVnYmPxDI)TwcHRgD+aFgtYploHnQJSFs*BJgTs~n*5~@;YMiZi% z_Ag=5p-^3j*tQ}p{GsNCjTf9jj&7;96S$UkQH}N^!_&umm8S>cqYmj6QYIx)yWj}% zOpnD^HW6Rr$M@d^E#ptJ>aqWU8I%nI>n4{Ji|)XsGBJSRAQtnR%vVOyy;Eu|3teHj zCdV9w4dBj`V`t}88#ijIF!}Q}(Mmo&3Dw?n(ELCMLCqM$^ci}Jn0 z%7~1Z5tDIQznyqhL8u`poF&XgK(QaJestC)o?FG~MlsU}1HKSDG~!3<;!dp5MJlwR zTbXAm>0bJGIGmpj7SSP>Uz&GKEV+PF7yE~B4#f`7`_jEDj7=h2@&(KDzXbjvB(uIz zg)R|p6A8yDh8{q+&R&*J_x+CeORTmdz8>~+_Q9Rp+I6yVf{c=KXw|4PCOaSB4^5aC zay^zwTdffUg1D3x6ISzKG0;(nGzrMILKs`90jGs&=TtA=6|g+6J_JVas0Ow`R1Ai{ zEo-gq7F=?Xvyk6Zdl)VWtux*f;h0QeX{g%#;TTF#Jq6(#YEM^@#W@UE$|aqW%gvdR zDmR#?nG_{S$KP~29!36Ad6d0M!+bU0gjSR@saRf$t!EvQz8~y}m+PEy>{mx+1b6z@ zf^0Ab|E-vTEBSa64{_8ua_vd^yiWbRsQF%4Ask1yiI8c&ppCKYU&c!IJ#@?Hsf`VB zAl8}^B_!lKnzFdUAsP7M`oVcIO(%gB4m{mLG0FBduJYZH30SiGsu9akyQQ-+!va$Z z#TA8PWp=nuvoPa{-9xaDD+rqly30!kGhnVI+>9@?Gqi4A+Q=swOJSk4(CCZzU0rag z=%7WHflDRau|o&m!g<`b zQQG>X*&i*LaEV7fqrEBi|Gw-Uoc7Qs`P zC0GRo<$O)2SjZzw>0|W823A>&>qkU`=f@R1Zv!lgKdP4ueW^iO;mgvzZZx+m|>o5e2NRI|bTks@}P$47~1FR`^E$WAeGR)b!<2gcq zmcW7qRXyJ=^5g{xX~1YY3!MstLCKJ^wOEI{AebzMAAM$;Nad?ckkWITv1;!L-g&CkSz-d%XukvO{ zPh)O5b8eK@^h@X5YBAM8Cg1K|q8TT>fs8D#(f5_uW(z6fh3W?rVShkU*REHReQ4!B zpcm(eB9uJTx=UO$;;{+==|vNXFU*P9esACp;H2VIS Date: Thu, 10 Sep 2026 00:17:12 +0800 Subject: [PATCH 08/23] ci: upgrade CircleCI config to 2.1 (#5530) --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a61cbb8..4418158a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 jobs: go-version-latest: docker: @@ -16,10 +16,11 @@ jobs: make env fmt git diff --exit-code make build - - run: make alltest + - run: + name: Run all tests + command: make alltest workflows: - version: 2 build_and_test: jobs: - go-version-latest From d20a232996007dfe6ab425abc0a39a3ae9a0889b Mon Sep 17 00:00:00 2001 From: fatedier Date: Thu, 10 Sep 2026 01:28:26 +0800 Subject: [PATCH 09/23] test: strengthen health check coverage (#5528) --- Release.md | 1 + client/health/health.go | 82 +++-- client/health/health_test.go | 556 ++++++++++++++++++++++++++++++++-- test/e2e/v1/features/group.go | 271 ++++++++++++++--- 4 files changed, 824 insertions(+), 86 deletions(-) diff --git a/Release.md b/Release.md index 5199f328..68d7929d 100644 --- a/Release.md +++ b/Release.md @@ -1,3 +1,4 @@ ## Fixes * Fixed VirtualNet route lifecycle issues during reconnect and shutdown, including stale route cleanup, shutdown races, and reconnect backoff overflow. +* Fixed health check failure counts not resetting after a successful check, ensuring `healthCheck.maxFailed` applies to consecutive failures. diff --git a/client/health/health.go b/client/health/health.go index 8cb1122a..cedbba37 100644 --- a/client/health/health.go +++ b/client/health/health.go @@ -30,6 +30,11 @@ import ( var ErrHealthCheckType = errors.New("error health check type") +func newHealthTimer(interval time.Duration) (<-chan time.Time, func()) { + timer := time.NewTimer(interval) + return timer.C, func() { timer.Stop() } +} + type Monitor struct { checkType string interval time.Duration @@ -49,6 +54,9 @@ type Monitor struct { ctx context.Context cancel context.CancelFunc + doneCh chan struct{} + + timerFactory func(time.Duration) (<-chan time.Time, func()) } func NewMonitor(ctx context.Context, cfg v1.HealthCheckConfig, addr string, @@ -91,6 +99,8 @@ func NewMonitor(ctx context.Context, cfg v1.HealthCheckConfig, addr string, statusFailedFn: statusFailedFn, ctx: newctx, cancel: cancel, + doneCh: make(chan struct{}), + timerFactory: newHealthTimer, } } @@ -102,40 +112,66 @@ func (monitor *Monitor) Stop() { monitor.cancel() } +// Done is closed when the worker launched by Start has exited. +func (monitor *Monitor) Done() <-chan struct{} { + return monitor.doneCh +} + func (monitor *Monitor) checkWorker() { - xl := xlog.FromContextSafe(monitor.ctx) + defer close(monitor.doneCh) + for { + if monitor.ctx.Err() != nil { + return + } + doCtx, cancel := context.WithDeadline(monitor.ctx, time.Now().Add(monitor.timeout)) err := monitor.doCheck(doCtx) + cancel() // check if this monitor has been closed - select { - case <-monitor.ctx.Done(): - cancel() + if monitor.ctx.Err() != nil { return - default: - cancel() } + monitor.handleCheckResult(err) - if err == nil { - xl.Tracef("do one health check success") - monitor.failedTimes = 0 - if !monitor.statusOK && monitor.statusNormalFn != nil { - xl.Infof("health check status change to success") - monitor.statusOK = true - monitor.statusNormalFn() - } - } else { - xl.Warnf("do one health check failed: %v", err) - monitor.failedTimes++ - if monitor.statusOK && int(monitor.failedTimes) >= monitor.maxFailedTimes && monitor.statusFailedFn != nil { - xl.Warnf("health check status change to failed") - monitor.statusOK = false - monitor.statusFailedFn() - } + if !monitor.waitForNextCheck() { + return } + } +} - time.Sleep(monitor.interval) +func (monitor *Monitor) handleCheckResult(err error) { + xl := xlog.FromContextSafe(monitor.ctx) + if err == nil { + xl.Tracef("do one health check success") + monitor.failedTimes = 0 + if !monitor.statusOK && monitor.statusNormalFn != nil { + xl.Infof("health check status change to success") + monitor.statusOK = true + monitor.statusNormalFn() + } + return + } + + xl.Warnf("do one health check failed: %v", err) + monitor.failedTimes++ + if monitor.statusOK && int(monitor.failedTimes) >= monitor.maxFailedTimes && monitor.statusFailedFn != nil { + xl.Warnf("health check status change to failed") + monitor.statusOK = false + monitor.statusFailedFn() + } +} + +func (monitor *Monitor) waitForNextCheck() bool { + timerC, stopTimer := monitor.timerFactory(monitor.interval) + defer stopTimer() + + select { + case <-monitor.ctx.Done(): + return false + case <-timerC: + return monitor.ctx.Err() == nil } } diff --git a/client/health/health_test.go b/client/health/health_test.go index f43edd04..80705384 100644 --- a/client/health/health_test.go +++ b/client/health/health_test.go @@ -16,9 +16,12 @@ package health import ( "context" + "errors" + "net" "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -28,38 +31,545 @@ import ( v1 "github.com/fatedier/frp/pkg/config/v1" ) -func TestMonitorResetsFailedTimesAfterSuccess(t *testing.T) { - var checkCount atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - count := checkCount.Add(1) - if count == 1 || count == 2 || count == 4 { - w.WriteHeader(http.StatusServiceUnavailable) +type tcpHealthBackend struct { + listener net.Listener + accepted chan struct{} + done chan struct{} +} + +func newTCPHealthBackend(t *testing.T, addr string, accepted chan struct{}) *tcpHealthBackend { + listener, err := net.Listen("tcp", addr) + require.NoError(t, err) + + backend := &tcpHealthBackend{ + listener: listener, + accepted: accepted, + done: make(chan struct{}), + } + go func() { + defer close(backend.done) + for { + conn, err := backend.listener.Accept() + if err != nil { + return + } + _ = conn.Close() + select { + case backend.accepted <- struct{}{}: + default: + } + } + }() + return backend +} + +func (backend *tcpHealthBackend) Close() { + _ = backend.listener.Close() + <-backend.done +} + +func TestMonitorConsecutiveFailureWindows(t *testing.T) { + tests := []struct { + name string + setup func(*testing.T, func(), func()) (*Monitor, func(bool)) + }{ + { + name: "HTTP", + setup: func(t *testing.T, normalFn, failedFn func()) (*Monitor, func(bool)) { + var healthy atomic.Bool + healthy.Store(true) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if !healthy.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + t.Cleanup(server.Close) + + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{ + Type: "http", + Path: "/health", + TimeoutSeconds: 1, + IntervalSeconds: 1, + MaxFailed: 3, + }, + strings.TrimPrefix(server.URL, "http://"), + normalFn, + failedFn, + ) + return monitor, healthy.Store + }, + }, + { + name: "TCP", + setup: func(t *testing.T, normalFn, failedFn func()) (*Monitor, func(bool)) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + _ = conn.Close() + } + }() + + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{ + Type: "tcp", + TimeoutSeconds: 1, + IntervalSeconds: 1, + MaxFailed: 3, + }, + listener.Addr().String(), + normalFn, + failedFn, + ) + healthyAddr := monitor.addr + return monitor, func(healthy bool) { + if healthy { + monitor.addr = healthyAddr + } else { + monitor.addr = "127.0.0.1:0" + } + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var events []string + monitor, setHealthy := test.setup( + t, + func() { events = append(events, "normal") }, + func() { events = append(events, "failed") }, + ) + t.Cleanup(monitor.Stop) + + runCheck := func(healthy bool) { + t.Helper() + setHealthy(healthy) + ctx, cancel := context.WithTimeout(monitor.ctx, time.Second) + err := monitor.doCheck(ctx) + cancel() + if healthy { + require.NoError(t, err) + } else { + require.Error(t, err) + } + monitor.handleCheckResult(err) + } + + runCheck(true) + require.True(t, monitor.statusOK) + require.Zero(t, monitor.failedTimes) + require.Equal(t, []string{"normal"}, events) + + runCheck(false) + runCheck(false) + require.True(t, monitor.statusOK) + require.Equal(t, uint64(2), monitor.failedTimes) + require.Equal(t, []string{"normal"}, events) + + runCheck(true) + require.True(t, monitor.statusOK) + require.Zero(t, monitor.failedTimes) + require.Equal(t, []string{"normal"}, events) + + runCheck(false) + require.True(t, monitor.statusOK) + require.Equal(t, uint64(1), monitor.failedTimes) + require.Equal(t, []string{"normal"}, events) + + runCheck(false) + require.True(t, monitor.statusOK) + require.Equal(t, uint64(2), monitor.failedTimes) + + runCheck(false) + require.False(t, monitor.statusOK) + require.Equal(t, uint64(3), monitor.failedTimes) + require.Equal(t, []string{"normal", "failed"}, events) + + runCheck(true) + require.True(t, monitor.statusOK) + require.Zero(t, monitor.failedTimes) + require.Equal(t, []string{"normal", "failed", "normal"}, events) + }) + } +} + +type workerStatus struct { + failedTimes uint64 + statusOK bool +} + +type workerInterval struct { + status workerStatus + timer chan time.Time +} + +func observeWorkerIntervals(monitor *Monitor) <-chan workerInterval { + intervals := make(chan workerInterval, 1) + monitor.timerFactory = func(time.Duration) (<-chan time.Time, func()) { + timer := make(chan time.Time, 1) + // The worker takes this snapshot after handleCheckResult. Sending it + // publishes the state to the test; no test goroutine reads live fields. + interval := workerInterval{ + status: workerStatus{failedTimes: monitor.failedTimes, statusOK: monitor.statusOK}, + timer: timer, + } + select { + case intervals <- interval: + case <-monitor.ctx.Done(): + } + return timer, func() {} + } + return intervals +} + +func awaitWorkerInterval(t *testing.T, intervals <-chan workerInterval, want workerStatus) chan time.Time { + t.Helper() + select { + case interval := <-intervals: + require.Equal(t, want, interval.status) + return interval.timer + case <-time.After(time.Second): + t.Fatal("health worker did not reach the interval barrier") + return nil + } +} + +func stopMonitorWorker(t *testing.T, monitor *Monitor) { + t.Helper() + monitor.Stop() + select { + case <-monitor.Done(): + case <-time.After(time.Second): + t.Error("health worker did not exit after Stop") + } +} + +func TestMonitorWorkerProcessesResults(t *testing.T) { + requestReady := make(chan struct{}, 1) + responses := make(chan int) + requestCanceled := make(chan struct{}) + var cancelOnce sync.Once + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case requestReady <- struct{}{}: + case <-r.Context().Done(): return } - w.WriteHeader(http.StatusOK) + select { + case code := <-responses: + w.WriteHeader(code) + case <-r.Context().Done(): + cancelOnce.Do(func() { close(requestCanceled) }) + } })) - defer server.Close() + t.Cleanup(server.Close) - var failedCount atomic.Int32 + events := make(chan string, 3) monitor := NewMonitor( context.Background(), v1.HealthCheckConfig{ - Type: "http", - Path: "/health", - TimeoutSeconds: 1, - IntervalSeconds: 1, - MaxFailed: 3, + Type: "http", + Path: "/health", + TimeoutSeconds: 5, + MaxFailed: 3, }, strings.TrimPrefix(server.URL, "http://"), - func() {}, - func() { failedCount.Add(1) }, + func() { events <- "normal" }, + func() { events <- "failed" }, ) - monitor.interval = 10 * time.Millisecond - monitor.Start() - defer monitor.Stop() + intervals := observeWorkerIntervals(monitor) + t.Cleanup(func() { stopMonitorWorker(t, monitor) }) - require.Eventually(t, func() bool { - return checkCount.Load() >= 5 - }, time.Second, 10*time.Millisecond) - require.Equal(t, int32(0), failedCount.Load()) + awaitRequest := func() { + t.Helper() + select { + case <-requestReady: + case <-time.After(time.Second): + t.Fatal("health check request did not start") + } + } + awaitEvent := func(want string) { + t.Helper() + select { + case got := <-events: + require.Equal(t, want, got) + case <-time.After(time.Second): + t.Fatalf("health check callback %q was not called", want) + } + } + checkResult := func(code int, want workerStatus, event string) { + t.Helper() + awaitRequest() + select { + case responses <- code: + case <-time.After(time.Second): + t.Fatal("health check handler did not accept the response") + } + timer := awaitWorkerInterval(t, intervals, want) + if event != "" { + awaitEvent(event) + } + // The worker has finished this result and cannot emit another + // callback until the test releases the next check. + select { + case got := <-events: + t.Fatalf("unexpected health check callback %q", got) + default: + } + timer <- time.Now() + } + + monitor.Start() + checkResult(http.StatusOK, workerStatus{statusOK: true}, "normal") + checkResult(http.StatusServiceUnavailable, workerStatus{failedTimes: 1, statusOK: true}, "") + checkResult(http.StatusServiceUnavailable, workerStatus{failedTimes: 2, statusOK: true}, "") + checkResult(http.StatusOK, workerStatus{statusOK: true}, "") + checkResult(http.StatusServiceUnavailable, workerStatus{failedTimes: 1, statusOK: true}, "") + checkResult(http.StatusServiceUnavailable, workerStatus{failedTimes: 2, statusOK: true}, "") + checkResult(http.StatusServiceUnavailable, workerStatus{failedTimes: 3}, "failed") + checkResult(http.StatusOK, workerStatus{statusOK: true}, "normal") + + awaitRequest() + stopMonitorWorker(t, monitor) + select { + case <-requestCanceled: + case <-time.After(time.Second): + t.Fatal("health check request context was not canceled") + } +} + +func TestMonitorTCPWorkerProcessesResults(t *testing.T) { + initialAccepted := make(chan struct{}, 1) + initialBackend := newTCPHealthBackend(t, "127.0.0.1:0", initialAccepted) + addr := initialBackend.listener.Addr().String() + recoveryAccepted := make(chan struct{}, 1) + + normalCallbacks := make(chan workerStatus, 2) + failedCallbacks := make(chan workerStatus, 2) + var monitor *Monitor + monitor = NewMonitor( + context.Background(), + v1.HealthCheckConfig{ + Type: "tcp", + TimeoutSeconds: 1, + MaxFailed: 3, + }, + addr, + func() { + normalCallbacks <- workerStatus{failedTimes: monitor.failedTimes, statusOK: monitor.statusOK} + }, + func() { + failedCallbacks <- workerStatus{failedTimes: monitor.failedTimes, statusOK: monitor.statusOK} + }, + ) + intervals := observeWorkerIntervals(monitor) + recoveryBackend := (*tcpHealthBackend)(nil) + t.Cleanup(func() { + stopMonitorWorker(t, monitor) + initialBackend.Close() + if recoveryBackend != nil { + recoveryBackend.Close() + } + }) + + awaitStatus := func(ch <-chan workerStatus, want workerStatus, message string) { + t.Helper() + select { + case got := <-ch: + require.Equal(t, want, got) + case <-time.After(time.Second): + t.Fatal(message) + } + } + + monitor.Start() + select { + case <-initialAccepted: + case <-time.After(time.Second): + t.Fatal("TCP health check did not reach the initial backend") + } + awaitStatus(normalCallbacks, workerStatus{failedTimes: 0, statusOK: true}, "TCP worker did not report the initial success") + + initialTimer := awaitWorkerInterval(t, intervals, workerStatus{statusOK: true}) + initialBackend.Close() + initialTimer <- time.Now() + + firstFailureTimer := awaitWorkerInterval(t, intervals, workerStatus{failedTimes: 1, statusOK: true}) + firstFailureTimer <- time.Now() + + secondFailureTimer := awaitWorkerInterval(t, intervals, workerStatus{failedTimes: 2, statusOK: true}) + secondFailureTimer <- time.Now() + + awaitStatus(failedCallbacks, workerStatus{failedTimes: 3, statusOK: false}, "TCP worker did not report the third failed health check") + thirdFailureTimer := awaitWorkerInterval(t, intervals, workerStatus{failedTimes: 3}) + + recoveryBackend = newTCPHealthBackend(t, addr, recoveryAccepted) + thirdFailureTimer <- time.Now() + select { + case <-recoveryAccepted: + case <-time.After(time.Second): + t.Fatal("TCP health check did not reach the recovery backend") + } + awaitStatus(normalCallbacks, workerStatus{failedTimes: 0, statusOK: true}, "TCP worker did not report recovery") + + recoveryTimer := awaitWorkerInterval(t, intervals, workerStatus{statusOK: true}) + recoveryBackend.Close() + recoveryTimer <- time.Now() + + firstRecoveryFailureTimer := awaitWorkerInterval(t, intervals, workerStatus{failedTimes: 1, statusOK: true}) + firstRecoveryFailureTimer <- time.Now() + + secondRecoveryFailureTimer := awaitWorkerInterval(t, intervals, workerStatus{failedTimes: 2, statusOK: true}) + secondRecoveryFailureTimer <- time.Now() + + awaitStatus(failedCallbacks, workerStatus{failedTimes: 3, statusOK: false}, "TCP worker did not report the third failed health check after recovery") + stopMonitorWorker(t, monitor) +} + +func TestMonitorMaxFailedOne(t *testing.T) { + var events []string + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{Type: "tcp", MaxFailed: 1}, + "", + func() { events = append(events, "normal") }, + func() { events = append(events, "failed") }, + ) + t.Cleanup(monitor.Stop) + + checkErr := errors.New("health check failed") + monitor.handleCheckResult(nil) + monitor.handleCheckResult(checkErr) + require.Equal(t, []string{"normal", "failed"}, events) + require.False(t, monitor.statusOK) + require.Equal(t, uint64(1), monitor.failedTimes) + + monitor.handleCheckResult(checkErr) + require.Equal(t, []string{"normal", "failed"}, events) + + monitor.handleCheckResult(nil) + monitor.handleCheckResult(checkErr) + require.Equal(t, []string{"normal", "failed", "normal", "failed"}, events) +} + +func TestMonitorStopCancelsWork(t *testing.T) { + t.Run("in-flight check", func(t *testing.T) { + requestStarted := make(chan struct{}) + requestCanceled := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + close(requestStarted) + <-r.Context().Done() + close(requestCanceled) + })) + t.Cleanup(server.Close) + + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{Type: "http", Path: "/health"}, + strings.TrimPrefix(server.URL, "http://"), + func() {}, + func() {}, + ) + t.Cleanup(func() { stopMonitorWorker(t, monitor) }) + monitor.Start() + + select { + case <-requestStarted: + case <-time.After(time.Second): + t.Fatal("health check request did not start") + } + stopMonitorWorker(t, monitor) + select { + case <-requestCanceled: + case <-time.After(time.Second): + t.Fatal("health check request context was not canceled") + } + }) + + t.Run("interval wait", func(t *testing.T) { + monitor := NewMonitor(context.Background(), v1.HealthCheckConfig{Type: "tcp"}, "", nil, nil) + intervals := observeWorkerIntervals(monitor) + t.Cleanup(func() { stopMonitorWorker(t, monitor) }) + monitor.Start() + + // A real worker has completed its first check and installed a timer + // that will never fire. Stop must release the wait and exit the loop. + awaitWorkerInterval(t, intervals, workerStatus{}) + stopMonitorWorker(t, monitor) + }) +} + +type gatedErrContext struct { + context.Context + errCalled chan struct{} + releaseErr chan struct{} +} + +func (ctx *gatedErrContext) Err() error { + select { + case ctx.errCalled <- struct{}{}: + default: + } + <-ctx.releaseErr + return ctx.Context.Err() +} + +func TestMonitorTimerCancellationAfterTimerFires(t *testing.T) { + baseCtx, cancel := context.WithCancel(context.Background()) + ctx := &gatedErrContext{ + Context: baseCtx, + errCalled: make(chan struct{}, 1), + releaseErr: make(chan struct{}), + } + monitor := NewMonitor( + context.Background(), + v1.HealthCheckConfig{Type: "tcp"}, + "", + nil, + nil, + ) + monitor.ctx = ctx + monitor.cancel = cancel + + timerReady := make(chan time.Time, 1) + timerReady <- time.Now() + var timerStopped atomic.Bool + monitor.timerFactory = func(time.Duration) (<-chan time.Time, func()) { + return timerReady, func() { timerStopped.Store(true) } + } + + waitResult := make(chan bool, 1) + go func() { + waitResult <- monitor.waitForNextCheck() + }() + + // ctx.Done is not ready when select runs, so receiving timerReady is the + // only possible branch. Err signals after that receive and blocks until the + // test cancels the context, deterministically exercising the cancellation + // re-check in the timer branch. + select { + case <-ctx.errCalled: + case <-time.After(time.Second): + t.Fatal("timer branch did not re-check the monitor context") + } + cancel() + close(ctx.releaseErr) + + select { + case shouldContinue := <-waitResult: + require.False(t, shouldContinue) + case <-time.After(time.Second): + t.Fatal("timer wait did not observe context cancellation") + } + require.True(t, timerStopped.Load()) } diff --git a/test/e2e/v1/features/group.go b/test/e2e/v1/features/group.go index 85b938a9..87270e7a 100644 --- a/test/e2e/v1/features/group.go +++ b/test/e2e/v1/features/group.go @@ -1,14 +1,20 @@ package features import ( + "context" "crypto/tls" + "encoding/json" "fmt" + "net/http" + "net/url" "strconv" "sync" "time" "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/fatedier/frp/pkg/sdk/client" "github.com/fatedier/frp/pkg/transport" "github.com/fatedier/frp/test/e2e/framework" "github.com/fatedier/frp/test/e2e/framework/consts" @@ -17,6 +23,80 @@ import ( "github.com/fatedier/frp/test/e2e/pkg/request" ) +func waitForProxyStatus(proxyClient *client.Client, proxyName, want string) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + + var lastStatus string + var lastErr error + for { + status, err := proxyClient.GetProxyStatus(ctx, proxyName) + if err == nil { + lastErr = nil + lastStatus = status.Status + if status.Status == want { + return nil + } + } else { + lastErr = err + } + + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("timeout waiting for proxy %q status %q: last error: %w", proxyName, want, lastErr) + } + return fmt.Errorf("timeout waiting for proxy %q status %q: last status %q", proxyName, want, lastStatus) + case <-ticker.C: + } + } +} + +func waitForSignal(signal <-chan struct{}) error { + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + + select { + case <-signal: + return nil + case <-timer.C: + return fmt.Errorf("timeout waiting for backend health-check signal") + } +} + +// Keep this longer than client/proxy.statusCheckInterval (3s). The wrapper's +// health notification is deliberately non-blocking, so the E2E assertion must +// also cover the fallback poll plus scheduling margin while recovery is gated. +const proxyFallbackObservationWindow = 4 * time.Second + +func waitForServerProxyStatus(port int, proxyName, want string, timeout time.Duration) error { + return waitForLifecycleCondition(timeout, func() error { + body, err := getLifecycleEndpoint(port, "/api/proxies/"+url.PathEscape(proxyName)) + if err != nil { + return err + } + var status struct { + Status string `json:"status"` + } + if err := json.Unmarshal([]byte(body), &status); err != nil { + return err + } + if status.Status != want { + return fmt.Errorf("frps proxy %q status %q, want %q", proxyName, status.Status, want) + } + return nil + }) +} + +type httpHealthStage struct { + healthy bool + signal chan<- struct{} + release <-chan struct{} +} + var _ = ginkgo.Describe("[Feature: Group]", func() { f := framework.NewDefaultFramework() @@ -252,7 +332,10 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { ginkgo.Describe("Health Check", func() { ginkgo.It("TCP", func() { - serverConf := consts.DefaultServerConfig + dashboardPort := f.AllocPort() + serverConf := consts.DefaultServerConfig + fmt.Sprintf(` + webServer.port = %d + `, dashboardPort) clientConf := consts.DefaultClientConfig fooPort := f.AllocPort() @@ -260,11 +343,17 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { f.RunServer("", fooServer) barPort := f.AllocPort() - barServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(barPort), streamserver.WithRespContent([]byte("bar"))) + newBarServer := func() *streamserver.Server { + return streamserver.New(streamserver.TCP, streamserver.WithBindPort(barPort), streamserver.WithRespContent([]byte("bar"))) + } + barServer := newBarServer() f.RunServer("", barServer) remotePort := f.AllocPort() + adminPort := f.AllocPort() clientConf += fmt.Sprintf(` + webServer.port = %d + [[proxies]] name = "foo" type = "tcp" @@ -284,11 +373,16 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { loadBalancer.groupKey = "123" healthCheck.type = "tcp" healthCheck.intervalSeconds = 1 - `, fooPort, remotePort, barPort, remotePort) + healthCheck.maxFailed = 3 + `, adminPort, fooPort, remotePort, barPort, remotePort) - _, clientProcesses := f.RunProcesses(serverConf, []string{clientConf}) + f.RunProcesses(serverConf, []string{clientConf}) + proxyClient := f.APIClientForFrpc(adminPort) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "foo", "running")) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running")) - // check foo and bar is ok + // Both requests traverse frps, the load-balancing group, frpc, and the + // corresponding local backend. results := []string{} for range 10 { framework.NewRequestExpect(f).Port(remotePort).Ensure(validateFooBarResponse, func(resp *request.Response) bool { @@ -298,33 +392,38 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { } framework.ExpectContainElements(results, []string{"foo", "bar"}) - // close bar server, check foo is ok - failedCount := clientProcesses[0].CountOutput("[bar] health check failed") - barServer.Close() - framework.ExpectNoError(clientProcesses[0].WaitForOutput("[bar] health check failed", failedCount+1, 5*time.Second)) - for range 10 { - framework.NewRequestExpect(f).Port(remotePort).ExpectResp([]byte("foo")).Ensure() - } + // frps removes the group listener before deleting the proxy from + // its manager. Its offline status is the removal barrier; frpc's + // local check-failed status alone does not acknowledge that work. + for range 2 { + framework.ExpectNoError(barServer.Close()) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "check failed")) + framework.ExpectNoError(waitForServerProxyStatus(dashboardPort, "bar", "offline", 5*time.Second)) + for range 10 { + framework.NewRequestExpect(f).Port(remotePort).ExpectResp([]byte("foo")).Ensure() + } - // resume bar server, check foo and bar is ok - successCount := clientProcesses[0].CountOutput("[bar] health check success") - f.RunServer("", barServer) - framework.ExpectNoError(clientProcesses[0].WaitForOutput("[bar] health check success", successCount+1, 5*time.Second)) - results = []string{} - for range 10 { - framework.NewRequestExpect(f).Port(remotePort).Ensure(validateFooBarResponse, func(resp *request.Response) bool { - results = append(results, string(resp.Content)) - return true - }) + barServer = newBarServer() + f.RunServer("", barServer) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running")) + results = []string{} + for range 10 { + framework.NewRequestExpect(f).Port(remotePort).Ensure(validateFooBarResponse, func(resp *request.Response) bool { + results = append(results, string(resp.Content)) + return true + }) + } + framework.ExpectContainElements(results, []string{"foo", "bar"}) } - framework.ExpectContainElements(results, []string{"foo", "bar"}) }) ginkgo.It("HTTP", func() { vhostPort := f.AllocPort() + dashboardPort := f.AllocPort() serverConf := consts.DefaultServerConfig + fmt.Sprintf(` vhostHTTPPort = %d - `, vhostPort) + webServer.port = %d + `, vhostPort, dashboardPort) clientConf := consts.DefaultClientConfig fooPort := f.AllocPort() @@ -332,10 +431,41 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { f.RunServer("", fooServer) barPort := f.AllocPort() - barServer := newHTTPServer(barPort, "bar") + var stageMu sync.RWMutex + stage := &httpHealthStage{healthy: true, signal: make(chan struct{}, 1)} + barServer := httpserver.New( + httpserver.WithBindPort(barPort), + httpserver.WithHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + stageMu.RLock() + currentStage := stage + stageMu.RUnlock() + + if r.URL.Path == "/healthz" { + if !currentStage.healthy { + w.WriteHeader(http.StatusServiceUnavailable) + } + select { + case currentStage.signal <- struct{}{}: + default: + } + if currentStage.release != nil { + select { + case <-currentStage.release: + case <-r.Context().Done(): + return + } + } + return + } + _, _ = w.Write([]byte("bar")) + })), + ) f.RunServer("", barServer) + adminPort := f.AllocPort() clientConf += fmt.Sprintf(` + webServer.port = %d + [[proxies]] name = "foo" type = "http" @@ -356,12 +486,17 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { loadBalancer.groupKey = "123" healthCheck.type = "http" healthCheck.intervalSeconds = 1 + healthCheck.maxFailed = 3 + healthCheck.timeoutSeconds = 10 healthCheck.path = "/healthz" - `, fooPort, barPort) + `, adminPort, fooPort, barPort) - _, clientProcesses := f.RunProcesses(serverConf, []string{clientConf}) + f.RunProcesses(serverConf, []string{clientConf}) + proxyClient := f.APIClientForFrpc(adminPort) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "foo", "running")) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running")) - // send first HTTP request + // Ordinary requests traverse the real HTTP proxy path. var contents []string framework.NewRequestExpect(f).Port(vhostPort). RequestModify(func(r *request.Request) { @@ -371,8 +506,6 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { contents = append(contents, string(resp.Content)) return true }) - - // send second HTTP request, should be forwarded to another service framework.NewRequestExpect(f).Port(vhostPort). RequestModify(func(r *request.Request) { r.HTTP().HTTPHost("example.com") @@ -381,25 +514,83 @@ var _ = ginkgo.Describe("[Feature: Group]", func() { contents = append(contents, string(resp.Content)) return true }) - framework.ExpectContainElements(contents, []string{"foo", "bar"}) - // check foo and bar is ok + // Every failed health response is gated by the test. This gives each + // stage a request-level barrier and avoids cumulative process logs. + runFailureRecovery := func() { + failureSignals := make(chan struct{}, 1) + release := make(chan struct{}, 1) + stageMu.Lock() + stage = &httpHealthStage{healthy: false, signal: failureSignals, release: release} + stageMu.Unlock() + + framework.ExpectNoError(waitForSignal(failureSignals)) + release <- struct{}{} + framework.ExpectNoError(waitForSignal(failureSignals)) + + recoverySignals := make(chan struct{}, 1) + recoveryRelease := make(chan struct{}) + stageMu.Lock() + stage = &httpHealthStage{healthy: true, signal: recoverySignals, release: recoveryRelease} + stageMu.Unlock() + release <- struct{}{} + + // The next request can only start after the worker consumed the + // second failed response. Hold this success response until the + // assertion finishes, so premature removal cannot be hidden by + // recovery. Consistently also gives the proxy worker time to act + // on an erroneous failure notification without retrying it away. + framework.ExpectNoError(waitForSignal(recoverySignals)) + gomega.Consistently(func() string { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + status, err := proxyClient.GetProxyStatus(ctx, "bar") + if err != nil { + return err.Error() + } + return status.Status + }, proxyFallbackObservationWindow, 25*time.Millisecond).Should(gomega.Equal("running")) + close(recoveryRelease) + } + runFailureRecovery() results := doFooBarHTTPRequest(vhostPort, "example.com") framework.ExpectContainElements(results, []string{"foo", "bar"}) - // close bar server, check foo is ok - failedCount := clientProcesses[0].CountOutput("[bar] health check failed") - barServer.Close() - framework.ExpectNoError(clientProcesses[0].WaitForOutput("[bar] health check failed", failedCount+1, 5*time.Second)) + // Repeat the two-failure window to verify that recovery cleared the + // counter before another failure sequence began. + runFailureRecovery() + results = doFooBarHTTPRequest(vhostPort, "example.com") + framework.ExpectContainElements(results, []string{"foo", "bar"}) + + // Three failed responses reach MaxFailed and remove bar from the + // group; a healthy response then re-registers it. + failureSignals := make(chan struct{}, 1) + release := make(chan struct{}, 1) + stageMu.Lock() + stage = &httpHealthStage{healthy: false, signal: failureSignals, release: release} + stageMu.Unlock() + for range 3 { + framework.ExpectNoError(waitForSignal(failureSignals)) + release <- struct{}{} + } + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "check failed")) + // HTTPProxy.Close unregisters the group route before frps reports + // offline. Only then assert that fresh data-plane requests use foo. + framework.ExpectNoError(waitForServerProxyStatus(dashboardPort, "bar", "offline", 5*time.Second)) results = doFooBarHTTPRequest(vhostPort, "example.com") framework.ExpectContainElements(results, []string{"foo"}) framework.ExpectNotContainElements(results, []string{"bar"}) - // resume bar server, check foo and bar is ok - successCount := clientProcesses[0].CountOutput("[bar] health check success") - f.RunServer("", barServer) - framework.ExpectNoError(clientProcesses[0].WaitForOutput("[bar] health check success", successCount+1, 5*time.Second)) + recoverySignals := make(chan struct{}, 1) + stageMu.Lock() + stage = &httpHealthStage{healthy: true, signal: recoverySignals} + stageMu.Unlock() + // Release any additional failed request already waiting while the + // removal and data-plane assertions were in progress. + close(release) + framework.ExpectNoError(waitForSignal(recoverySignals)) + framework.ExpectNoError(waitForProxyStatus(proxyClient, "bar", "running")) results = doFooBarHTTPRequest(vhostPort, "example.com") framework.ExpectContainElements(results, []string{"foo", "bar"}) }) From d5636108e659ea6ba177aa347ffec008bb10fe3e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 8 Oct 2026 15:22:43 +0200 Subject: [PATCH 10/23] [fix] race condition when trying to reconnect (#5547) --- client/service.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/client/service.go b/client/service.go index 7964050f..c18e94dd 100644 --- a/client/service.go +++ b/client/service.go @@ -275,7 +275,15 @@ func (svr *Service) Run(ctx context.Context) error { } func (svr *Service) keepControllerWorking() { - <-svr.ctl.Done() + // ctl is swapped under ctlMu in loopLoginUntilSuccess / stop; take a + // snapshot before waiting so stop's nil write does not race this read. + svr.ctlMu.RLock() + ctl := svr.ctl + svr.ctlMu.RUnlock() + if ctl == nil { + return + } + <-ctl.Done() // There is a situation where the login is successful but due to certain reasons, // the control immediately exits. It is necessary to limit the frequency of reconnection in this case. @@ -285,8 +293,11 @@ func (svr *Service) keepControllerWorking() { // loopLoginUntilSuccess is another layer of loop that will continuously attempt to // login to the server until successful. svr.loopLoginUntilSuccess(20*time.Second, false) - if svr.ctl != nil { - <-svr.ctl.Done() + svr.ctlMu.RLock() + ctl := svr.ctl + svr.ctlMu.RUnlock() + if ctl != nil { + <-ctl.Done() return false, errors.New("control is closed and try another loop") } // If the control is nil, it means that the login failed and the service is also closed. @@ -377,7 +388,7 @@ func (svr *Service) UpdateAllConfigurer(proxyCfgs []v1.ProxyConfigurer, visitorC svr.ctlMu.RUnlock() if ctl != nil { - return svr.ctl.UpdateAllConfigurer(proxyCfgs, visitorCfgs) + return ctl.UpdateAllConfigurer(proxyCfgs, visitorCfgs) } return nil } From f8be1cdfe21f417a7cf80501784a49f9c22b3911 Mon Sep 17 00:00:00 2001 From: stefanzier-prophet Date: Thu, 8 Oct 2026 07:06:35 -0700 Subject: [PATCH 11/23] deps: bump golang.org/x/crypto to v0.56.0 and build with Go 1.26 (#5558) x/crypto v0.56.0 fixes two denial-of-service bugs in golang.org/x/crypto/ssh channel handling (GO-2026-6354 / CVE-2026-78662, GO-2026-6355 / CVE-2026-56855), reachable through the frps SSH tunnel gateway in pkg/ssh. v0.55.0 fixes GO-2026-6303 / CVE-2026-56854. v0.56.0 requires Go 1.26, so the go directive, CI and Docker builders move from 1.25 to 1.26; Go 1.25 stopped receiving security fixes when Go 1.27 was released. Also bumps github.com/Azure/go-ntlmssp to v0.1.1 (GO-2026-5543 / CVE-2026-32952). Claude-Session: https://claude.ai/code/session_01FV4GkYDtbatepp2KH3s247 Co-authored-by: Claude Opus 5.5 (1M context) --- .circleci/config.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/goreleaser.yml | 2 +- dockerfiles/Dockerfile-for-frpc | 2 +- dockerfiles/Dockerfile-for-frps | 2 +- go.mod | 12 ++++++------ go.sum | 20 ++++++++++---------- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4418158a..ac10fc9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 jobs: go-version-latest: docker: - - image: cimg/go:1.25-node + - image: cimg/go:1.26-node resource_class: large steps: - checkout diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 79eaa6df..1e9b9f46 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: - go-version: '1.25' + go-version: '1.26' cache: false - uses: actions/setup-node@v6 with: diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index 5f3f156f..16fcbc97 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.25' + go-version: '1.26' - uses: actions/setup-node@v6 with: node-version: '22' diff --git a/dockerfiles/Dockerfile-for-frpc b/dockerfiles/Dockerfile-for-frpc index 6760bb16..e83b10f0 100644 --- a/dockerfiles/Dockerfile-for-frpc +++ b/dockerfiles/Dockerfile-for-frpc @@ -8,7 +8,7 @@ RUN npm install WORKDIR /web/frpc RUN npm run build -FROM golang:1.25 AS building +FROM golang:1.26 AS building COPY . /building COPY --from=web-builder /web/frpc/dist /building/web/frpc/dist diff --git a/dockerfiles/Dockerfile-for-frps b/dockerfiles/Dockerfile-for-frps index 8ef7dff4..c13004d3 100644 --- a/dockerfiles/Dockerfile-for-frps +++ b/dockerfiles/Dockerfile-for-frps @@ -8,7 +8,7 @@ RUN npm install WORKDIR /web/frps RUN npm run build -FROM golang:1.25 AS building +FROM golang:1.26 AS building COPY . /building COPY --from=web-builder /web/frps/dist /building/web/frps/dist diff --git a/go.mod b/go.mod index 87d47021..838ec8d1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/fatedier/frp -go 1.25.0 +go 1.26.0 require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 @@ -25,8 +25,8 @@ require ( github.com/tidwall/gjson v1.17.1 github.com/vishvananda/netlink v1.3.0 github.com/xtaci/kcp-go/v5 v5.6.13 - golang.org/x/crypto v0.54.0 - golang.org/x/net v0.56.0 + golang.org/x/crypto v0.56.0 + golang.org/x/net v0.57.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -39,7 +39,7 @@ require ( ) require ( - github.com/Azure/go-ntlmssp v0.1.0 // indirect + github.com/Azure/go-ntlmssp v0.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -64,8 +64,8 @@ require ( github.com/tjfoc/gmsm v1.4.1 // indirect github.com/vishvananda/netns v0.0.4 // indirect go.uber.org/automaxprocs v1.6.0 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.48.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index ce3512c9..e5b5ba7e 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= -github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= +github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw= +github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -151,8 +151,8 @@ go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -163,8 +163,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= @@ -186,8 +186,8 @@ golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -195,8 +195,8 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= From 4cd4747525bc4be523ead8a0bd061026378eef24 Mon Sep 17 00:00:00 2001 From: fatedier Date: Thu, 8 Oct 2026 22:28:31 +0800 Subject: [PATCH 12/23] fix: use Go 1.26 new expressions for pointer initialization (#5559) --- client/service_test.go | 4 +--- pkg/config/legacy/conversion.go | 14 ++++++-------- pkg/config/v1/client.go | 8 ++++---- pkg/config/v1/proxy_plugin.go | 6 ++---- pkg/config/v1/server.go | 4 ++-- server/http/controller_v2_test.go | 10 +++------- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/client/service_test.go b/client/service_test.go index 29f141a1..cae26da3 100644 --- a/client/service_test.go +++ b/client/service_test.go @@ -9,8 +9,6 @@ import ( "strings" "testing" - "github.com/samber/lo" - "github.com/fatedier/frp/pkg/config/source" v1 "github.com/fatedier/frp/pkg/config/v1" ) @@ -49,7 +47,7 @@ func TestRunStopsStartedComponentsOnInitialLoginFailure(t *testing.T) { svr, err := NewService(ServiceOptions{ Common: &v1.ClientCommonConfig{ - LoginFailExit: lo.ToPtr(true), + LoginFailExit: new(true), WebServer: v1.WebServerConfig{ Addr: "127.0.0.1", Port: port, diff --git a/pkg/config/legacy/conversion.go b/pkg/config/legacy/conversion.go index ec8d0698..f394fea4 100644 --- a/pkg/config/legacy/conversion.go +++ b/pkg/config/legacy/conversion.go @@ -17,8 +17,6 @@ package legacy import ( "strings" - "github.com/samber/lo" - "github.com/fatedier/frp/pkg/config/types" v1 "github.com/fatedier/frp/pkg/config/v1" ) @@ -49,7 +47,7 @@ func Convert_ClientCommonConf_To_v1(conf *ClientCommonConf) *v1.ClientCommonConf out.Transport.ConnectServerLocalIP = conf.ConnectServerLocalIP out.Transport.ProxyURL = conf.HTTPProxy out.Transport.PoolCount = conf.PoolCount - out.Transport.TCPMux = lo.ToPtr(conf.TCPMux) + out.Transport.TCPMux = new(conf.TCPMux) out.Transport.TCPMuxKeepaliveInterval = conf.TCPMuxKeepaliveInterval out.Transport.Protocol = conf.Protocol out.Transport.HeartbeatInterval = conf.HeartbeatInterval @@ -57,8 +55,8 @@ func Convert_ClientCommonConf_To_v1(conf *ClientCommonConf) *v1.ClientCommonConf out.Transport.QUIC.KeepalivePeriod = conf.QUICKeepalivePeriod out.Transport.QUIC.MaxIdleTimeout = conf.QUICMaxIdleTimeout out.Transport.QUIC.MaxIncomingStreams = conf.QUICMaxIncomingStreams - out.Transport.TLS.Enable = lo.ToPtr(conf.TLSEnable) - out.Transport.TLS.DisableCustomTLSFirstByte = lo.ToPtr(conf.DisableCustomTLSFirstByte) + out.Transport.TLS.Enable = new(conf.TLSEnable) + out.Transport.TLS.DisableCustomTLSFirstByte = new(conf.DisableCustomTLSFirstByte) out.Transport.TLS.CertFile = conf.TLSCertFile out.Transport.TLS.KeyFile = conf.TLSKeyFile out.Transport.TLS.TrustedCaFile = conf.TLSTrustedCaFile @@ -77,7 +75,7 @@ func Convert_ClientCommonConf_To_v1(conf *ClientCommonConf) *v1.ClientCommonConf out.WebServer.PprofEnable = conf.PprofEnable out.DNSServer = conf.DNSServer - out.LoginFailExit = lo.ToPtr(conf.LoginFailExit) + out.LoginFailExit = new(conf.LoginFailExit) out.Start = conf.Start out.UDPPacketSize = conf.UDPPacketSize out.Metadatas = conf.Metas @@ -134,14 +132,14 @@ func Convert_ServerCommonConf_To_v1(conf *ServerCommonConf) *v1.ServerConfig { out.Log.MaxDays = conf.LogMaxDays out.Log.DisablePrintColor = conf.DisableLogColor - out.DetailedErrorsToClient = lo.ToPtr(conf.DetailedErrorsToClient) + out.DetailedErrorsToClient = new(conf.DetailedErrorsToClient) out.SubDomainHost = conf.SubDomainHost out.Custom404Page = conf.Custom404Page out.UserConnTimeout = conf.UserConnTimeout out.UDPPacketSize = conf.UDPPacketSize out.NatHoleAnalysisDataReserveHours = conf.NatHoleAnalysisDataReserveHours - out.Transport.TCPMux = lo.ToPtr(conf.TCPMux) + out.Transport.TCPMux = new(conf.TCPMux) out.Transport.TCPMuxKeepaliveInterval = conf.TCPMuxKeepaliveInterval out.Transport.TCPKeepAlive = conf.TCPKeepAlive out.Transport.MaxPoolCount = conf.MaxPoolCount diff --git a/pkg/config/v1/client.go b/pkg/config/v1/client.go index 05d02c94..6736030a 100644 --- a/pkg/config/v1/client.go +++ b/pkg/config/v1/client.go @@ -85,7 +85,7 @@ type ClientCommonConfig struct { func (c *ClientCommonConfig) Complete() error { c.ServerAddr = util.EmptyOr(c.ServerAddr, "0.0.0.0") c.ServerPort = util.EmptyOr(c.ServerPort, 7000) - c.LoginFailExit = util.EmptyOr(c.LoginFailExit, lo.ToPtr(true)) + c.LoginFailExit = util.EmptyOr(c.LoginFailExit, new(true)) c.NatHoleSTUNServer = util.EmptyOr(c.NatHoleSTUNServer, "stun.easyvoip.com:3478") if err := c.Auth.Complete(); err != nil { @@ -151,7 +151,7 @@ func (c *ClientTransportConfig) Complete() { c.DialServerKeepAlive = util.EmptyOr(c.DialServerKeepAlive, 7200) c.ProxyURL = util.EmptyOr(c.ProxyURL, os.Getenv("http_proxy")) c.PoolCount = util.EmptyOr(c.PoolCount, 1) - c.TCPMux = util.EmptyOr(c.TCPMux, lo.ToPtr(true)) + c.TCPMux = util.EmptyOr(c.TCPMux, new(true)) c.TCPMuxKeepaliveInterval = util.EmptyOr(c.TCPMuxKeepaliveInterval, 30) if lo.FromPtr(c.TCPMux) { // If TCPMux is enabled, heartbeat of application layer is unnecessary because we can rely on heartbeat in tcpmux. @@ -180,8 +180,8 @@ type TLSClientConfig struct { } func (c *TLSClientConfig) Complete() { - c.Enable = util.EmptyOr(c.Enable, lo.ToPtr(true)) - c.DisableCustomTLSFirstByte = util.EmptyOr(c.DisableCustomTLSFirstByte, lo.ToPtr(true)) + c.Enable = util.EmptyOr(c.Enable, new(true)) + c.DisableCustomTLSFirstByte = util.EmptyOr(c.DisableCustomTLSFirstByte, new(true)) } type AuthClientConfig struct { diff --git a/pkg/config/v1/proxy_plugin.go b/pkg/config/v1/proxy_plugin.go index 0d6ca3d0..7c63c2f9 100644 --- a/pkg/config/v1/proxy_plugin.go +++ b/pkg/config/v1/proxy_plugin.go @@ -17,8 +17,6 @@ package v1 import ( "reflect" - "github.com/samber/lo" - "github.com/fatedier/frp/pkg/util/jsonx" "github.com/fatedier/frp/pkg/util/util" ) @@ -125,7 +123,7 @@ type HTTPS2HTTPPluginOptions struct { } func (o *HTTPS2HTTPPluginOptions) Complete() { - o.EnableHTTP2 = util.EmptyOr(o.EnableHTTP2, lo.ToPtr(true)) + o.EnableHTTP2 = util.EmptyOr(o.EnableHTTP2, new(true)) } func (o *HTTPS2HTTPPluginOptions) Clone() ClientPluginOptions { @@ -149,7 +147,7 @@ type HTTPS2HTTPSPluginOptions struct { } func (o *HTTPS2HTTPSPluginOptions) Complete() { - o.EnableHTTP2 = util.EmptyOr(o.EnableHTTP2, lo.ToPtr(true)) + o.EnableHTTP2 = util.EmptyOr(o.EnableHTTP2, new(true)) } func (o *HTTPS2HTTPSPluginOptions) Clone() ClientPluginOptions { diff --git a/pkg/config/v1/server.go b/pkg/config/v1/server.go index 67c1d84b..6c1c75c4 100644 --- a/pkg/config/v1/server.go +++ b/pkg/config/v1/server.go @@ -118,7 +118,7 @@ func (c *ServerConfig) Complete() error { } c.VhostHTTPTimeout = util.EmptyOr(c.VhostHTTPTimeout, 60) - c.DetailedErrorsToClient = util.EmptyOr(c.DetailedErrorsToClient, lo.ToPtr(true)) + c.DetailedErrorsToClient = util.EmptyOr(c.DetailedErrorsToClient, new(true)) c.UserConnTimeout = util.EmptyOr(c.UserConnTimeout, 10) c.UDPPacketSize = util.EmptyOr(c.UDPPacketSize, 1500) c.NatHoleAnalysisDataReserveHours = util.EmptyOr(c.NatHoleAnalysisDataReserveHours, 7*24) @@ -180,7 +180,7 @@ type ServerTransportConfig struct { } func (c *ServerTransportConfig) Complete() { - c.TCPMux = util.EmptyOr(c.TCPMux, lo.ToPtr(true)) + c.TCPMux = util.EmptyOr(c.TCPMux, new(true)) c.TCPMuxKeepaliveInterval = util.EmptyOr(c.TCPMuxKeepaliveInterval, 30) c.TCPKeepAlive = util.EmptyOr(c.TCPKeepAlive, 7200) c.MaxPoolCount = util.EmptyOr(c.MaxPoolCount, 5) diff --git a/server/http/controller_v2_test.go b/server/http/controller_v2_test.go index 042cedbc..6a233a36 100644 --- a/server/http/controller_v2_test.go +++ b/server/http/controller_v2_test.go @@ -615,7 +615,7 @@ func TestMatchV2ProxyQueryMatchesSpecFields(t *testing.T) { name: "tcp remote port", item: model.V2ProxyResp{Name: "tcp-proxy", Spec: model.V2ProxySpec{ Type: "tcp", - TCP: &model.V2TCPProxySpec{RemotePort: v2TestIntPtr(6000)}, + TCP: &model.V2TCPProxySpec{RemotePort: new(6000)}, }}, q: "6000", want: true, @@ -624,7 +624,7 @@ func TestMatchV2ProxyQueryMatchesSpecFields(t *testing.T) { name: "udp remote port", item: model.V2ProxyResp{Name: "udp-proxy", Spec: model.V2ProxySpec{ Type: "udp", - UDP: &model.V2UDPProxySpec{RemotePort: v2TestIntPtr(7000)}, + UDP: &model.V2UDPProxySpec{RemotePort: new(7000)}, }}, q: "7000", want: true, @@ -633,7 +633,7 @@ func TestMatchV2ProxyQueryMatchesSpecFields(t *testing.T) { name: "remote port does not match colon form", item: model.V2ProxyResp{Name: "tcp-proxy", Spec: model.V2ProxySpec{ Type: "tcp", - TCP: &model.V2TCPProxySpec{RemotePort: v2TestIntPtr(6000)}, + TCP: &model.V2TCPProxySpec{RemotePort: new(6000)}, }}, q: ":6000", want: false, @@ -767,10 +767,6 @@ func TestLegacyAPIResponsesRemainBare(t *testing.T) { } } -func v2TestIntPtr(value int) *int { - return &value -} - func newV2TestController(t *testing.T) *Controller { t.Helper() From 55107d003da383c9049cdc3195fef9f9f14df950 Mon Sep 17 00:00:00 2001 From: fatedier Date: Fri, 9 Oct 2026 12:27:12 +0800 Subject: [PATCH 13/23] fix: synchronize service startup and shutdown (#5560) --- client/proxy/proxy_wrapper.go | 3 +- client/proxy/proxy_wrapper_test.go | 76 ++++++++++++ client/service.go | 33 ++++- client/service_lifecycle_test.go | 186 +++++++++++++++++++++++++++++ server/service.go | 50 ++++++-- server/service_lifecycle_test.go | 151 +++++++++++++++++++++++ 6 files changed, 485 insertions(+), 14 deletions(-) create mode 100644 client/proxy/proxy_wrapper_test.go create mode 100644 client/service_lifecycle_test.go create mode 100644 server/service_lifecycle_test.go diff --git a/client/proxy/proxy_wrapper.go b/client/proxy/proxy_wrapper.go index 37a71277..30ea5e33 100644 --- a/client/proxy/proxy_wrapper.go +++ b/client/proxy/proxy_wrapper.go @@ -267,8 +267,9 @@ func (pw *Wrapper) InWorkConn(workConn net.Conn, m *msg.StartWorkConn) { xl := pw.xl pw.mu.RLock() pxy := pw.pxy + running := pw.Phase == ProxyPhaseRunning pw.mu.RUnlock() - if pxy != nil && pw.Phase == ProxyPhaseRunning { + if pxy != nil && running { xl.Debugf("start a new work connection, localAddr: %s remoteAddr: %s", workConn.LocalAddr().String(), workConn.RemoteAddr().String()) go pxy.InWorkConn(workConn, m) } else { diff --git a/client/proxy/proxy_wrapper_test.go b/client/proxy/proxy_wrapper_test.go new file mode 100644 index 00000000..639f3cf1 --- /dev/null +++ b/client/proxy/proxy_wrapper_test.go @@ -0,0 +1,76 @@ +//go:build !frps + +package proxy + +import ( + "io" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fatedier/frp/pkg/msg" + "github.com/fatedier/frp/pkg/util/xlog" +) + +type closingWorkProxy struct { + Proxy + accepted atomic.Int64 +} + +func (p *closingWorkProxy) InWorkConn(conn net.Conn, _ *msg.StartWorkConn) { + p.accepted.Add(1) + _ = conn.Close() +} + +func assertWrapperWorkConnClosed(t *testing.T, pw *Wrapper) { + t.Helper() + conn, peer := net.Pipe() + defer conn.Close() + defer peer.Close() + require.NoError(t, peer.SetReadDeadline(time.Now().Add(time.Second))) + pw.InWorkConn(conn, &msg.StartWorkConn{}) + _, err := peer.Read(make([]byte, 1)) + require.ErrorIs(t, err, io.EOF) +} + +func TestWrapperInWorkConnPhaseSnapshot(t *testing.T) { + pxy := &closingWorkProxy{} + pw := &Wrapper{pxy: pxy, xl: xlog.New()} + for _, phase := range []string{ProxyPhaseNew, ProxyPhaseWaitStart, ProxyPhaseStartErr, ProxyPhaseCheckFailed, ProxyPhaseClosed} { + pw.Phase = phase + assertWrapperWorkConnClosed(t, pw) + } + require.Zero(t, pxy.accepted.Load(), "inactive proxy received a work connection") + pw.Phase = ProxyPhaseRunning + pw.pxy = nil + assertWrapperWorkConnClosed(t, pw) + pw.pxy = pxy + assertWrapperWorkConnClosed(t, pw) + require.Equal(t, int64(1), pxy.accepted.Load()) + + start, done := make(chan struct{}), make(chan struct{}) + go func() { + <-start + for range 20000 { + pw.mu.Lock() + pw.Phase = ProxyPhaseRunning + pw.mu.Unlock() + pw.mu.Lock() + pw.Phase = ProxyPhaseClosed + pw.mu.Unlock() + } + close(done) + }() + close(start) + for range 10000 { + assertWrapperWorkConnClosed(t, pw) + } + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("phase writer did not finish") + } +} diff --git a/client/service.go b/client/service.go index c18e94dd..055de40d 100644 --- a/client/service.go +++ b/client/service.go @@ -155,6 +155,9 @@ type Service struct { ctx context.Context // call cancel to stop service cancel context.CancelCauseFunc + // Protect cancel publication and shutdown before the first Run. + lifecycleMu sync.Mutex + closed bool connectorCreator func(context.Context, *v1.ClientCommonConfig) Connector handleWorkConnCb func(*v1.ProxyBaseConfig, net.Conn, *msg.StartWorkConn) bool @@ -225,8 +228,20 @@ func NewService(options ServiceOptions) (*Service, error) { func (svr *Service) Run(ctx context.Context) error { ctx, cancel := context.WithCancelCause(ctx) + svr.lifecycleMu.Lock() + if svr.closed { + svr.lifecycleMu.Unlock() + cancel(nil) + return ctx.Err() + } svr.ctx = xlog.NewContext(ctx, xlog.FromContextSafe(ctx)) svr.cancel = cancel + svr.lifecycleMu.Unlock() + defer cancel(nil) + defer svr.stop() + if err := ctx.Err(); err != nil { + return err + } // set custom DNSServer if svr.common.DNSServer != "" { @@ -237,7 +252,6 @@ func (svr *Service) Run(ctx context.Context) error { vnetController := svr.vnetController if err := svr.vnetController.Init(); err != nil { log.Errorf("init virtual network controller error: %v", err) - svr.stop() return err } go func() { @@ -263,14 +277,12 @@ func (svr *Service) Run(ctx context.Context) error { if svr.ctl == nil { cancelCause := cancelErr{} _ = errors.As(context.Cause(svr.ctx), &cancelCause) - svr.stop() return fmt.Errorf("login to the server failed: %v. With loginFailExit enabled, no additional retries will be attempted", cancelCause.Err) } go svr.keepControllerWorking() <-svr.ctx.Done() - svr.stop() return nil } @@ -428,7 +440,20 @@ func (svr *Service) Close() { func (svr *Service) GracefulClose(d time.Duration) { svr.gracefulShutdownDuration.Store(int64(d)) - svr.cancel(nil) + svr.lifecycleMu.Lock() + if svr.closed { + svr.lifecycleMu.Unlock() + return + } + svr.closed = true + cancel := svr.cancel + svr.lifecycleMu.Unlock() + if cancel != nil { + // Once Run has claimed startup, it also owns cleanup. + cancel(nil) + } else { + svr.stop() + } } func (svr *Service) stop() { diff --git a/client/service_lifecycle_test.go b/client/service_lifecycle_test.go new file mode 100644 index 00000000..13969506 --- /dev/null +++ b/client/service_lifecycle_test.go @@ -0,0 +1,186 @@ +package client + +import ( + "context" + "io" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fatedier/frp/pkg/config/source" + v1 "github.com/fatedier/frp/pkg/config/v1" + "github.com/fatedier/frp/pkg/config/v1/validation" + "github.com/fatedier/frp/pkg/msg" +) + +func newLifecycleTestService(t *testing.T, creator func(context.Context, *v1.ClientCommonConfig) Connector) (*Service, string) { + t.Helper() + svr, err := NewService(ServiceOptions{ + Common: &v1.ClientCommonConfig{ + LoginFailExit: new(false), + WebServer: v1.WebServerConfig{Addr: "127.0.0.1", Port: getFreeTCPPort(t)}, + }, + ConfigSourceAggregator: source.NewAggregator(source.NewConfigSource()), + ConnectorCreator: creator, + }) + require.NoError(t, err) + _, err = validation.NewConfigValidator(nil).ValidateClientCommonConfig(svr.common) + require.NoError(t, err) + address := svr.webServer.Address() + assertLifecycleAdminPort(t, address, false) + return svr, address +} + +func assertLifecycleAdminPort(t *testing.T, address string, released bool) { + t.Helper() + ln, err := net.Listen("tcp", address) + if ln != nil { + _ = ln.Close() + } + if released { + require.NoError(t, err, "admin listener was not released") + } else { + require.Error(t, err, "admin listener was not bound") + } +} + +func waitLifecycleSignal(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("service lifecycle operation did not finish") + } +} + +func TestServiceRunAfterEarlyShutdown(t *testing.T) { + for _, mode := range []string{"close before Run", "parent already cancelled"} { + t.Run(mode, func(t *testing.T) { + var attempts atomic.Int64 + svr, address := newLifecycleTestService(t, func(context.Context, *v1.ClientCommonConfig) Connector { + attempts.Add(1) + return &failingConnector{err: net.ErrClosed} + }) + // Detect startup's global DNS side effect without making DNS requests. + resolver := net.DefaultResolver + t.Cleanup(func() { net.DefaultResolver = resolver }) + svr.common.DNSServer = "127.0.0.1:53" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "close before Run" { + svr.Close() + assertLifecycleAdminPort(t, address, true) + } else { + cancel() + } + done := make(chan struct{}) + var runErr error + go func() { + runErr = svr.Run(ctx) + close(done) + }() + waitLifecycleSignal(t, done) + require.ErrorIs(t, runErr, context.Canceled) + require.Zero(t, attempts.Load(), "closed service attempted login") + require.Same(t, resolver, net.DefaultResolver, "closed service changed DNS") + assertLifecycleAdminPort(t, address, true) + }) + } +} + +func TestServiceStartupShutdownLeavesCleanupToRun(t *testing.T) { + for _, mode := range []string{"Close", "parent cancellation"} { + t.Run(mode, func(t *testing.T) { + entered, resume := make(chan struct{}), make(chan struct{}) + svr, address := newLifecycleTestService(t, func(ctx context.Context, _ *v1.ClientCommonConfig) Connector { + close(entered) + <-resume + return &failingConnector{err: ctx.Err()} + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer close(resume) + runDone := make(chan struct{}) + go func() { _ = svr.Run(ctx); close(runDone) }() + waitLifecycleSignal(t, entered) + closeDone := make(chan struct{}) + go func() { + if mode == "Close" { + svr.Close() + } else { + cancel() + } + close(closeDone) + }() + waitLifecycleSignal(t, closeDone) + select { + case <-runDone: + t.Fatal("Run exited while startup still owned its resources") + default: + } + assertLifecycleAdminPort(t, address, false) + resume <- struct{}{} + waitLifecycleSignal(t, runDone) + assertLifecycleAdminPort(t, address, true) + require.ErrorIs(t, svr.ctx.Err(), context.Canceled) + }) + } +} + +func TestServiceShutdownAfterLogin(t *testing.T) { + for _, mode := range []string{"Close", "parent cancellation"} { + t.Run(mode, func(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + connector := &testConnector{conn: clientConn} + svr, address := newLifecycleTestService(t, func(context.Context, *v1.ClientCommonConfig) Connector { + return connector + }) + defer svr.Close() + peerDone := make(chan error, 1) + go func() { + rw := msg.NewV1ReadWriter(serverConn) + var login msg.Login + if err := rw.ReadMsgInto(&login); err != nil { + peerDone <- err + return + } + if err := rw.WriteMsg(&msg.LoginResp{RunID: "lifecycle-test"}); err != nil { + peerDone <- err + return + } + _, err := io.Copy(io.Discard, serverConn) + peerDone <- err + }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runDone := make(chan struct{}) + var runErr error + go func() { runErr = svr.Run(ctx); close(runDone) }() + require.Eventually(t, func() bool { + svr.ctlMu.RLock() + defer svr.ctlMu.RUnlock() + return svr.ctl != nil + }, 3*time.Second, time.Millisecond, "login did not establish control") + if mode == "Close" { + svr.Close() + } else { + cancel() + } + waitLifecycleSignal(t, runDone) + require.NoError(t, runErr) + require.True(t, connector.closed.Load()) + select { + case err := <-peerDone: + require.NoError(t, err) + case <-time.After(3 * time.Second): + t.Fatal("control connection was not released") + } + assertLifecycleAdminPort(t, address, true) + }) + } +} diff --git a/server/service.go b/server/service.go index 6ea24cc4..4556358b 100644 --- a/server/service.go +++ b/server/service.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "strconv" + "sync" "time" "github.com/fatedier/golib/crypto" @@ -120,6 +121,9 @@ type Service struct { // web server for dashboard UI and apis webServer *httppkg.Server + // HTTP vhost resources are started and owned by NewService. + httpVhostServer *http.Server + httpVhostListener net.Listener sshTunnelGateway *ssh.Gateway @@ -133,7 +137,10 @@ type Service struct { // service context ctx context.Context // call cancel to stop service - cancel context.CancelFunc + cancel context.CancelFunc + lifecycleMu sync.Mutex + closed bool + closeOnce sync.Once } func NewService(cfg *v1.ServerConfig) (*Service, error) { @@ -319,6 +326,8 @@ func NewService(cfg *v1.ServerConfig) (*Service, error) { return nil, fmt.Errorf("create vhost http listener error, %v", err) } } + svr.httpVhostServer = server + svr.httpVhostListener = l go func() { _ = server.Serve(l) }() @@ -365,8 +374,22 @@ func NewService(cfg *v1.ServerConfig) (*Service, error) { func (svr *Service) Run(ctx context.Context) { ctx, cancel := context.WithCancel(ctx) + svr.lifecycleMu.Lock() + if svr.closed { + svr.lifecycleMu.Unlock() + cancel() + return + } svr.ctx = ctx svr.cancel = cancel + svr.lifecycleMu.Unlock() + defer svr.Close() + if ctx.Err() != nil { + return + } + // Cancellation must unblock the synchronous main listener's Accept. + stopClose := context.AfterFunc(ctx, func() { _ = svr.Close() }) + defer stopClose() // run dashboard web server. if svr.webServer != nil { @@ -400,13 +423,21 @@ func (svr *Service) Run(ctx context.Context) { svr.HandleListener(svr.listener, false) <-svr.ctx.Done() - // service context may not be canceled by svr.Close(), we should call it here to release resources - if svr.listener != nil { - svr.Close() - } } func (svr *Service) Close() error { + svr.lifecycleMu.Lock() + svr.closed = true + cancel := svr.cancel + svr.lifecycleMu.Unlock() + if cancel != nil { + cancel() + } + svr.closeOnce.Do(svr.closeResources) + return nil +} + +func (svr *Service) closeResources() { if svr.kcpListener != nil { svr.kcpListener.Close() } @@ -428,16 +459,17 @@ func (svr *Service) Close() error { if svr.webServer != nil { svr.webServer.Close() } + if svr.httpVhostServer != nil { + svr.httpVhostServer.Close() + // Close even if Serve has not registered its listener yet. + svr.httpVhostListener.Close() + } if svr.sshTunnelGateway != nil { svr.sshTunnelGateway.Close() } svr.rc.Close() svr.muxer.Close() svr.ctlManager.Close() - if svr.cancel != nil { - svr.cancel() - } - return nil } func (svr *Service) handleConnection(ctx context.Context, conn net.Conn, internal bool) { diff --git a/server/service_lifecycle_test.go b/server/service_lifecycle_test.go new file mode 100644 index 00000000..61481b32 --- /dev/null +++ b/server/service_lifecycle_test.go @@ -0,0 +1,151 @@ +package server + +import ( + "context" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/fatedier/frp/pkg/config/v1" + "github.com/fatedier/frp/pkg/config/v1/validation" +) + +type lifecycleListener struct { + net.Listener + accepted, closeEntered, closeResume chan struct{} + acceptOnce, closeOnce sync.Once + accepts, closes atomic.Int64 +} + +func (l *lifecycleListener) Accept() (net.Conn, error) { + l.accepts.Add(1) + l.acceptOnce.Do(func() { close(l.accepted) }) + return l.Listener.Accept() +} + +func (l *lifecycleListener) Close() error { + l.closes.Add(1) + err := l.Listener.Close() + if l.closeEntered != nil { + l.closeOnce.Do(func() { close(l.closeEntered) }) + <-l.closeResume + } + return err +} + +func freeLifecycleTCPPort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func newLifecycleTestService(t *testing.T, web bool) (*Service, *lifecycleListener, []string) { + t.Helper() + cfg := &v1.ServerConfig{ + BindAddr: "127.0.0.1", ProxyBindAddr: "127.0.0.1", + } + if web { + cfg.WebServer = v1.WebServerConfig{Addr: "127.0.0.1", Port: freeLifecycleTCPPort(t)} + cfg.VhostHTTPPort = freeLifecycleTCPPort(t) + } + require.NoError(t, cfg.Complete()) + cfg.BindPort = 0 + _, err := validation.NewConfigValidator(nil).ValidateServerConfig(cfg) + require.NoError(t, err) + svr, err := NewService(cfg) + require.NoError(t, err) + l := &lifecycleListener{Listener: svr.listener, accepted: make(chan struct{})} + svr.listener = l + addresses := []string{l.Addr().String()} + if web { + addresses = append(addresses, svr.webServer.Address()) + addresses = append(addresses, net.JoinHostPort(cfg.ProxyBindAddr, fmt.Sprint(cfg.VhostHTTPPort))) + } + assertLifecyclePorts(t, addresses, false) + return svr, l, addresses +} + +func assertLifecyclePorts(t *testing.T, addresses []string, released bool) { + t.Helper() + for _, address := range addresses { + ln, err := net.Listen("tcp", address) + if ln != nil { + _ = ln.Close() + } + if released { + require.NoError(t, err, "listener %s was not released", address) + } else { + require.Error(t, err, "listener %s was not bound", address) + } + } +} + +func TestServiceRunAfterEarlyShutdown(t *testing.T) { + for _, mode := range []string{"close before Run", "close before Run with web", "parent already cancelled"} { + t.Run(mode, func(t *testing.T) { + svr, listener, addresses := newLifecycleTestService(t, mode == "close before Run with web") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer svr.Close() + if mode != "parent already cancelled" { + require.NoError(t, svr.Close()) + assertLifecyclePorts(t, addresses, true) + } else { + cancel() + } + done := make(chan struct{}) + go func() { svr.Run(ctx); close(done) }() + waitForSignal(t, done, "Run after early shutdown") + require.Zero(t, listener.accepts.Load(), "closed service started accepting") + require.Equal(t, int64(1), listener.closes.Load()) + assertLifecyclePorts(t, addresses, true) + }) + } +} + +func TestServiceCloseMarksClosedBeforeCleanup(t *testing.T) { + svr, listener, addresses := newLifecycleTestService(t, false) + listener.closeEntered, listener.closeResume = make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(listener.closeResume) }) } + defer release() + closeDone := make(chan struct{}) + go func() { _ = svr.Close(); close(closeDone) }() + waitForSignal(t, listener.closeEntered, "Close cleanup barrier") + runDone := make(chan struct{}) + go func() { svr.Run(context.Background()); close(runDone) }() + waitForSignal(t, runDone, "Run to observe closed before cleanup finishes") + require.Zero(t, listener.accepts.Load(), "Run started during cleanup") + assertLifecyclePorts(t, addresses, false) + release() + waitForSignal(t, closeDone, "Close cleanup to finish") + assertLifecyclePorts(t, addresses, true) +} + +func TestServiceParentCancellationAndCloseShareCleanup(t *testing.T) { + svr, listener, addresses := newLifecycleTestService(t, false) + listener.closeEntered, listener.closeResume = make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(listener.closeResume) }) } + defer release() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + runDone := make(chan struct{}) + go func() { svr.Run(ctx); close(runDone) }() + waitForSignal(t, listener.accepted, "main listener Accept") + cancel() + waitForSignal(t, listener.closeEntered, "parent cancellation to close main listener") + closeDone := make(chan struct{}) + go func() { _ = svr.Close(); close(closeDone) }() + release() + waitForSignal(t, closeDone, "external Close to finish") + waitForSignal(t, runDone, "Run cleanup to finish") + require.Equal(t, int64(1), listener.closes.Load(), "cleanup ran more than once") + assertLifecyclePorts(t, addresses, true) +} From fccc7b0e9f8855882677c827a71255bc1f11d488 Mon Sep 17 00:00:00 2001 From: Cyoid <77887516+Zheng-Yu7463@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:17:08 +0800 Subject: [PATCH 14/23] fix(msg): close control connection on write failure (#5550) * fix(msg): close control connection on write failure * fix(msg): handle shutdown enqueue race and preserve constructor API --- pkg/msg/handler.go | 55 +++- pkg/msg/handler_shutdown_test.go | 87 +++++++ pkg/msg/handler_write_error_test.go | 387 ++++++++++++++++++++++++++++ 3 files changed, 524 insertions(+), 5 deletions(-) create mode 100644 pkg/msg/handler_shutdown_test.go create mode 100644 pkg/msg/handler_write_error_test.go diff --git a/pkg/msg/handler.go b/pkg/msg/handler.go index 1f2775c9..9e437198 100644 --- a/pkg/msg/handler.go +++ b/pkg/msg/handler.go @@ -19,6 +19,7 @@ import ( "io" "net" "reflect" + "sync" "github.com/fatedier/frp/pkg/proto/wire" ) @@ -111,14 +112,20 @@ type Dispatcher struct { sendCh chan Message doneCh chan struct{} + stopCh chan struct{} + stopOnce sync.Once msgHandlers map[reflect.Type]func(Message) } +// NewDispatcher uses Close, when rw implements io.Closer, to interrupt ReadMsg +// after a write failure. Otherwise, the caller must arrange for ReadMsg to return +// before Done can close. func NewDispatcher(rw ReadWriter) *Dispatcher { return &Dispatcher{ rw: rw, sendCh: make(chan Message, 100), doneCh: make(chan struct{}), + stopCh: make(chan struct{}), msgHandlers: make(map[reflect.Type]func(Message)), } } @@ -129,37 +136,75 @@ func (d *Dispatcher) Run() { go d.readLoop() } +// stop rejects new sends and interrupts ReadMsg when rw is closable. Done remains +// owned by readLoop so the control owner cannot clean up while a handler is running. +func (d *Dispatcher) stop() { + d.stopOnce.Do(func() { + close(d.stopCh) + if closer, ok := d.rw.(io.Closer); ok { + _ = closer.Close() + } + }) +} + func (d *Dispatcher) sendLoop() { for { select { - case <-d.doneCh: + case <-d.stopCh: return case m := <-d.sendCh: - _ = d.rw.WriteMsg(m) + if err := d.rw.WriteMsg(m); err != nil { + d.stop() + return + } } } } func (d *Dispatcher) readLoop() { + defer close(d.doneCh) + defer d.stop() for { m, err := d.rw.ReadMsg() if err != nil { - close(d.doneCh) return } + select { + case <-d.stopCh: + return + default: + } + if handler, ok := d.msgHandlers[reflect.TypeOf(m)]; ok { handler(m) } } } +// Send queues m for writing. A nil error does not guarantee delivery. func (d *Dispatcher) Send(m Message) error { select { - case <-d.doneCh: + case <-d.stopCh: + return io.EOF + default: + } + return d.enqueue(m) +} + +func (d *Dispatcher) enqueue(m Message) error { + select { + case <-d.stopCh: return io.EOF case d.sendCh <- m: - return nil + // Both cases can be ready if shutdown races with enqueueing. Do not + // report success when shutdown is already observable after the send. + select { + case <-d.stopCh: + return io.EOF + default: + return nil + } } } diff --git a/pkg/msg/handler_shutdown_test.go b/pkg/msg/handler_shutdown_test.go new file mode 100644 index 00000000..ee6f193a --- /dev/null +++ b/pkg/msg/handler_shutdown_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 The frp Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msg + +import ( + "bytes" + "errors" + "io" + "testing" +) + +// Preserve the original exported constructor function type. +var _ func(ReadWriter) *Dispatcher = NewDispatcher + +func TestDispatcherEnqueueRejectsStoppedDispatcher(t *testing.T) { + c := newRegressionControl(t) + d := NewDispatcher(c) + d.stop() + + // Model shutdown after Send's initial check: the stop and buffered send + // cases are both ready. A send selected here must still return EOF. + enqueued := false + for range 100 { + if err := d.enqueue(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("enqueue racing with shutdown: %v, want EOF", err) + } + if len(d.sendCh) > 0 { + enqueued = true + <-d.sendCh + } + } + if !enqueued { + t.Fatal("did not exercise the ready send case") + } +} + +func TestDispatcherAcceptsReadWriterWithoutClose(t *testing.T) { + t.Run("codec factory", func(t *testing.T) { + d := NewDispatcher(NewV1ReadWriter(&bytes.Buffer{})) + d.Run() + regressionAwait(t, d.Done(), "non-closable codec EOF") + }) + + t.Run("write failure", func(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + // Hide Close to model an existing caller implementing only ReadWriter. + d := NewDispatcher(struct{ ReadWriter }{c}) + d.Run() + t.Cleanup(func() { + c.transport.close() + regressionAwait(t, d.Done(), "externally interrupted reader") + }) + + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatalf("injected write failure: %v", err) + } + regressionAwait(t, d.stopCh, "write failure stops sends") + if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("Send after write failure: %v, want EOF", err) + } + if c.closeCalls.Load() != 0 { + t.Fatal("dispatcher closed a connection hidden by ReadWriter") + } + select { + case <-d.Done(): + t.Fatal("Done closed before the non-closable reader returned") + default: + } + }) +} diff --git a/pkg/msg/handler_write_error_test.go b/pkg/msg/handler_write_error_test.go new file mode 100644 index 00000000..0e59fc7b --- /dev/null +++ b/pkg/msg/handler_write_error_test.go @@ -0,0 +1,387 @@ +// Copyright 2026 The frp Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msg + +import ( + "bytes" + "errors" + "io" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + libcrypto "github.com/fatedier/golib/crypto" +) + +var ( + errRegressionWriteTimeout = errors.New("injected recoverable connection write timeout") + regressionKey = []byte("isolated-public-test-key") +) + +// regressionTransport fails exactly one Write, then accepts later writes. This +// models an individual mux stream write timing out while its session survives. +type regressionTransport struct { + mu sync.Mutex + buffer bytes.Buffer + calls int + failures int + failNext bool + closed chan struct{} + once sync.Once +} + +func (w *regressionTransport) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.calls++ + if w.failNext { + w.failNext = false + w.failures++ + return 0, errRegressionWriteTimeout + } + select { + case <-w.closed: + return 0, io.ErrClosedPipe + default: + } + return w.buffer.Write(p) +} + +func (w *regressionTransport) arm() { + w.mu.Lock() + w.failNext = true + w.mu.Unlock() +} + +func (w *regressionTransport) snapshot() (calls, failures, size int) { + w.mu.Lock() + defer w.mu.Unlock() + return w.calls, w.failures, w.buffer.Len() +} + +func (w *regressionTransport) close() { + w.once.Do(func() { close(w.closed) }) +} + +// regressionControl uses the production FRP message encoder and production +// golib crypto.Writer. Only the final transport and incoming messages are fake. +type regressionControl struct { + w *libcrypto.Writer + transport *regressionTransport + writes chan error + incoming chan Message + beforeRead func() error + beforeWrite func() + closeCalls atomic.Int32 +} + +func newRegressionControl(t *testing.T) *regressionControl { + t.Helper() + transport := ®ressionTransport{closed: make(chan struct{})} + w, err := libcrypto.NewWriter(transport, regressionKey) + if err != nil { + t.Fatal(err) + } + return ®ressionControl{ + w: w, transport: transport, writes: make(chan error, 100), incoming: make(chan Message, 1), + } +} + +func (c *regressionControl) ReadMsg() (Message, error) { + if c.beforeRead != nil { + if err := c.beforeRead(); err != nil { + return nil, err + } + } + select { + case m := <-c.incoming: + return m, nil + case <-c.transport.closed: + return nil, io.EOF + } +} + +func (c *regressionControl) ReadMsgInto(Message) error { + _, err := c.ReadMsg() + return err +} + +func (c *regressionControl) WriteMsg(m Message) error { + if c.beforeWrite != nil { + c.beforeWrite() + } + err := WriteMsg(c.w, m) + c.writes <- err + return err +} + +func (c *regressionControl) Close() error { + c.closeCalls.Add(1) + c.transport.close() + return nil +} + +func regressionReceiveWrite(t *testing.T, c *regressionControl) error { + t.Helper() + select { + case err := <-c.writes: + return err + case <-time.After(time.Second): + t.Fatal("dispatcher did not attempt WriteMsg") + return nil + } +} + +func regressionAwait(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +func regressionWarm(t *testing.T, c *regressionControl) { + t.Helper() + if err := c.WriteMsg(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + <-c.writes + c.transport.mu.Lock() + raw := append([]byte(nil), c.transport.buffer.Bytes()...) + c.transport.mu.Unlock() + m, err := ReadMsg(libcrypto.NewReader(bytes.NewReader(raw), regressionKey)) + if err != nil { + t.Fatalf("encrypted control roundtrip: %v", err) + } + if _, ok := m.(*ReqWorkConn); !ok { + t.Fatalf("decoded %T", m) + } +} + +func TestControlCryptoWriterKeepsErrorAfterTransportRecovery(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + c.transport.arm() + if err := c.WriteMsg(&ReqWorkConn{}); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatalf("want timeout, got %v", err) + } + <-c.writes + beforeCalls, failures, beforeBytes := c.transport.snapshot() + for range 5 { + if err := c.WriteMsg(&ReqWorkConn{}); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatalf("sticky error missing: %v", err) + } + <-c.writes + } + afterCalls, _, afterBytes := c.transport.snapshot() + if beforeCalls != afterCalls || beforeBytes != afterBytes || failures != 1 { + t.Fatalf("unexpected retry: calls %d -> %d, failures %d, bytes %d -> %d", + beforeCalls, afterCalls, failures, beforeBytes, afterBytes) + } + if n, err := c.transport.Write([]byte("healthy underlying transport")); err != nil || n == 0 { + t.Fatalf("underlying transport did not recover: %v", err) + } + t.Log("one recoverable write error permanently invalidates this crypto writer; recovered underlying transport is never retried") +} + +func TestDispatcherWriteErrorClosesControlAndFreshSessionWorks(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + d := NewDispatcher(c) + d.Run() + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatal(err) + } + regressionAwait(t, c.transport.closed, "failed control connection close") + regressionAwait(t, d.Done(), "failed dispatcher completion") + for range 100 { + if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("Send after completed shutdown: %v", err) + } + } + if c.closeCalls.Load() != 1 { + t.Fatalf("connection closed %d times", c.closeCalls.Load()) + } + + fresh := newRegressionControl(t) + defer fresh.transport.close() + d2 := NewDispatcher(fresh) + d2.Run() + for range 5 { + if err := d2.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + if err := regressionReceiveWrite(t, fresh); err != nil { + t.Fatalf("fresh session write failed: %v", err) + } + } + fresh.transport.mu.Lock() + raw := append([]byte(nil), fresh.transport.buffer.Bytes()...) + fresh.transport.mu.Unlock() + r := libcrypto.NewReader(bytes.NewReader(raw), regressionKey) + for i := range 5 { + m, err := ReadMsg(r) + if err != nil { + t.Fatalf("new stream decode %d: %v", i, err) + } + if _, ok := m.(*ReqWorkConn); !ok { + t.Fatalf("decoded %T", m) + } + } + fresh.transport.close() + regressionAwait(t, d2.Done(), "new-session cleanup") + t.Log("write error closes failed control; new session sends and decrypts five valid ReqWorkConn messages") +} + +func TestDispatcherWriteErrorWaitsForSynchronousHandler(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + d := NewDispatcher(c) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + d.RegisterHandler(&ReqWorkConn{}, func(Message) { + close(entered) + <-release + }) + d.Run() + c.incoming <- &ReqWorkConn{} + regressionAwait(t, entered, "synchronous handler entry") + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatal(err) + } + regressionAwait(t, c.transport.closed, "write failure closes control during handler") + for range 100 { + if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("Send after stop while handler is active: %v", err) + } + } + select { + case <-d.Done(): + t.Fatal("Done closed before synchronous handler returned; owner cleanup could race proxy registration") + case <-time.After(100 * time.Millisecond): + } + releaseOnce.Do(func() { close(release) }) + regressionAwait(t, d.Done(), "completion after handler returned") +} + +func TestDispatcherWriteErrorUnblocksSenderWithFullQueue(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + c.beforeWrite = func() { close(entered); <-release } + c.transport.arm() + d := NewDispatcher(c) + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + d.Run() + regressionAwait(t, entered, "blocked transport write") + for i := 0; i < cap(d.sendCh); i++ { + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + } + sent := make(chan error, 1) + go func() { sent <- d.Send(&ReqWorkConn{}) }() + // Wait until the sender has reached its full-queue channel wait. + synctest.Wait() + select { + case err := <-sent: + t.Fatalf("Send did not block on full queue: %v", err) + default: + } + releaseOnce.Do(func() { close(release) }) + select { + case err := <-sent: + if !errors.Is(err, io.EOF) { + t.Fatalf("blocked Send after write failure: %v", err) + } + case <-time.After(time.Second): + t.Fatal("write failure did not unblock full-queue sender") + } + regressionAwait(t, d.Done(), "full-queue dispatcher cleanup") + }) +} + +func TestDispatcherConcurrentReadWriteFailureClosesOnce(t *testing.T) { + for range 100 { + func() { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + readEntered, writeEntered := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + c.beforeRead = func() error { + close(readEntered) + <-release + return io.EOF + } + c.beforeWrite = func() { + close(writeEntered) + <-release + } + d := NewDispatcher(c) + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + readExited, writeExited := make(chan struct{}), make(chan struct{}) + go func() { + defer close(readExited) + d.readLoop() + }() + go func() { + defer close(writeExited) + d.sendLoop() + }() + regressionAwait(t, readEntered, "read path before injected failure") + regressionAwait(t, writeEntered, "write path before injected failure") + releaseOnce.Do(func() { close(release) }) + regressionAwait(t, readExited, "failed read loop exit") + regressionAwait(t, writeExited, "failed write loop exit") + regressionAwait(t, d.Done(), "concurrent read/write shutdown") + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatalf("concurrent WriteMsg failure: %v", err) + } + if _, failures, _ := c.transport.snapshot(); failures != 1 { + t.Fatalf("injected transport failed %d times", failures) + } + if c.closeCalls.Load() != 1 { + t.Fatalf("connection closed %d times", c.closeCalls.Load()) + } + }() + } +} From 1dbd44edd347e81ec5d90f2e8e00af22c4ccd857 Mon Sep 17 00:00:00 2001 From: fatedier Date: Fri, 9 Oct 2026 17:46:31 +0800 Subject: [PATCH 15/23] fix(msg): interrupt control reads during dispatcher shutdown (#5561) --- pkg/msg/handler.go | 14 +- pkg/msg/handler_shutdown_test.go | 19 +-- pkg/msg/handler_write_error_test.go | 164 ++++++++++---------- pkg/msg/handler_yamux_test.go | 225 ++++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 100 deletions(-) create mode 100644 pkg/msg/handler_yamux_test.go diff --git a/pkg/msg/handler.go b/pkg/msg/handler.go index 9e437198..b0ac187a 100644 --- a/pkg/msg/handler.go +++ b/pkg/msg/handler.go @@ -20,6 +20,7 @@ import ( "net" "reflect" "sync" + "time" "github.com/fatedier/frp/pkg/proto/wire" ) @@ -117,9 +118,11 @@ type Dispatcher struct { msgHandlers map[reflect.Type]func(Message) } -// NewDispatcher uses Close, when rw implements io.Closer, to interrupt ReadMsg -// after a write failure. Otherwise, the caller must arrange for ReadMsg to return -// before Done can close. +// NewDispatcher uses SetReadDeadline, when supported, and Close, when rw +// implements io.Closer, to interrupt ReadMsg after a write failure. Without a +// working read deadline, Close or the caller must arrange for ReadMsg to return. +// These methods must return in bounded time and must not wait for Done: Done +// closes only after ReadMsg, any synchronous handler, and stop have returned. func NewDispatcher(rw ReadWriter) *Dispatcher { return &Dispatcher{ rw: rw, @@ -136,11 +139,14 @@ func (d *Dispatcher) Run() { go d.readLoop() } -// stop rejects new sends and interrupts ReadMsg when rw is closable. Done remains +// stop rejects new sends and interrupts ReadMsg when supported by rw. Done remains // owned by readLoop so the control owner cannot clean up while a handler is running. func (d *Dispatcher) stop() { d.stopOnce.Do(func() { close(d.stopCh) + if conn, ok := d.rw.(interface{ SetReadDeadline(time.Time) error }); ok { + _ = conn.SetReadDeadline(time.Now()) + } if closer, ok := d.rw.(io.Closer); ok { _ = closer.Close() } diff --git a/pkg/msg/handler_shutdown_test.go b/pkg/msg/handler_shutdown_test.go index ee6f193a..3f74c06d 100644 --- a/pkg/msg/handler_shutdown_test.go +++ b/pkg/msg/handler_shutdown_test.go @@ -31,18 +31,13 @@ func TestDispatcherEnqueueRejectsStoppedDispatcher(t *testing.T) { // Model shutdown after Send's initial check: the stop and buffered send // cases are both ready. A send selected here must still return EOF. - enqueued := false - for range 100 { - if err := d.enqueue(&ReqWorkConn{}); !errors.Is(err, io.EOF) { - t.Fatalf("enqueue racing with shutdown: %v, want EOF", err) - } - if len(d.sendCh) > 0 { - enqueued = true - <-d.sendCh - } + if err := d.enqueue(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("enqueue racing with shutdown: %v, want EOF", err) } - if !enqueued { - t.Fatal("did not exercise the ready send case") + // This single call checks that a stopped dispatcher's enqueue returns EOF; it + // does not guarantee exercising the send case and post-enqueue check on every run. + if len(d.sendCh) > 0 { + t.Log("selected the send case; post-enqueue check returned EOF") } } @@ -65,7 +60,7 @@ func TestDispatcherAcceptsReadWriterWithoutClose(t *testing.T) { }) c.transport.arm() - if err := d.Send(&ReqWorkConn{}); err != nil { + if err := d.Send(&ReqWorkConn{}); err != nil && !errors.Is(err, io.EOF) { t.Fatal(err) } if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { diff --git a/pkg/msg/handler_write_error_test.go b/pkg/msg/handler_write_error_test.go index 0e59fc7b..17142297 100644 --- a/pkg/msg/handler_write_error_test.go +++ b/pkg/msg/handler_write_error_test.go @@ -207,7 +207,7 @@ func TestDispatcherWriteErrorClosesControlAndFreshSessionWorks(t *testing.T) { d := NewDispatcher(c) d.Run() c.transport.arm() - if err := d.Send(&ReqWorkConn{}); err != nil { + if err := d.Send(&ReqWorkConn{}); err != nil && !errors.Is(err, io.EOF) { t.Fatal(err) } if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { @@ -215,10 +215,8 @@ func TestDispatcherWriteErrorClosesControlAndFreshSessionWorks(t *testing.T) { } regressionAwait(t, c.transport.closed, "failed control connection close") regressionAwait(t, d.Done(), "failed dispatcher completion") - for range 100 { - if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { - t.Fatalf("Send after completed shutdown: %v", err) - } + if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("Send after completed shutdown: %v", err) } if c.closeCalls.Load() != 1 { t.Fatalf("connection closed %d times", c.closeCalls.Load()) @@ -255,40 +253,42 @@ func TestDispatcherWriteErrorClosesControlAndFreshSessionWorks(t *testing.T) { } func TestDispatcherWriteErrorWaitsForSynchronousHandler(t *testing.T) { - c := newRegressionControl(t) - regressionWarm(t, c) - defer c.transport.close() - d := NewDispatcher(c) - entered, release := make(chan struct{}), make(chan struct{}) - var releaseOnce sync.Once - defer releaseOnce.Do(func() { close(release) }) - d.RegisterHandler(&ReqWorkConn{}, func(Message) { - close(entered) - <-release - }) - d.Run() - c.incoming <- &ReqWorkConn{} - regressionAwait(t, entered, "synchronous handler entry") - c.transport.arm() - if err := d.Send(&ReqWorkConn{}); err != nil { - t.Fatal(err) - } - if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { - t.Fatal(err) - } - regressionAwait(t, c.transport.closed, "write failure closes control during handler") - for range 100 { + synctest.Test(t, func(t *testing.T) { + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + d := NewDispatcher(c) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + d.RegisterHandler(&ReqWorkConn{}, func(Message) { + close(entered) + <-release + }) + d.Run() + c.incoming <- &ReqWorkConn{} + regressionAwait(t, entered, "synchronous handler entry") + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil && !errors.Is(err, io.EOF) { + t.Fatal(err) + } + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatal(err) + } + regressionAwait(t, c.transport.closed, "write failure closes control during handler") if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { t.Fatalf("Send after stop while handler is active: %v", err) } - } - select { - case <-d.Done(): - t.Fatal("Done closed before synchronous handler returned; owner cleanup could race proxy registration") - case <-time.After(100 * time.Millisecond): - } - releaseOnce.Do(func() { close(release) }) - regressionAwait(t, d.Done(), "completion after handler returned") + // Let the writer finish stopping while the handler remains blocked. + synctest.Wait() + select { + case <-d.Done(): + t.Fatal("Done closed before synchronous handler returned; owner cleanup could race proxy registration") + default: + } + releaseOnce.Do(func() { close(release) }) + regressionAwait(t, d.Done(), "completion after handler returned") + }) } func TestDispatcherWriteErrorUnblocksSenderWithFullQueue(t *testing.T) { @@ -335,53 +335,49 @@ func TestDispatcherWriteErrorUnblocksSenderWithFullQueue(t *testing.T) { } func TestDispatcherConcurrentReadWriteFailureClosesOnce(t *testing.T) { - for range 100 { - func() { - c := newRegressionControl(t) - regressionWarm(t, c) - defer c.transport.close() - readEntered, writeEntered := make(chan struct{}), make(chan struct{}) - release := make(chan struct{}) - var releaseOnce sync.Once - defer releaseOnce.Do(func() { close(release) }) - c.beforeRead = func() error { - close(readEntered) - <-release - return io.EOF - } - c.beforeWrite = func() { - close(writeEntered) - <-release - } - d := NewDispatcher(c) - c.transport.arm() - if err := d.Send(&ReqWorkConn{}); err != nil { - t.Fatal(err) - } - readExited, writeExited := make(chan struct{}), make(chan struct{}) - go func() { - defer close(readExited) - d.readLoop() - }() - go func() { - defer close(writeExited) - d.sendLoop() - }() - regressionAwait(t, readEntered, "read path before injected failure") - regressionAwait(t, writeEntered, "write path before injected failure") - releaseOnce.Do(func() { close(release) }) - regressionAwait(t, readExited, "failed read loop exit") - regressionAwait(t, writeExited, "failed write loop exit") - regressionAwait(t, d.Done(), "concurrent read/write shutdown") - if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { - t.Fatalf("concurrent WriteMsg failure: %v", err) - } - if _, failures, _ := c.transport.snapshot(); failures != 1 { - t.Fatalf("injected transport failed %d times", failures) - } - if c.closeCalls.Load() != 1 { - t.Fatalf("connection closed %d times", c.closeCalls.Load()) - } - }() + c := newRegressionControl(t) + regressionWarm(t, c) + defer c.transport.close() + readEntered, writeEntered := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(release) }) + c.beforeRead = func() error { + close(readEntered) + <-release + return io.EOF + } + c.beforeWrite = func() { + close(writeEntered) + <-release + } + d := NewDispatcher(c) + c.transport.arm() + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + readExited, writeExited := make(chan struct{}), make(chan struct{}) + go func() { + defer close(readExited) + d.readLoop() + }() + go func() { + defer close(writeExited) + d.sendLoop() + }() + regressionAwait(t, readEntered, "read path before injected failure") + regressionAwait(t, writeEntered, "write path before injected failure") + releaseOnce.Do(func() { close(release) }) + regressionAwait(t, readExited, "failed read loop exit") + regressionAwait(t, writeExited, "failed write loop exit") + regressionAwait(t, d.Done(), "concurrent read/write shutdown") + if err := regressionReceiveWrite(t, c); !errors.Is(err, errRegressionWriteTimeout) { + t.Fatalf("concurrent WriteMsg failure: %v", err) + } + if _, failures, _ := c.transport.snapshot(); failures != 1 { + t.Fatalf("injected transport failed %d times", failures) + } + if c.closeCalls.Load() != 1 { + t.Fatalf("connection closed %d times", c.closeCalls.Load()) } } diff --git a/pkg/msg/handler_yamux_test.go b/pkg/msg/handler_yamux_test.go new file mode 100644 index 00000000..25d91a9e --- /dev/null +++ b/pkg/msg/handler_yamux_test.go @@ -0,0 +1,225 @@ +// Copyright 2026 The frp Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package msg + +import ( + "errors" + "io" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + fmux "github.com/fatedier/yamux" + + "github.com/fatedier/frp/pkg/proto/wire" + netpkg "github.com/fatedier/frp/pkg/util/net" +) + +// Delay one real TCP write without injecting an error. The stream's write +// failure must come from yamux's ConnectionWriteTimeout. +type delayedTCPConn struct { + net.Conn + armed atomic.Bool + entered chan struct{} + release chan struct{} +} + +func (c *delayedTCPConn) Write(p []byte) (int, error) { + if c.armed.Swap(false) { + close(c.entered) + <-c.release + } + return c.Conn.Write(p) +} + +type observedYamuxConn struct { + net.Conn + writes chan error + readEntered chan struct{} + readOnce sync.Once + closed chan struct{} + closeCalls atomic.Int32 +} + +func (c *observedYamuxConn) Read(p []byte) (int, error) { + c.readOnce.Do(func() { close(c.readEntered) }) + return c.Conn.Read(p) +} + +func (c *observedYamuxConn) Write(p []byte) (int, error) { + n, err := c.Conn.Write(p) + c.writes <- err + return n, err +} + +func (c *observedYamuxConn) Close() error { + c.closeCalls.Add(1) + err := c.Conn.Close() + close(c.closed) + return err +} + +func newLoopbackYamuxPair(t *testing.T) (*fmux.Session, *fmux.Session, *delayedTCPConn) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + clientTCP, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { clientTCP.Close() }) + serverTCP, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { serverTCP.Close() }) + gate := &delayedTCPConn{Conn: clientTCP, entered: make(chan struct{}), release: make(chan struct{})} + config := fmux.DefaultConfig() + config.ConnectionWriteTimeout = 80 * time.Millisecond + config.LogOutput = io.Discard + client, err := fmux.Client(gate, config) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { client.Close() }) + server, err := fmux.Server(serverTCP, config) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { server.Close() }) + return client, server, gate +} + +func loopbackCryptoConn(t *testing.T, conn net.Conn, protocol string, role netpkg.AEADCryptoRole) *Conn { + t.Helper() + var rw io.ReadWriter + var err error + if protocol == wire.ProtocolV2 { + rw, err = netpkg.NewAEADCryptoReadWriter( + conn, regressionKey, role, wire.AEADAlgorithmAES256GCM, []byte("isolated-test-transcript")) + } else { + rw, err = netpkg.NewCryptoReadWriter(conn, regressionKey) + } + if err != nil { + t.Fatal(err) + } + return NewConn(conn, NewReadWriter(rw, protocol)) +} + +func loopbackControlPair(t *testing.T, client, server *fmux.Session, protocol string) (*Conn, *Conn, *observedYamuxConn) { + t.Helper() + local, err := client.OpenStream() + if err != nil { + t.Fatal(err) + } + remote, err := server.AcceptStream() + if err != nil { + t.Fatal(err) + } + observed := &observedYamuxConn{ + Conn: local, writes: make(chan error, 16), readEntered: make(chan struct{}), closed: make(chan struct{}), + } + return loopbackCryptoConn(t, observed, protocol, netpkg.AEADCryptoRoleClient), + loopbackCryptoConn(t, remote, protocol, netpkg.AEADCryptoRoleServer), observed +} + +func loopbackReadReq(t *testing.T, conn *Conn) { + t.Helper() + if err := conn.SetReadDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + defer conn.SetReadDeadline(time.Time{}) + m, err := conn.ReadMsg() + if err != nil { + t.Fatal(err) + } + if _, ok := m.(*ReqWorkConn); !ok { + t.Fatalf("decoded %T, want ReqWorkConn", m) + } +} + +func TestDispatcherYamuxWriteTimeoutInterruptsRead(t *testing.T) { + for _, protocol := range []string{wire.ProtocolV1, wire.ProtocolV2} { + t.Run(protocol, func(t *testing.T) { + client, server, gate := newLoopbackYamuxPair(t) + var releaseOnce sync.Once + defer releaseOnce.Do(func() { close(gate.release) }) + control, peer, observed := loopbackControlPair(t, client, server, protocol) + if err := control.WriteMsg(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + loopbackReadReq(t, peer) + // Warm-up has completed, including all crypto header writes. + for len(observed.writes) > 0 { + <-observed.writes + } + d := NewDispatcher(control) + readExited := make(chan struct{}) + go func() { + defer close(readExited) + d.readLoop() + }() + regressionAwait(t, observed.readEntered, "control reader entry") + gate.armed.Store(true) + // Queue before starting the writer so this Send cannot race its failure. + if err := d.Send(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + go d.sendLoop() + regressionAwait(t, gate.entered, "TCP write delay barrier") + select { + case err := <-observed.writes: + if !errors.Is(err, fmux.ErrConnectionWriteTimeout) { + t.Fatalf("stream write error: %v, want yamux connection write timeout", err) + } + t.Logf("production yamux write failed: %v", err) + case <-time.After(time.Second): + t.Fatal("yamux write did not time out") + } + regressionAwait(t, d.stopCh, "write failure stops sends") + if err := d.Send(&ReqWorkConn{}); !errors.Is(err, io.EOF) { + t.Fatalf("Send after stop: %v, want EOF", err) + } + releaseOnce.Do(func() { close(gate.release) }) + regressionAwait(t, observed.closed, "control Close returns") + // peer never closes its write half: local yamux Close alone cannot + // interrupt Read. Neither a peer FIN nor session cleanup may help. + regressionAwait(t, d.Done(), "Done without peer FIN") + regressionAwait(t, readExited, "control readLoop exit without peer FIN") + if observed.closeCalls.Load() != 1 { + t.Fatalf("control closed %d times, want once", observed.closeCalls.Load()) + } + if client.IsClosed() || server.IsClosed() { + t.Fatal("yamux session unexpectedly closed") + } + fresh, freshPeer, freshObserved := loopbackControlPair(t, client, server, protocol) + for range 5 { + if err := fresh.WriteMsg(&ReqWorkConn{}); err != nil { + t.Fatal(err) + } + loopbackReadReq(t, freshPeer) + for len(freshObserved.writes) > 0 { + <-freshObserved.writes + } + } + t.Log("Done/readLoop exited without peer FIN; Close once; same yamux session decrypted five fresh messages") + }) + } +} From ab0959868ad8848aa15861f66f6fb30f0814f529 Mon Sep 17 00:00:00 2001 From: dhairya Date: Fri, 9 Oct 2026 17:43:03 +0530 Subject: [PATCH 16/23] fix: rotate daily counters by calendar date across DST (#5555) * Fix daily counter rotation across daylight-saving changes * Test daily counter rotation across DST and skipped midnight --- pkg/util/metric/date_counter.go | 11 +++-- pkg/util/metric/date_counter_test.go | 68 ++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/pkg/util/metric/date_counter.go b/pkg/util/metric/date_counter.go index 5e6b889f..444db3c2 100644 --- a/pkg/util/metric/date_counter.go +++ b/pkg/util/metric/date_counter.go @@ -58,7 +58,7 @@ func newStandardDateCounterWithClock(reserveDays int64, clk clock.PassiveClock) reserveDays: reserveDays, counts: make([]int64, reserveDays), clock: clk, - lastUpdateDate: startOfDay(clk.Now()), + lastUpdateDate: calendarDate(clk.Now()), } } @@ -114,7 +114,7 @@ func (c *StandardDateCounter) Clear() { // rotate // Must hold the lock before calling this function. func (c *StandardDateCounter) rotate(now time.Time) { - now = startOfDay(now) + now = calendarDate(now) days := int(now.Sub(c.lastUpdateDate).Hours() / 24) reserveDays := int(c.reserveDays) @@ -132,7 +132,8 @@ func (c *StandardDateCounter) rotate(now time.Time) { c.lastUpdateDate = now } -// startOfDay returns midnight in t's location. -func startOfDay(t time.Time) time.Time { - return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) +// calendarDate represents t's local date as midnight UTC for day arithmetic. +// This avoids variable day lengths and nonexistent local midnights during DST. +func calendarDate(t time.Time) time.Time { + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) } diff --git a/pkg/util/metric/date_counter_test.go b/pkg/util/metric/date_counter_test.go index cbd29386..94830452 100644 --- a/pkg/util/metric/date_counter_test.go +++ b/pkg/util/metric/date_counter_test.go @@ -31,7 +31,7 @@ func TestDateCounter(t *testing.T) { func TestDateCounterRotate(t *testing.T) { loc := time.FixedZone("test", 8*60*60) - lastUpdateDate := time.Date(2026, time.May, 8, 0, 0, 0, 0, loc) + lastUpdateDate := time.Date(2026, time.May, 8, 0, 0, 0, 0, time.UTC) tests := []struct { name string @@ -55,19 +55,19 @@ func TestDateCounterRotate(t *testing.T) { name: "one day", now: time.Date(2026, time.May, 9, 12, 30, 0, 0, loc), want: []int64{0, 10, 7}, - wantLastUpdateDate: time.Date(2026, time.May, 9, 0, 0, 0, 0, loc), + wantLastUpdateDate: time.Date(2026, time.May, 9, 0, 0, 0, 0, time.UTC), }, { name: "two days", now: time.Date(2026, time.May, 10, 12, 30, 0, 0, loc), want: []int64{0, 0, 10}, - wantLastUpdateDate: time.Date(2026, time.May, 10, 0, 0, 0, 0, loc), + wantLastUpdateDate: time.Date(2026, time.May, 10, 0, 0, 0, 0, time.UTC), }, { name: "all reserved days elapsed", now: time.Date(2026, time.May, 11, 12, 30, 0, 0, loc), want: []int64{0, 0, 0}, - wantLastUpdateDate: time.Date(2026, time.May, 11, 0, 0, 0, 0, loc), + wantLastUpdateDate: time.Date(2026, time.May, 11, 0, 0, 0, 0, time.UTC), }, } @@ -132,3 +132,63 @@ func TestDateCounterConcurrentAccess(t *testing.T) { } wg.Wait() } + +func TestDateCounterCalendarDaysAcrossDST(t *testing.T) { + loc, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + midnightLoc, err := time.LoadLocation("America/Sao_Paulo") + require.NoError(t, err) + tests := []struct { + name string + start time.Time + end time.Time + want []int64 + }{ + { + name: "spring next day", + start: time.Date(2026, time.March, 8, 12, 0, 0, 0, loc), + end: time.Date(2026, time.March, 9, 12, 0, 0, 0, loc), + want: []int64{0, 10, 0}, + }, + { + name: "spring two days", + start: time.Date(2026, time.March, 8, 12, 0, 0, 0, loc), + end: time.Date(2026, time.March, 10, 12, 0, 0, 0, loc), + want: []int64{0, 0, 10}, + }, + { + name: "spring retention expiry", + start: time.Date(2026, time.March, 8, 12, 0, 0, 0, loc), + end: time.Date(2026, time.March, 11, 12, 0, 0, 0, loc), + want: []int64{0, 0, 0}, + }, + { + name: "fall next day", + start: time.Date(2026, time.November, 1, 12, 0, 0, 0, loc), + end: time.Date(2026, time.November, 2, 12, 0, 0, 0, loc), + want: []int64{0, 10, 0}, + }, + { + name: "skipped midnight", + start: time.Date(2018, time.November, 3, 12, 0, 0, 0, midnightLoc), + end: time.Date(2018, time.November, 4, 12, 0, 0, 0, midnightLoc), + want: []int64{0, 10, 0}, + }, + { + name: "day after skipped midnight", + start: time.Date(2018, time.November, 4, 12, 0, 0, 0, midnightLoc), + end: time.Date(2018, time.November, 5, 12, 0, 0, 0, midnightLoc), + want: []int64{0, 10, 0}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clk := clocktesting.NewFakeClock(tt.start) + dc := newStandardDateCounterWithClock(3, clk) + dc.Inc(10) + clk.SetTime(tt.end) + require.Equal(t, tt.want[0], dc.TodayCount()) + require.Equal(t, tt.want, dc.GetLastDaysCount(3)) + }) + } +} From 9c7d9898b319215823274f40fa70bb12f0aa99c8 Mon Sep 17 00:00:00 2001 From: dhairya Date: Fri, 9 Oct 2026 20:37:48 +0530 Subject: [PATCH 17/23] fix(vhost): preserve buffered HTTP CONNECT tunnel data (#5556) * fix(vhost): preserve buffered HTTP CONNECT tunnel data * test(vhost): cover pipelined CONNECT data and bidirectional tunnel traffic --- pkg/util/vhost/http.go | 4 +- pkg/util/vhost/http_test.go | 80 +++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/pkg/util/vhost/http.go b/pkg/util/vhost/http.go index 05960fdc..33b11223 100644 --- a/pkg/util/vhost/http.go +++ b/pkg/util/vhost/http.go @@ -212,7 +212,7 @@ func (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Req return } - client, _, err := hj.Hijack() + client, buffered, err := hj.Hijack() if err != nil { rw.WriteHeader(http.StatusInternalServerError) return @@ -225,7 +225,7 @@ func (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Req return } _ = req.Write(remote) - go libio.Join(remote, client) + go libio.Join(remote, libio.WrapReadWriteCloser(buffered.Reader, client, client.Close)) } func getRequestRouteUser(req *http.Request) string { diff --git a/pkg/util/vhost/http_test.go b/pkg/util/vhost/http_test.go index a6493b08..e993da48 100644 --- a/pkg/util/vhost/http_test.go +++ b/pkg/util/vhost/http_test.go @@ -3,6 +3,7 @@ package vhost import ( "bufio" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -12,8 +13,87 @@ import ( "github.com/stretchr/testify/require" httppkg "github.com/fatedier/frp/pkg/util/http" + netpkg "github.com/fatedier/frp/pkg/util/net" ) +func TestHTTPConnectTunnel(t *testing.T) { + for _, pipelined := range []bool{false, true} { + t.Run(fmt.Sprintf("pipelined=%v", pipelined), func(t *testing.T) { + remote, backend := net.Pipe() + defer remote.Close() + defer backend.Close() + require.NoError(t, backend.SetDeadline(time.Now().Add(5*time.Second))) + + rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) + require.NoError(t, rp.Register(RouteConfig{ + Domain: "target.example.com", + CreateConnFn: func(string) (net.Conn, error) { + return remote, nil + }, + })) + listener := netpkg.NewInternalListener() + server := &http.Server{Handler: rp, ReadHeaderTimeout: time.Second} + serveErr := make(chan error, 1) + go func() { + serveErr <- server.Serve(listener) + }() + defer func() { + require.NoError(t, server.Close()) + require.ErrorIs(t, <-serveErr, http.ErrServerClosed) + }() + + client, incoming := net.Pipe() + defer client.Close() + defer incoming.Close() + require.NoError(t, client.SetDeadline(time.Now().Add(5*time.Second))) + require.NoError(t, listener.PutConn(incoming)) + + request := "CONNECT target.example.com:443 HTTP/1.1\r\nHost: target.example.com:443\r\n\r\n" + if pipelined { + // One pipe write puts the tunnel data in net/http's request buffer. + request += "early data" + } + _, err := io.WriteString(client, request) + require.NoError(t, err) + backendReader := bufio.NewReader(backend) + req, err := http.ReadRequest(backendReader) + require.NoError(t, err) + defer req.Body.Close() + require.Equal(t, http.MethodConnect, req.Method) + require.Equal(t, "target.example.com:443", req.Host) + if pipelined { + data := make([]byte, len("early data")) + _, err = io.ReadFull(backendReader, data) + require.NoError(t, err) + require.Equal(t, "early data", string(data)) + } + + writeErr := make(chan error, 1) + go func() { + _, err := io.WriteString(backend, "HTTP/1.1 200 Connection Established\r\n\r\nreply") + writeErr <- err + }() + clientReader := bufio.NewReader(client) + response, err := http.ReadResponse(clientReader, req) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + reply := make([]byte, len("reply")) + _, err = io.ReadFull(clientReader, reply) + require.NoError(t, err) + require.Equal(t, "reply", string(reply)) + require.NoError(t, <-writeErr) + + _, err = io.WriteString(client, "later data") + require.NoError(t, err) + data := make([]byte, len("later data")) + _, err = io.ReadFull(backendReader, data) + require.NoError(t, err) + require.Equal(t, "later data", string(data)) + }) + } +} + func TestHTTPServerProtocols(t *testing.T) { rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) protocols := new(http.Protocols) From 1cc2dd8a23dfc070563753cbe01359a9217917f5 Mon Sep 17 00:00:00 2001 From: fatedier Date: Fri, 9 Oct 2026 23:53:05 +0800 Subject: [PATCH 18/23] fix(vhost): avoid request body access after CONNECT hijack (#5562) --- pkg/util/vhost/http.go | 22 ++++- pkg/util/vhost/http_test.go | 181 ++++++++++++++++++++++++++++++++++-- 2 files changed, 196 insertions(+), 7 deletions(-) diff --git a/pkg/util/vhost/http.go b/pkg/util/vhost/http.go index 33b11223..9394e913 100644 --- a/pkg/util/vhost/http.go +++ b/pkg/util/vhost/http.go @@ -206,12 +206,28 @@ func checkRouteAuthByRequest(req *http.Request, rc *RouteConfig) bool { } func (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Request) { + if req.ContentLength > 0 || len(req.TransferEncoding) > 0 { + http.Error(rw, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + hj, ok := rw.(http.Hijacker) if !ok { rw.WriteHeader(http.StatusInternalServerError) return } + // Forward only CONNECT headers; the original Body must not be used after Hijack. + outreq := req.Clone(req.Context()) + outreq.Body = nil + outreq.GetBody = nil + outreq.ContentLength = 0 + outreq.TransferEncoding = nil + outreq.Trailer = nil + outreq.Header.Del("Content-Length") + outreq.Header.Del("Transfer-Encoding") + outreq.Header.Del("Trailer") + client, buffered, err := hj.Hijack() if err != nil { rw.WriteHeader(http.StatusInternalServerError) @@ -224,7 +240,11 @@ func (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Req client.Close() return } - _ = req.Write(remote) + if err := outreq.Write(remote); err != nil { + _ = remote.Close() + _ = client.Close() + return + } go libio.Join(remote, libio.WrapReadWriteCloser(buffered.Reader, client, client.Close)) } diff --git a/pkg/util/vhost/http_test.go b/pkg/util/vhost/http_test.go index e993da48..e661ca08 100644 --- a/pkg/util/vhost/http_test.go +++ b/pkg/util/vhost/http_test.go @@ -7,6 +7,8 @@ import ( "net" "net/http" "net/http/httptest" + "strings" + "sync/atomic" "testing" "time" @@ -17,8 +19,17 @@ import ( ) func TestHTTPConnectTunnel(t *testing.T) { - for _, pipelined := range []bool{false, true} { - t.Run(fmt.Sprintf("pipelined=%v", pipelined), func(t *testing.T) { + for _, tc := range []struct { + name string + framing string + pipelined bool + }{ + {name: "plain"}, + {name: "plain-early", pipelined: true}, + {name: "cl0", framing: "Content-Length: 0\r\n"}, + {name: "cl0-early", framing: "Content-Length: 0\r\n", pipelined: true}, + } { + t.Run(tc.name, func(t *testing.T) { remote, backend := net.Pipe() defer remote.Close() defer backend.Close() @@ -26,7 +37,9 @@ func TestHTTPConnectTunnel(t *testing.T) { rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) require.NoError(t, rp.Register(RouteConfig{ - Domain: "target.example.com", + Domain: "target.example.com", + Username: "alice", + Password: "secret", CreateConnFn: func(string) (net.Conn, error) { return remote, nil }, @@ -48,8 +61,10 @@ func TestHTTPConnectTunnel(t *testing.T) { require.NoError(t, client.SetDeadline(time.Now().Add(5*time.Second))) require.NoError(t, listener.PutConn(incoming)) - request := "CONNECT target.example.com:443 HTTP/1.1\r\nHost: target.example.com:443\r\n\r\n" - if pipelined { + proxyAuth := httppkg.BasicAuth("alice", "secret") + request := "CONNECT target.example.com:443 HTTP/1.1\r\nHost: target.example.com:443\r\n" + + "X-Connect-Test: tunnel\r\nProxy-Authorization: " + proxyAuth + "\r\n" + tc.framing + "\r\n" + if tc.pipelined { // One pipe write puts the tunnel data in net/http's request buffer. request += "early data" } @@ -61,7 +76,12 @@ func TestHTTPConnectTunnel(t *testing.T) { defer req.Body.Close() require.Equal(t, http.MethodConnect, req.Method) require.Equal(t, "target.example.com:443", req.Host) - if pipelined { + require.Equal(t, "target.example.com:443", req.URL.Host) + require.Equal(t, "tunnel", req.Header.Get("X-Connect-Test")) + require.Equal(t, proxyAuth, req.Header.Get("Proxy-Authorization")) + require.Zero(t, req.ContentLength) + require.Empty(t, req.TransferEncoding) + if tc.pipelined { data := make([]byte, len("early data")) _, err = io.ReadFull(backendReader, data) require.NoError(t, err) @@ -94,6 +114,155 @@ func TestHTTPConnectTunnel(t *testing.T) { } } +func TestHTTPConnectRejectsContent(t *testing.T) { + for _, tc := range []struct { + name string + framing string + body string + }{ + {name: "content-length", framing: "Content-Length: 1\r\n", body: "X"}, + {name: "chunked", framing: "Transfer-Encoding: chunked\r\n", body: "1\r\nX\r\n0\r\n\r\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + backendCalls := 0 + rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) + require.NoError(t, rp.Register(RouteConfig{ + Domain: "target.example.com", + CreateConnFn: func(string) (net.Conn, error) { + backendCalls++ + return nil, io.ErrClosedPipe + }, + })) + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader( + "CONNECT target.example.com:443 HTTP/1.1\r\nHost: target.example.com:443\r\n" + tc.framing + "\r\n" + tc.body))) + require.NoError(t, err) + defer req.Body.Close() + rw := &connectTestResponseWriter{ResponseRecorder: httptest.NewRecorder()} + + rp.ServeHTTP(rw, req) + + require.Equal(t, http.StatusBadRequest, rw.Code) + require.Zero(t, rw.hijackCalls, "reject content before Hijack") + require.Zero(t, backendCalls, "reject content before creating a backend") + }) + } +} + +func TestHTTPConnectHeaderWriteFailure(t *testing.T) { + for _, originalBody := range []bool{false, true} { + t.Run(fmt.Sprintf("originalBody=%v", originalBody), func(t *testing.T) { + client, incoming := net.Pipe() + defer client.Close() + defer incoming.Close() + remote, backend := net.Pipe() + defer remote.Close() + defer backend.Close() + clientConn := &connectTestConn{Conn: incoming} + remoteConn := &connectTestConn{Conn: remote, failWrites: true} + rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) + require.NoError(t, rp.Register(RouteConfig{ + Domain: "target.example.com", + CreateConnFn: func(string) (net.Conn, error) { + return remoteConn, nil + }, + })) + req := rp.injectRequestInfoToCtx(httptest.NewRequest(http.MethodConnect, "target.example.com:443", nil)) + require.Equal(t, "target.example.com:443", req.Host) + body := &connectTestBody{} + if originalBody { + req.Body = body + } + requestBody := req.Body + getBodyCalls := 0 + req.GetBody = func() (io.ReadCloser, error) { + getBodyCalls++ + return body, nil + } + req.Header.Set("Content-Length", "0") + req.Header.Set("Trailer", "X-Connect-Trailer") + req.Trailer = http.Header{"X-Connect-Trailer": {"value"}} + originalHeader, originalTrailer := req.Header.Clone(), req.Trailer.Clone() + rw := &connectTestResponseWriter{ + ResponseRecorder: httptest.NewRecorder(), + conn: clientConn, + buffered: bufio.NewReadWriter(bufio.NewReader(clientConn), bufio.NewWriter(clientConn)), + } + + rp.connectHandler(rw, req) + + require.Equal(t, 1, rw.hijackCalls) + require.Zero(t, body.reads, "do not read the original Body after Hijack") + require.Zero(t, body.closes, "do not close the original Body after Hijack") + require.Positive(t, remoteConn.writes.Load(), "exercise CONNECT header write failure") + require.True(t, remoteConn.closed.Load(), "close backend on CONNECT header write failure") + require.True(t, clientConn.closed.Load(), "close client on CONNECT header write failure") + require.Zero(t, remoteConn.reads.Load(), "do not start Join after header write failure") + require.Zero(t, clientConn.reads.Load(), "do not start Join after header write failure") + require.Zero(t, getBodyCalls) + require.Equal(t, requestBody, req.Body) + require.NotNil(t, req.GetBody) + require.Equal(t, originalHeader, req.Header) + require.Equal(t, originalTrailer, req.Trailer) + }) + } +} + +type connectTestResponseWriter struct { + *httptest.ResponseRecorder + conn net.Conn + buffered *bufio.ReadWriter + hijackCalls int +} + +func (rw *connectTestResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + rw.hijackCalls++ + if rw.conn == nil { + return nil, nil, io.ErrClosedPipe + } + return rw.conn, rw.buffered, nil +} + +type connectTestConn struct { + net.Conn + failWrites bool + closed atomic.Bool + reads atomic.Int32 + writes atomic.Int32 +} + +func (c *connectTestConn) Read(p []byte) (int, error) { + c.reads.Add(1) + return c.Conn.Read(p) +} + +func (c *connectTestConn) Write(p []byte) (int, error) { + c.writes.Add(1) + if c.failWrites { + return 0, io.ErrClosedPipe + } + return c.Conn.Write(p) +} + +func (c *connectTestConn) Close() error { + c.closed.Store(true) + return c.Conn.Close() +} + +type connectTestBody struct { + reads int + closes int +} + +func (b *connectTestBody) Read([]byte) (int, error) { + b.reads++ + return 0, io.ErrUnexpectedEOF +} + +func (b *connectTestBody) Close() error { + b.closes++ + return nil +} + func TestHTTPServerProtocols(t *testing.T) { rp := NewHTTPReverseProxy(HTTPReverseProxyOptions{}, NewRouters()) protocols := new(http.Protocols) From 71b31978b3685c355165f9bf1a3584036b8097e5 Mon Sep 17 00:00:00 2001 From: Mxmilu Date: Sat, 10 Oct 2026 10:45:34 +0800 Subject: [PATCH 19/23] fix(server): close underlying work conn when server bandwidth limiter is enabled (#5553) When bandwidthLimitMode is "server", the work connection is wrapped by libio.WrapReadWriteCloser with a close callback that captured the variable being reassigned (local/rwc). At call time the callback referenced the wrapper itself, whose closed guard was already set, so Close returned nil without closing the underlying work connection. For TCP proxies this left libio.Join blocked on the other copy goroutine after the user disconnected, keeping the work connection and the backend connection open. HTTP and UDP proxies had the same pattern. Bind the Close method value before reassignment, and add an e2e test verifying that the local connection is closed after the user disconnects. --- server/proxy/http.go | 4 +- server/proxy/proxy.go | 4 +- server/proxy/udp.go | 4 +- test/e2e/v1/features/bandwidth_limit.go | 81 +++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/server/proxy/http.go b/server/proxy/http.go index 05afc2e9..c3da8383 100644 --- a/server/proxy/http.go +++ b/server/proxy/http.go @@ -139,9 +139,7 @@ func (pxy *HTTPProxy) GetRealConn(remoteAddr string) (workConn net.Conn, err err } if pxy.GetLimiter() != nil { - rwc = libio.WrapReadWriteCloser(limit.NewReader(rwc, pxy.GetLimiter()), limit.NewWriter(rwc, pxy.GetLimiter()), func() error { - return rwc.Close() - }) + rwc = libio.WrapReadWriteCloser(limit.NewReader(rwc, pxy.GetLimiter()), limit.NewWriter(rwc, pxy.GetLimiter()), rwc.Close) } workConn = netpkg.WrapReadWriteCloserToConn(rwc, tmpConn) diff --git a/server/proxy/proxy.go b/server/proxy/proxy.go index f4e75f0a..705aa9da 100644 --- a/server/proxy/proxy.go +++ b/server/proxy/proxy.go @@ -308,9 +308,7 @@ func (pxy *BaseProxy) handleUserTCPConnection(userConn net.Conn) { } if pxy.GetLimiter() != nil { - local = libio.WrapReadWriteCloser(limit.NewReader(local, pxy.GetLimiter()), limit.NewWriter(local, pxy.GetLimiter()), func() error { - return local.Close() - }) + local = libio.WrapReadWriteCloser(limit.NewReader(local, pxy.GetLimiter()), limit.NewWriter(local, pxy.GetLimiter()), local.Close) } xl.Debugf("join connections, workConn(l[%s] r[%s]) userConn(l[%s] r[%s])", workConn.LocalAddr().String(), diff --git a/server/proxy/udp.go b/server/proxy/udp.go index 47bdfc8c..8a69c360 100644 --- a/server/proxy/udp.go +++ b/server/proxy/udp.go @@ -217,9 +217,7 @@ func (pxy *UDPProxy) Run() (remoteAddr string, err error) { } if pxy.GetLimiter() != nil { - rwc = libio.WrapReadWriteCloser(limit.NewReader(rwc, pxy.GetLimiter()), limit.NewWriter(rwc, pxy.GetLimiter()), func() error { - return rwc.Close() - }) + rwc = libio.WrapReadWriteCloser(limit.NewReader(rwc, pxy.GetLimiter()), limit.NewWriter(rwc, pxy.GetLimiter()), rwc.Close) } pxy.workConn = netpkg.WrapReadWriteCloserToConn(rwc, workConn) diff --git a/test/e2e/v1/features/bandwidth_limit.go b/test/e2e/v1/features/bandwidth_limit.go index 11503adb..a975bfe5 100644 --- a/test/e2e/v1/features/bandwidth_limit.go +++ b/test/e2e/v1/features/bandwidth_limit.go @@ -2,6 +2,9 @@ package features import ( "fmt" + "io" + "net" + "strconv" "strings" "time" @@ -105,4 +108,82 @@ var _ = ginkgo.Describe("[Feature: Bandwidth Limit]", func() { framework.ExpectTrue(duration.Seconds() > 8, "100Kb with 10KB limit, want > 8 seconds, but got %s", duration.String()) }) + + ginkgo.It("Server Bandwidth Limit closes work connection when user disconnects", func() { + newFunc := func() *plugin.Request { + var r plugin.Request + r.Content = &plugin.NewProxyContent{} + return &r + } + pluginPort := f.AllocPort() + handler := func(req *plugin.Request) *plugin.Response { + var ret plugin.Response + content := req.Content.(*plugin.NewProxyContent) + content.BandwidthLimit = "10MB" + content.BandwidthLimitMode = "server" + ret.Content = content + return &ret + } + pluginServer := pluginpkg.NewHTTPPluginServer(pluginPort, newFunc, handler, nil) + + f.RunServer("", pluginServer) + + serverConf := consts.DefaultServerConfig + fmt.Sprintf(` + [[httpPlugins]] + name = "test" + addr = "127.0.0.1:%d" + path = "/handler" + ops = ["NewProxy"] + `, pluginPort) + clientConf := consts.DefaultClientConfig + + // The local server echoes the first message, then stays idle until the + // connection from frpc is closed. + localClosed := make(chan struct{}) + localPort := f.AllocPort() + localServer := streamserver.New(streamserver.TCP, streamserver.WithBindPort(localPort), + streamserver.WithCustomHandler(func(c net.Conn) { + defer c.Close() + buf := make([]byte, 1024) + n, err := c.Read(buf) + if err != nil { + return + } + if _, err := c.Write(buf[:n]); err != nil { + return + } + _, _ = io.Copy(io.Discard, c) + close(localClosed) + })) + f.RunServer("", localServer) + + remotePort := f.AllocPort() + clientConf += fmt.Sprintf(` + [[proxies]] + name = "tcp" + type = "tcp" + localPort = %d + remotePort = %d + `, localPort, remotePort) + + f.RunProcesses(serverConf, []string{clientConf}) + + conn, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(remotePort))) + framework.ExpectNoError(err) + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + _, err = conn.Write([]byte("ping")) + framework.ExpectNoError(err) + buf := make([]byte, 4) + _, err = io.ReadFull(conn, buf) + framework.ExpectNoError(err) + framework.ExpectEqual(string(buf), "ping") + + conn.Close() + + select { + case <-localClosed: + case <-time.After(5 * time.Second): + framework.Failf("local connection is not closed after user connection is closed") + } + }) }) From c4d2db417749c721c6ec5b630e517a11a3ecb457 Mon Sep 17 00:00:00 2001 From: Abdellatif Anaflous <62770500+hktitof@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:19:59 +0100 Subject: [PATCH 20/23] fix(websocket): release a pending Accept when the listener is closed (#5552) the websocket listener kept its accept channel open on Close, so an Accept call that was already waiting blocked forever every other listener in pkg/util/net closes the channel and returns an error, but this one never did, so the HandleListener goroutine stayed stuck after the server closed i close the channel on Close, guard the hand off send so a late connection does not panic, and return ErrWebsocketListenerClosed which was declared but never reached Signed-off-by: Abdel <62770500+hktitof@users.noreply.github.com> --- pkg/util/net/websocket.go | 24 ++++++++++- pkg/util/net/websocket_test.go | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 pkg/util/net/websocket_test.go diff --git a/pkg/util/net/websocket.go b/pkg/util/net/websocket.go index 57641b31..0124fc83 100644 --- a/pkg/util/net/websocket.go +++ b/pkg/util/net/websocket.go @@ -4,8 +4,10 @@ import ( "errors" "net" "net/http" + "sync" "time" + gerr "github.com/fatedier/golib/errors" "golang.org/x/net/websocket" ) @@ -20,6 +22,9 @@ type WebsocketListener struct { acceptCh chan net.Conn server *http.Server + + mu sync.Mutex + closed bool } // NewWebsocketListener to handle websocket connections @@ -41,7 +46,15 @@ func NewWebsocketListener(ln net.Listener) (wl *WebsocketListener) { conn := WrapCloseNotifyConn(c, func(_ error) { close(notifyCh) }) - wl.acceptCh <- conn + // The listener may be closed while this connection is being handed + // over, so the send has to tolerate a closed channel. A nil error + // means the connection was accepted and is owned by the caller. + if err := gerr.PanicToError(func() { + wl.acceptCh <- conn + }); err != nil { + conn.Close() + return + } <-notifyCh })) @@ -66,6 +79,15 @@ func (p *WebsocketListener) Accept() (net.Conn, error) { } func (p *WebsocketListener) Close() error { + p.mu.Lock() + if !p.closed { + p.closed = true + // Closing acceptCh releases a pending Accept, matching the other + // listeners in this package. Otherwise it blocks forever and the + // caller never learns that the listener is closed. + close(p.acceptCh) + } + p.mu.Unlock() return p.server.Close() } diff --git a/pkg/util/net/websocket_test.go b/pkg/util/net/websocket_test.go new file mode 100644 index 00000000..f0f6b9d9 --- /dev/null +++ b/pkg/util/net/websocket_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 The frp Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package net + +import ( + "errors" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type closeStubListener struct { + closed chan struct{} +} + +func (l *closeStubListener) Accept() (net.Conn, error) { + <-l.closed + return nil, net.ErrClosed +} + +func (l *closeStubListener) Close() error { + select { + case <-l.closed: + default: + close(l.closed) + } + return nil +} + +func (l *closeStubListener) Addr() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} +} + +func TestWebsocketListenerAcceptUnblocksOnClose(t *testing.T) { + wl := NewWebsocketListener(&closeStubListener{closed: make(chan struct{})}) + + errCh := make(chan error, 1) + go func() { + _, err := wl.Accept() + errCh <- err + }() + + select { + case err := <-errCh: + t.Fatalf("Accept returned before the listener was closed: %v", err) + case <-time.After(100 * time.Millisecond): + } + + require.NoError(t, wl.Close()) + // Closing twice must stay safe and keep reporting the closed error. + require.NoError(t, wl.Close()) + + select { + case err := <-errCh: + require.True(t, errors.Is(err, ErrWebsocketListenerClosed), "got %v", err) + case <-time.After(time.Second): + t.Fatal("Accept did not return after the listener was closed") + } +} From 2542d11a6201c0f2bbe3b1d93031fc1e83201056 Mon Sep 17 00:00:00 2001 From: fatedier Date: Sat, 10 Oct 2026 19:58:05 +0800 Subject: [PATCH 21/23] fix(net): coordinate websocket handoff with listener shutdown (#5563) --- pkg/util/net/websocket.go | 39 ++++--- pkg/util/net/websocket_test.go | 182 +++++++++++++++++++++++++++------ 2 files changed, 171 insertions(+), 50 deletions(-) diff --git a/pkg/util/net/websocket.go b/pkg/util/net/websocket.go index 0124fc83..f3636ec0 100644 --- a/pkg/util/net/websocket.go +++ b/pkg/util/net/websocket.go @@ -7,7 +7,6 @@ import ( "sync" "time" - gerr "github.com/fatedier/golib/errors" "golang.org/x/net/websocket" ) @@ -20,11 +19,11 @@ const ( type WebsocketListener struct { ln net.Listener acceptCh chan net.Conn + done chan struct{} server *http.Server - mu sync.Mutex - closed bool + closeOnce sync.Once } // NewWebsocketListener to handle websocket connections @@ -33,6 +32,7 @@ func NewWebsocketListener(ln net.Listener) (wl *WebsocketListener) { wl = &WebsocketListener{ ln: ln, acceptCh: make(chan net.Conn), + done: make(chan struct{}), } muxer := http.NewServeMux() @@ -46,14 +46,13 @@ func NewWebsocketListener(ln net.Listener) (wl *WebsocketListener) { conn := WrapCloseNotifyConn(c, func(_ error) { close(notifyCh) }) - // The listener may be closed while this connection is being handed - // over, so the send has to tolerate a closed channel. A nil error - // means the connection was accepted and is owned by the caller. - if err := gerr.PanicToError(func() { - wl.acceptCh <- conn - }); err != nil { + select { + case <-wl.done: conn.Close() return + case wl.acceptCh <- conn: + // A ready handoff may win when Close races with it. Once delivered, + // the connection is owned by the caller of Accept. } <-notifyCh })) @@ -71,23 +70,21 @@ func NewWebsocketListener(ln net.Listener) (wl *WebsocketListener) { } func (p *WebsocketListener) Accept() (net.Conn, error) { - c, ok := <-p.acceptCh - if !ok { + select { + case <-p.done: return nil, ErrWebsocketListenerClosed + default: + } + select { + case <-p.done: + return nil, ErrWebsocketListenerClosed + case c := <-p.acceptCh: + return c, nil } - return c, nil } func (p *WebsocketListener) Close() error { - p.mu.Lock() - if !p.closed { - p.closed = true - // Closing acceptCh releases a pending Accept, matching the other - // listeners in this package. Otherwise it blocks forever and the - // caller never learns that the listener is closed. - close(p.acceptCh) - } - p.mu.Unlock() + p.closeOnce.Do(func() { close(p.done) }) return p.server.Close() } diff --git a/pkg/util/net/websocket_test.go b/pkg/util/net/websocket_test.go index f0f6b9d9..baa095ba 100644 --- a/pkg/util/net/websocket_test.go +++ b/pkg/util/net/websocket_test.go @@ -15,16 +15,25 @@ package net import ( + "bufio" "errors" + "io" "net" + "net/http" + "net/http/httptest" + "sync" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/require" + "golang.org/x/net/websocket" ) type closeStubListener struct { closed chan struct{} + once sync.Once + err error } func (l *closeStubListener) Accept() (net.Conn, error) { @@ -33,12 +42,8 @@ func (l *closeStubListener) Accept() (net.Conn, error) { } func (l *closeStubListener) Close() error { - select { - case <-l.closed: - default: - close(l.closed) - } - return nil + l.once.Do(func() { close(l.closed) }) + return l.err } func (l *closeStubListener) Addr() net.Addr { @@ -46,28 +51,147 @@ func (l *closeStubListener) Addr() net.Addr { } func TestWebsocketListenerAcceptUnblocksOnClose(t *testing.T) { - wl := NewWebsocketListener(&closeStubListener{closed: make(chan struct{})}) + synctest.Test(t, func(t *testing.T) { + wl := NewWebsocketListener(&closeStubListener{closed: make(chan struct{})}) + t.Cleanup(func() { wl.Close() }) + errCh := make(chan error, 3) + for range 3 { + go func() { + c, err := wl.Accept() + if c != nil { + t.Error("Accept returned a connection without a sender") + } + errCh <- err + }() + } + synctest.Wait() // All Accept calls are blocked before Close. + require.Empty(t, errCh) - errCh := make(chan error, 1) - go func() { - _, err := wl.Accept() - errCh <- err - }() - - select { - case err := <-errCh: - t.Fatalf("Accept returned before the listener was closed: %v", err) - case <-time.After(100 * time.Millisecond): - } - - require.NoError(t, wl.Close()) - // Closing twice must stay safe and keep reporting the closed error. - require.NoError(t, wl.Close()) - - select { - case err := <-errCh: - require.True(t, errors.Is(err, ErrWebsocketListenerClosed), "got %v", err) - case <-time.After(time.Second): - t.Fatal("Accept did not return after the listener was closed") - } + var wg sync.WaitGroup + for range 3 { + wg.Go(func() { + if err := wl.Close(); err != nil { + t.Errorf("Close: %v", err) + } + }) + } + wg.Wait() + synctest.Wait() + require.Len(t, errCh, 3) + for range 3 { + require.ErrorIs(t, <-errCh, ErrWebsocketListenerClosed) + } + c, err := wl.Accept() + require.Nil(t, c) + require.ErrorIs(t, err, ErrWebsocketListenerClosed) + }) +} + +func TestWebsocketListenerCloseError(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + closeErr := errors.New("listener close failed") + wl := NewWebsocketListener(&closeStubListener{closed: make(chan struct{}), err: closeErr}) + synctest.Wait() // Let Serve register its listener before Close. + require.ErrorIs(t, wl.Close(), closeErr) + require.NoError(t, wl.Close()) + }) +} + +type websocketPipeHijacker struct { + *httptest.ResponseRecorder + conn net.Conn + rw *bufio.ReadWriter +} + +func (h *websocketPipeHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return h.conn, h.rw, nil +} + +func newWebsocketPipe(t *testing.T) (*WebsocketListener, *websocket.Conn, net.Conn, <-chan struct{}) { + t.Helper() + wl := NewWebsocketListener(&closeStubListener{closed: make(chan struct{})}) + serverConn, clientConn := net.Pipe() + t.Cleanup(func() { + wl.Close() + serverConn.Close() + clientConn.Close() + }) + require.NoError(t, serverConn.SetDeadline(time.Now().Add(time.Second))) + require.NoError(t, clientConn.SetDeadline(time.Now().Add(time.Second))) + handlerDone := make(chan struct{}) + go func() { + defer close(handlerDone) + rw := bufio.NewReadWriter(bufio.NewReader(serverConn), bufio.NewWriter(serverConn)) + req, err := http.ReadRequest(rw.Reader) + if err != nil { + t.Errorf("ReadRequest: %v", err) + return + } + wl.server.Handler.ServeHTTP(&websocketPipeHijacker{httptest.NewRecorder(), serverConn, rw}, req) + }() + cfg, err := websocket.NewConfig("ws://localhost"+FrpWebsocketPath, "http://localhost") + require.NoError(t, err) + peer, err := websocket.NewClient(cfg, clientConn) + require.NoError(t, err) + return wl, peer, serverConn, handlerDone +} + +func TestWebsocketListenerUnacceptedConnectionClose(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + wl, peer, raw, handlerDone := newWebsocketPipe(t) + peerErr := make(chan error, 1) + go func() { + _, err := peer.Read(make([]byte, 1)) + peerErr <- err + }() + // No Accept consumer exists. Do not Wait here: it would synchronize + // the handler's send with Close and hide the baseline send/close race. + require.NoError(t, wl.Close()) + require.ErrorIs(t, <-peerErr, io.EOF) + <-handlerDone + _, err := raw.Read(make([]byte, 1)) + require.ErrorIs(t, err, io.ErrClosedPipe) + }) +} + +func TestWebsocketListenerAcceptedConnectionOwnership(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + wl, peer, raw, handlerDone := newWebsocketPipe(t) + owner, err := wl.Accept() + require.NoError(t, err) + t.Cleanup(func() { + raw.Close() + owner.Close() + }) + require.NoError(t, wl.Close()) + + payload := []byte{0, 255, 128, 42} + reply := []byte("reply") + gotReply := make([]byte, len(reply)) + peerErr := make(chan error, 2) + go func() { + if _, err := peer.Write(payload); err != nil { + peerErr <- err + return + } + _, err := io.ReadFull(peer, gotReply) + peerErr <- err + _, err = peer.Read(make([]byte, 1)) + peerErr <- err + }() + got := make([]byte, len(payload)) + _, err = io.ReadFull(owner, got) + require.NoError(t, err) + require.Equal(t, payload, got) + _, err = owner.Write(reply) + require.NoError(t, err) + require.NoError(t, <-peerErr) + require.Equal(t, reply, gotReply) + require.NoError(t, owner.Close()) + require.NoError(t, owner.Close()) + require.ErrorIs(t, <-peerErr, io.EOF) + <-handlerDone + _, err = raw.Read(make([]byte, 1)) + require.ErrorIs(t, err, io.ErrClosedPipe) + }) } From 12b0093ccfeb49fc5026e4b8ae8fb599c23b608b Mon Sep 17 00:00:00 2001 From: fatedier Date: Tue, 13 Oct 2026 10:46:29 +0800 Subject: [PATCH 22/23] chore: prepare v0.71.1 release notes (#5564) --- Release.md | 13 +++++++++++-- pkg/util/version/version.go | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Release.md b/Release.md index 68d7929d..35d0442c 100644 --- a/Release.md +++ b/Release.md @@ -1,4 +1,13 @@ +## Improvements + +* Building frp from source now requires Go 1.26.0 or later. + ## Fixes -* Fixed VirtualNet route lifecycle issues during reconnect and shutdown, including stale route cleanup, shutdown races, and reconnect backoff overflow. -* Fixed health check failure counts not resetting after a successful check, ensuring `healthCheck.maxFailed` applies to consecutive failures. +* Updated dependencies to fix SSH tunnel gateway channel deadlocks and panics caused by malformed challenges from NTLM-authenticated HTTP proxies. +* Fixed VirtualNet routing issues during reconnection and shutdown, and excessive retries after repeated connection failures. +* Health checks now reset the failure count after success, so `healthCheck.maxFailed` applies to consecutive failures. +* Fixed control connections getting stuck after message write failures, which could block reconnection. +* Fixed incorrect daily traffic statistics around daylight-saving transitions and skipped local midnights. +* Fixed early-data loss in HTTP virtual-host CONNECT tunnels. Requests with a positive content length or transfer encoding now return HTTP 400. +* Fixed work connections remaining open when server-side bandwidth limiting is enabled for TCP, HTTP, and UDP proxies. diff --git a/pkg/util/version/version.go b/pkg/util/version/version.go index 5c232712..8217f6be 100644 --- a/pkg/util/version/version.go +++ b/pkg/util/version/version.go @@ -14,7 +14,7 @@ package version -var version = "0.71.0" +var version = "0.71.1" func Full() string { return version From 41ed2aecf884930b33a77ddbe014c46a42811197 Mon Sep 17 00:00:00 2001 From: fatedier Date: Wed, 14 Oct 2026 10:49:55 +0800 Subject: [PATCH 23/23] test: align UDP compatibility protocol boundaries (#5567) --- test/e2e/compatibility/compatibility_test.go | 101 ++++++++++++++++--- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/test/e2e/compatibility/compatibility_test.go b/test/e2e/compatibility/compatibility_test.go index ad505510..a643f06e 100644 --- a/test/e2e/compatibility/compatibility_test.go +++ b/test/e2e/compatibility/compatibility_test.go @@ -192,19 +192,52 @@ transport.wireProtocol = "v2" }) }) -var _ = ginkgo.Describe("[Compatibility: BinaryUDPPacket]", func() { +var _ = ginkgo.Describe("[Compatibility: UDPDefault]", func() { + f := framework.NewDefaultFramework() + + ginkgo.It("current frps exchanges UDP with baseline frpc using the default wire protocol", func() { + portName := port.GenName("CompatDefaultUDPBaselineFRPC") + f.RunProcessesWithBinaries( + compatCtx.CurrentFRPSPath, + compatCtx.BaselineFRPCPath, + consts.DefaultServerConfig, + []string{udpClientConfig(portName, "")}, + ) + framework.NewRequestExpect(f).Protocol("udp").PortName(portName).Ensure() + }) + + ginkgo.It("baseline frps exchanges UDP with current frpc defaulting to v1", func() { + portName := port.GenName("CompatDefaultUDPBaselineFRPS") + f.RunProcessesWithBinaries( + compatCtx.BaselineFRPSPath, + compatCtx.CurrentFRPCPath, + consts.DefaultServerConfig, + []string{udpClientConfig(portName, "")}, + ) + framework.NewRequestExpect(f).Protocol("udp").PortName(portName).Ensure() + }) +}) + +var _ = ginkgo.Describe("[Compatibility: UDPWireV2]", func() { f := framework.NewDefaultFramework() ginkgo.BeforeEach(func() { - supportsV2, knownVersion := baselineSupportsControlWireProtocolV2(compatCtx.BaselineVersion) - if !knownVersion || !supportsV2 { - ginkgo.Skip(fmt.Sprintf("baseline version %q does not have known wire protocol v2 support", compatCtx.BaselineVersion)) + supportsV2, knownVersion := baselineSupportsUDPWireProtocolV2(compatCtx.BaselineVersion) + if !knownVersion { + ginkgo.Skip(fmt.Sprintf("baseline version %q has unknown ordinary UDP wire v2 framing support", compatCtx.BaselineVersion)) + } + if !supportsV2 { + // v0.69.0 supports control v2 but still uses legacy ordinary UDP framing. + // Mixing that early v2 implementation with the v0.69.1 framing was not supported. + ginkgo.Skip(fmt.Sprintf("baseline %q predates ordinary UDP wire v2 framing (v0.69.1); default UDP is tested separately", + compatCtx.BaselineVersion)) } }) - ginkgo.It("current frps falls back to JSON for baseline frpc", func() { - portName := port.GenName("CompatBinaryUDPBaselineFRPC") - clientConf := udpClientConfig("udp", portName, `transport.wireProtocol = "v2"`) + // The negotiated payload codec may be JSON or binary-v1; both use v2 framing. + ginkgo.It("current frps exchanges v2-framed UDP with baseline frpc", func() { + portName := port.GenName("CompatV2UDPBaselineFRPC") + clientConf := udpClientConfig(portName, `transport.wireProtocol = "v2"`) f.RunProcessesWithBinaries( compatCtx.CurrentFRPSPath, compatCtx.BaselineFRPCPath, @@ -214,9 +247,9 @@ var _ = ginkgo.Describe("[Compatibility: BinaryUDPPacket]", func() { framework.NewRequestExpect(f).Protocol("udp").PortName(portName).Ensure() }) - ginkgo.It("current frpc falls back to JSON for baseline frps", func() { - portName := port.GenName("CompatBinaryUDPBaselineFRPS") - clientConf := udpClientConfig("udp", portName, `transport.wireProtocol = "v2"`) + ginkgo.It("current frpc exchanges v2-framed UDP with baseline frps", func() { + portName := port.GenName("CompatV2UDPBaselineFRPS") + clientConf := udpClientConfig(portName, `transport.wireProtocol = "v2"`) f.RunProcessesWithBinaries( compatCtx.BaselineFRPSPath, compatCtx.CurrentFRPCPath, @@ -243,7 +276,7 @@ remotePort = {{ .%s }} `, consts.PortServerName, extra, proxyName, framework.TCPEchoServerPort, remotePortName) } -func udpClientConfig(proxyName string, remotePortName string, extra string) string { +func udpClientConfig(remotePortName string, extra string) string { return fmt.Sprintf(` serverAddr = "127.0.0.1" serverPort = {{ .%s }} @@ -252,11 +285,11 @@ log.level = "trace" %s [[proxies]] -name = "%s" +name = "udp" type = "udp" localPort = {{ .%s }} remotePort = {{ .%s }} -`, consts.PortServerName, extra, proxyName, framework.UDPEchoServerPort, remotePortName) +`, consts.PortServerName, extra, framework.UDPEchoServerPort, remotePortName) } func expectProcessExit(p *process.Process, timeout time.Duration) { @@ -268,6 +301,14 @@ func expectProcessExit(p *process.Process, timeout time.Duration) { } func baselineSupportsControlWireProtocolV2(version string) (supports bool, known bool) { + return baselineVersionAtLeast(version, [3]int{0, 69, 0}) +} + +func baselineSupportsUDPWireProtocolV2(version string) (supports bool, known bool) { + return baselineVersionAtLeast(version, [3]int{0, 69, 1}) +} + +func baselineVersionAtLeast(version string, minimum [3]int) (supports bool, known bool) { version = strings.TrimPrefix(version, "v") parts := strings.Split(version, ".") if len(parts) != 3 { @@ -287,7 +328,39 @@ func baselineSupportsControlWireProtocolV2(version string) (supports bool, known return false, false } - return compareSemanticVersion(major, minor, patch, 0, 69, 0) >= 0, true + return compareSemanticVersion(major, minor, patch, minimum[0], minimum[1], minimum[2]) >= 0, true +} + +func TestBaselineWireProtocolSupport(t *testing.T) { + for _, tc := range []struct { + version string + known bool + controlV2 bool + udpV2 bool + }{ + {version: "0.61.0", known: true}, + {version: "0.68.1", known: true}, + {version: "0.69.0", known: true, controlV2: true}, + {version: "v0.69.0", known: true, controlV2: true}, + {version: "0.69.1", known: true, controlV2: true, udpV2: true}, + {version: "v0.69.1", known: true, controlV2: true, udpV2: true}, + {version: "0.70.0", known: true, controlV2: true, udpV2: true}, + {version: "0.71.0", known: true, controlV2: true, udpV2: true}, + {version: "1.0.0", known: true, controlV2: true, udpV2: true}, + {version: "custom"}, + {version: "0.69"}, + {version: "0.69.x"}, + {version: "0.69.1-rc.1"}, + } { + t.Run(tc.version, func(t *testing.T) { + if supports, known := baselineSupportsControlWireProtocolV2(tc.version); supports != tc.controlV2 || known != tc.known { + t.Errorf("control v2 support = (%v, %v), want (%v, %v)", supports, known, tc.controlV2, tc.known) + } + if supports, known := baselineSupportsUDPWireProtocolV2(tc.version); supports != tc.udpV2 || known != tc.known { + t.Errorf("UDP v2 framing support = (%v, %v), want (%v, %v)", supports, known, tc.udpV2, tc.known) + } + }) + } } func compareSemanticVersion(major, minor, patch int, baseMajor, baseMinor, basePatch int) int {