server/group: refactor with shared abstractions and fix concurrency issues (#5222)

* server/group: refactor group package with shared abstractions and fix concurrency issues

Extract common patterns into reusable components:
- groupRegistry[G]: generic concurrent map for group lifecycle management
- baseGroup: shared plumbing for listener-based groups (TCP, HTTPS, TCPMux)
- Listener: unified virtual listener replacing 3 identical implementations

Fix concurrency issues:
- Stale-pointer race: isCurrent check + errGroupStale + controller retry loops
- Worker generation safety: pass realLn and acceptCh as params instead of reading mutable fields
- Connection leak: close conn on worker panic recovery path
- ABBA deadlock in HTTP UnRegister: consistent lock ordering (group.mu -> registry.mu)
- Round-robin overflow in HTTPGroup: use unsigned modulo

Add unit tests (17 tests) for registry, listener, and baseGroup.
Add TCPMux group load balancing e2e test.

* server/group: replace tautological assertion with require.NotPanics

* server/group: remove blank line between doc comment and type declaration
This commit is contained in:
fatedier
2026-03-08 18:57:21 +08:00
committed by GitHub
parent eeb0dacfc1
commit c7ac12ea0f
12 changed files with 720 additions and 436 deletions

View File

@@ -17,25 +17,19 @@ package group
import (
"context"
"net"
"sync"
gerr "github.com/fatedier/golib/errors"
"github.com/fatedier/frp/pkg/util/vhost"
)
type HTTPSGroupController struct {
groups map[string]*HTTPSGroup
groupRegistry[*HTTPSGroup]
httpsMuxer *vhost.HTTPSMuxer
mu sync.Mutex
}
func NewHTTPSGroupController(httpsMuxer *vhost.HTTPSMuxer) *HTTPSGroupController {
return &HTTPSGroupController{
groups: make(map[string]*HTTPSGroup),
httpsMuxer: httpsMuxer,
groupRegistry: newGroupRegistry[*HTTPSGroup](),
httpsMuxer: httpsMuxer,
}
}
@@ -44,41 +38,28 @@ func (ctl *HTTPSGroupController) Listen(
group, groupKey string,
routeConfig vhost.RouteConfig,
) (l net.Listener, err error) {
indexKey := group
ctl.mu.Lock()
g, ok := ctl.groups[indexKey]
if !ok {
g = NewHTTPSGroup(ctl)
ctl.groups[indexKey] = g
for {
g := ctl.getOrCreate(group, func() *HTTPSGroup {
return NewHTTPSGroup(ctl)
})
l, err = g.Listen(ctx, group, groupKey, routeConfig)
if err == errGroupStale {
continue
}
return
}
ctl.mu.Unlock()
return g.Listen(ctx, group, groupKey, routeConfig)
}
func (ctl *HTTPSGroupController) RemoveGroup(group string) {
ctl.mu.Lock()
defer ctl.mu.Unlock()
delete(ctl.groups, group)
}
type HTTPSGroup struct {
group string
groupKey string
domain string
baseGroup
acceptCh chan net.Conn
httpsLn *vhost.Listener
lns []*HTTPSGroupListener
ctl *HTTPSGroupController
mu sync.Mutex
domain string
ctl *HTTPSGroupController
}
func NewHTTPSGroup(ctl *HTTPSGroupController) *HTTPSGroup {
return &HTTPSGroup{
lns: make([]*HTTPSGroupListener, 0),
ctl: ctl,
acceptCh: make(chan net.Conn),
ctl: ctl,
}
}
@@ -86,23 +67,27 @@ func (g *HTTPSGroup) Listen(
ctx context.Context,
group, groupKey string,
routeConfig vhost.RouteConfig,
) (ln *HTTPSGroupListener, err error) {
) (ln *Listener, err error) {
g.mu.Lock()
defer g.mu.Unlock()
if !g.ctl.isCurrent(group, func(cur *HTTPSGroup) bool { return cur == g }) {
return nil, errGroupStale
}
if len(g.lns) == 0 {
// the first listener, listen on the real address
httpsLn, errRet := g.ctl.httpsMuxer.Listen(ctx, &routeConfig)
if errRet != nil {
return nil, errRet
}
ln = newHTTPSGroupListener(group, g, httpsLn.Addr())
g.group = group
g.groupKey = groupKey
g.domain = routeConfig.Domain
g.httpsLn = httpsLn
g.lns = append(g.lns, ln)
go g.worker()
g.initBase(group, groupKey, httpsLn, func() {
g.ctl.removeIf(g.group, func(cur *HTTPSGroup) bool {
return cur == g
})
})
ln = g.newListener(httpsLn.Addr())
go g.worker(httpsLn, g.acceptCh)
} else {
// route config in the same group must be equal
if g.group != group || g.domain != routeConfig.Domain {
@@ -111,87 +96,7 @@ func (g *HTTPSGroup) Listen(
if g.groupKey != groupKey {
return nil, ErrGroupAuthFailed
}
ln = newHTTPSGroupListener(group, g, g.lns[0].Addr())
g.lns = append(g.lns, ln)
ln = g.newListener(g.lns[0].Addr())
}
return
}
func (g *HTTPSGroup) worker() {
for {
c, err := g.httpsLn.Accept()
if err != nil {
return
}
err = gerr.PanicToError(func() {
g.acceptCh <- c
})
if err != nil {
return
}
}
}
func (g *HTTPSGroup) Accept() <-chan net.Conn {
return g.acceptCh
}
func (g *HTTPSGroup) CloseListener(ln *HTTPSGroupListener) {
g.mu.Lock()
defer g.mu.Unlock()
for i, tmpLn := range g.lns {
if tmpLn == ln {
g.lns = append(g.lns[:i], g.lns[i+1:]...)
break
}
}
if len(g.lns) == 0 {
close(g.acceptCh)
if g.httpsLn != nil {
g.httpsLn.Close()
}
g.ctl.RemoveGroup(g.group)
}
}
type HTTPSGroupListener struct {
groupName string
group *HTTPSGroup
addr net.Addr
closeCh chan struct{}
}
func newHTTPSGroupListener(name string, group *HTTPSGroup, addr net.Addr) *HTTPSGroupListener {
return &HTTPSGroupListener{
groupName: name,
group: group,
addr: addr,
closeCh: make(chan struct{}),
}
}
func (ln *HTTPSGroupListener) Accept() (c net.Conn, err error) {
var ok bool
select {
case <-ln.closeCh:
return nil, ErrListenerClosed
case c, ok = <-ln.group.Accept():
if !ok {
return nil, ErrListenerClosed
}
return c, nil
}
}
func (ln *HTTPSGroupListener) Addr() net.Addr {
return ln.addr
}
func (ln *HTTPSGroupListener) Close() (err error) {
close(ln.closeCh)
// remove self from HTTPSGroup
ln.group.CloseListener(ln)
return
}