vnet: serialize VirtualNet route lifecycle (#5512)

* vnet: serialize virtual net route lifecycle

* test: remove fixed virtual net helper parameters

pkg/plugin/visitor/virtual_net.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"sync"
 	"time"
@@ -33,10 +34,16 @@ func init() {
 	Register(v1.VisitorPluginVirtualNet, NewVirtualNetPlugin)
 }
 
+type clientRouteController interface {
+	RegisterClientRoute(context.Context, string, []net.IPNet, io.ReadWriteCloser)
+	UnregisterClientRoute(string, io.Writer) bool
+}
+
 type VirtualNetPlugin struct {
 	pluginCtx PluginContext
 
-	routes []net.IPNet
+	routeController clientRouteController
+	routes          []net.IPNet
 
 	mu             sync.Mutex
 	controllerConn net.Conn
@@ -60,6 +67,9 @@ func NewVirtualNetPlugin(pluginCtx PluginContext, options v1.VisitorPluginOption
 		pluginCtx: pluginCtx,
 		routes:    make([]net.IPNet, 0),
 	}
+	if pluginCtx.VnetController != nil {
+		p.routeController = pluginCtx.VnetController
+	}
 
 	p.ctx, p.cancel = context.WithCancel(pluginCtx.Ctx)
 
@@ -90,7 +100,7 @@ func (p *VirtualNetPlugin) Name() string {
 
 func (p *VirtualNetPlugin) Start() {
 	xl := xlog.FromContextSafe(p.pluginCtx.Ctx)
-	if p.pluginCtx.VnetController == nil {
+	if p.routeController == nil {
 		return
 	}
 
@@ -116,16 +126,17 @@ func (p *VirtualNetPlugin) run() {
 		select {
 		case <-p.ctx.Done():
 			xl.Infof("VirtualNetPlugin run loop for visitor [%s] stopping (context cancelled before pipe creation).", p.pluginCtx.Name)
-			p.cleanupControllerConn(xl)
+			p.cleanupCurrentControllerConn(xl)
 			return
 		default:
 		}
 
 		controllerConn, pluginConn := net.Pipe()
-
-		p.mu.Lock()
-		p.controllerConn = controllerConn
-		p.mu.Unlock()
+		xl.Infof("attempting to register client route for visitor [%s]", p.pluginCtx.Name)
+		if !p.registerControllerConn(controllerConn, pluginConn) {
+			xl.Infof("VirtualNetPlugin run loop for visitor [%s] stopping (context cancelled before route registration).", p.pluginCtx.Name)
+			return
+		}
 
 		// Wrap with CloseNotifyConn which supports both close notification and error recording
 		var closeErr error
@@ -134,8 +145,6 @@ func (p *VirtualNetPlugin) run() {
 			close(currentCloseSignal) // Signal the run loop on close.
 		})
 
-		xl.Infof("attempting to register client route for visitor [%s]", p.pluginCtx.Name)
-		p.pluginCtx.VnetController.RegisterClientRoute(p.ctx, p.pluginCtx.Name, p.routes, controllerConn)
 		xl.Infof("successfully registered client route for visitor [%s]. Starting connection handler with CloseNotifyConn.", p.pluginCtx.Name)
 
 		// Pass the CloseNotifyConn to the visitor for handling.
@@ -146,7 +155,7 @@ func (p *VirtualNetPlugin) run() {
 		select {
 		case <-p.ctx.Done():
 			xl.Infof("VirtualNetPlugin run loop stopping for visitor [%s] (context cancelled while waiting).", p.pluginCtx.Name)
-			p.cleanupControllerConn(xl)
+			p.cleanupControllerConn(xl, controllerConn)
 			return
 		case <-currentCloseSignal:
 			// Determine reconnect delay based on error with exponential backoff
@@ -171,7 +180,7 @@ func (p *VirtualNetPlugin) run() {
 			}
 
 			// The visitor closed the plugin side. Close the controller side.
-			p.cleanupControllerConn(xl)
+			p.cleanupControllerConn(xl, controllerConn)
 
 			xl.Infof("waiting %v before attempting reconnection for visitor [%s]...", reconnectDelay, p.pluginCtx.Name)
 			select {
@@ -186,6 +195,23 @@ func (p *VirtualNetPlugin) run() {
 	}
 }
 
+// registerControllerConn publishes and registers controllerConn atomically with
+// respect to Close. A canceled plugin cannot register a new route.
+func (p *VirtualNetPlugin) registerControllerConn(controllerConn, pluginConn net.Conn) bool {
+	p.mu.Lock()
+	if p.ctx.Err() != nil || p.routeController == nil {
+		p.mu.Unlock()
+		_ = controllerConn.Close()
+		_ = pluginConn.Close()
+		return false
+	}
+
+	p.controllerConn = controllerConn
+	p.routeController.RegisterClientRoute(p.ctx, p.pluginCtx.Name, p.routes, controllerConn)
+	p.mu.Unlock()
+	return true
+}
+
 // virtualNetReconnectDelay returns a bounded reconnect delay without allowing
 // the exponential shift to overflow for large consecutive error counts.
 func virtualNetReconnectDelay(consecutiveErrors int) time.Duration {
@@ -198,16 +224,37 @@ func virtualNetReconnectDelay(consecutiveErrors int) time.Duration {
 	return virtualNetReconnectBaseDelay * time.Duration(1<<uint(consecutiveErrors-1))
 }
 
-// cleanupControllerConn closes the current controllerConn (if it exists) under lock.
-func (p *VirtualNetPlugin) cleanupControllerConn(xl *xlog.Logger) {
+// cleanupControllerConn unregisters and closes one connection round without
+// affecting a replacement route owned by another connection.
+func (p *VirtualNetPlugin) cleanupControllerConn(xl *xlog.Logger, controllerConn net.Conn) {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	if p.controllerConn != nil {
-		xl.Debugf("cleaning up controllerConn for visitor [%s]", p.pluginCtx.Name)
-		p.controllerConn.Close()
-		p.controllerConn = nil
+	p.cleanupControllerConnLocked(xl, controllerConn)
+}
+
+func (p *VirtualNetPlugin) cleanupCurrentControllerConn(xl *xlog.Logger) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.cleanupControllerConnLocked(xl, p.controllerConn)
+}
+
+// cleanupControllerConnLocked must be called with p.mu held.
+func (p *VirtualNetPlugin) cleanupControllerConnLocked(xl *xlog.Logger, controllerConn net.Conn) {
+	if controllerConn == nil {
+		p.closeSignal = nil
+		return
+	}
+
+	if p.routeController != nil &&
+		p.routeController.UnregisterClientRoute(p.pluginCtx.Name, controllerConn) {
+		xl.Infof("unregistered client route for visitor [%s]", p.pluginCtx.Name)
+	}
+	xl.Debugf("cleaning up controllerConn for visitor [%s]", p.pluginCtx.Name)
+	_ = controllerConn.Close()
+	if p.controllerConn == controllerConn {
+		p.controllerConn = nil
+		p.closeSignal = nil
 	}
-	p.closeSignal = nil
 }
 
 // Close initiates the plugin shutdown.
@@ -218,18 +265,9 @@ func (p *VirtualNetPlugin) Close() error {
 	// Signal the run loop goroutine to stop.
 	p.cancel()
 
-	// Unregister the route only if it is still owned by this plugin instance.
-	p.mu.Lock()
-	controllerConn := p.controllerConn
-	p.mu.Unlock()
-	if p.pluginCtx.VnetController != nil && controllerConn != nil &&
-		p.pluginCtx.VnetController.UnregisterClientRoute(p.pluginCtx.Name, controllerConn) {
-		xl.Infof("unregistered client route for visitor [%s]", p.pluginCtx.Name)
-	}
-
-	// Explicitly close the controller side of the pipe.
-	// This ensures the pipe is broken even if the run loop is stuck or the visitor hasn't closed its end.
-	p.cleanupControllerConn(xl)
+	// Unregister and close the current connection while holding the same lock
+	// used to check cancellation and register a route in run.
+	p.cleanupCurrentControllerConn(xl)
 	xl.Infof("finished cleaning up connections during close for visitor [%s]", p.pluginCtx.Name)
 
 	return nil