add persistent proxy/visitor store with CRUD API and web UI (#5188)

cmd/frpc/sub/proxy.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/config/source"
 	v1 "github.com/fatedier/frp/pkg/config/v1"
 	"github.com/fatedier/frp/pkg/config/v1/validation"
 	"github.com/fatedier/frp/pkg/policy/security"
@@ -86,13 +87,14 @@ func NewProxyCommand(name string, c v1.ProxyConfigurer, clientCfg *v1.ClientComm
 				os.Exit(1)
 			}
 
-			c.Complete(clientCfg.User)
 			c.GetBaseConfig().Type = name
-			if err := validation.ValidateProxyConfigurerForClient(c); err != nil {
+			c.Complete()
+			proxyCfg := c
+			if err := validation.ValidateProxyConfigurerForClient(proxyCfg); err != nil {
 				fmt.Println(err)
 				os.Exit(1)
 			}
-			err := startService(clientCfg, []v1.ProxyConfigurer{c}, nil, unsafeFeatures, "")
+			err := startService(clientCfg, []v1.ProxyConfigurer{proxyCfg}, nil, unsafeFeatures, "")
 			if err != nil {
 				fmt.Println(err)
 				os.Exit(1)
@@ -117,13 +119,14 @@ func NewVisitorCommand(name string, c v1.VisitorConfigurer, clientCfg *v1.Client
 				os.Exit(1)
 			}
 
-			c.Complete(clientCfg)
 			c.GetBaseConfig().Type = name
-			if err := validation.ValidateVisitorConfigurer(c); err != nil {
+			c.Complete()
+			visitorCfg := c
+			if err := validation.ValidateVisitorConfigurer(visitorCfg); err != nil {
 				fmt.Println(err)
 				os.Exit(1)
 			}
-			err := startService(clientCfg, nil, []v1.VisitorConfigurer{c}, unsafeFeatures, "")
+			err := startService(clientCfg, nil, []v1.VisitorConfigurer{visitorCfg}, unsafeFeatures, "")
 			if err != nil {
 				fmt.Println(err)
 				os.Exit(1)
@@ -131,3 +134,18 @@ func NewVisitorCommand(name string, c v1.VisitorConfigurer, clientCfg *v1.Client
 		},
 	}
 }
+
+func startService(
+	cfg *v1.ClientCommonConfig,
+	proxyCfgs []v1.ProxyConfigurer,
+	visitorCfgs []v1.VisitorConfigurer,
+	unsafeFeatures *security.UnsafeFeatures,
+	cfgFile string,
+) error {
+	configSource := source.NewConfigSource()
+	if err := configSource.ReplaceAll(proxyCfgs, visitorCfgs); err != nil {
+		return fmt.Errorf("failed to set config source: %w", err)
+	}
+	aggregator := source.NewAggregator(configSource)
+	return startServiceWithAggregator(cfg, aggregator, unsafeFeatures, cfgFile)
+}

cmd/frpc/sub/root.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/fatedier/frp/client"
 	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/config/source"
 	v1 "github.com/fatedier/frp/pkg/config/v1"
 	"github.com/fatedier/frp/pkg/config/v1/validation"
 	"github.com/fatedier/frp/pkg/policy/featuregate"
@@ -120,22 +121,64 @@ func handleTermSignal(svr *client.Service) {
 }
 
 func runClient(cfgFilePath string, unsafeFeatures *security.UnsafeFeatures) error {
-	cfg, proxyCfgs, visitorCfgs, isLegacyFormat, err := config.LoadClientConfig(cfgFilePath, strictConfigMode)
+	// Load configuration
+	result, err := config.LoadClientConfigResult(cfgFilePath, strictConfigMode)
 	if err != nil {
 		return err
 	}
-	if isLegacyFormat {
+	if result.IsLegacyFormat {
 		fmt.Printf("WARNING: ini format is deprecated and the support will be removed in the future, " +
 			"please use yaml/json/toml format instead!\n")
 	}
 
-	if len(cfg.FeatureGates) > 0 {
-		if err := featuregate.SetFromMap(cfg.FeatureGates); err != nil {
+	if len(result.Common.FeatureGates) > 0 {
+		if err := featuregate.SetFromMap(result.Common.FeatureGates); err != nil {
 			return err
 		}
 	}
 
-	warning, err := validation.ValidateAllClientConfig(cfg, proxyCfgs, visitorCfgs, unsafeFeatures)
+	return runClientWithAggregator(result, unsafeFeatures, cfgFilePath)
+}
+
+// runClientWithAggregator runs the client using the internal source aggregator.
+func runClientWithAggregator(result *config.ClientConfigLoadResult, unsafeFeatures *security.UnsafeFeatures, cfgFilePath string) error {
+	configSource := source.NewConfigSource()
+	if err := configSource.ReplaceAll(result.Proxies, result.Visitors); err != nil {
+		return fmt.Errorf("failed to set config source: %w", err)
+	}
+
+	var storeSource *source.StoreSource
+
+	if result.Common.Store.IsEnabled() {
+		storePath := result.Common.Store.Path
+		if storePath != "" && cfgFilePath != "" && !filepath.IsAbs(storePath) {
+			storePath = filepath.Join(filepath.Dir(cfgFilePath), storePath)
+		}
+
+		s, err := source.NewStoreSource(source.StoreSourceConfig{
+			Path: storePath,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create store source: %w", err)
+		}
+		storeSource = s
+	}
+
+	aggregator := source.NewAggregator(configSource)
+	if storeSource != nil {
+		aggregator.SetStoreSource(storeSource)
+	}
+
+	proxyCfgs, visitorCfgs, err := aggregator.Load()
+	if err != nil {
+		return fmt.Errorf("failed to load config from sources: %w", err)
+	}
+
+	proxyCfgs, visitorCfgs = config.FilterClientConfigurers(result.Common, proxyCfgs, visitorCfgs)
+	proxyCfgs = config.CompleteProxyConfigurers(proxyCfgs)
+	visitorCfgs = config.CompleteVisitorConfigurers(visitorCfgs)
+
+	warning, err := validation.ValidateAllClientConfig(result.Common, proxyCfgs, visitorCfgs, unsafeFeatures)
 	if warning != nil {
 		fmt.Printf("WARNING: %v\n", warning)
 	}
@@ -143,35 +186,32 @@ func runClient(cfgFilePath string, unsafeFeatures *security.UnsafeFeatures) erro
 		return err
 	}
 
-	return startService(cfg, proxyCfgs, visitorCfgs, unsafeFeatures, cfgFilePath)
+	return startServiceWithAggregator(result.Common, aggregator, unsafeFeatures, cfgFilePath)
 }
 
-func startService(
+func startServiceWithAggregator(
 	cfg *v1.ClientCommonConfig,
-	proxyCfgs []v1.ProxyConfigurer,
-	visitorCfgs []v1.VisitorConfigurer,
+	aggregator *source.Aggregator,
 	unsafeFeatures *security.UnsafeFeatures,
 	cfgFile string,
 ) error {
 	log.InitLogger(cfg.Log.To, cfg.Log.Level, int(cfg.Log.MaxDays), cfg.Log.DisablePrintColor)
 
 	if cfgFile != "" {
-		log.Infof("start frpc service for config file [%s]", cfgFile)
+		log.Infof("start frpc service for config file [%s] with aggregated configuration", cfgFile)
 		defer log.Infof("frpc service for config file [%s] stopped", cfgFile)
 	}
 	svr, err := client.NewService(client.ServiceOptions{
-		Common:         cfg,
-		ProxyCfgs:      proxyCfgs,
-		VisitorCfgs:    visitorCfgs,
-		UnsafeFeatures: unsafeFeatures,
-		ConfigFilePath: cfgFile,
+		Common:                 cfg,
+		ConfigSourceAggregator: aggregator,
+		UnsafeFeatures:         unsafeFeatures,
+		ConfigFilePath:         cfgFile,
 	})
 	if err != nil {
 		return err
 	}
 
 	shouldGracefulClose := cfg.Transport.Protocol == "kcp" || cfg.Transport.Protocol == "quic"
-	// Capture the exit signal if we use kcp or quic.
 	if shouldGracefulClose {
 		go handleTermSignal(svr)
 	}