frpc: support 'includes' in common section to include proxy configs in other files(#2421)

cmd/frpc/sub/root.go

@@ -15,11 +15,14 @@
 package sub
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -179,24 +182,84 @@ func parseClientCommonCfgFromCmd() (cfg config.ClientCommonConf, err error) {
 	return
 }
 
-func runClient(cfgFilePath string) (err error) {
+func runClient(cfgFilePath string) error {
+	cfg, pxyCfgs, visitorCfgs, err := parseConfig(cfgFilePath)
+	if err != nil {
+		return err
+	}
+	return startService(cfg, pxyCfgs, visitorCfgs, cfgFilePath)
+}
+
+func parseConfig(cfgFilePath string) (
+	cfg config.ClientCommonConf,
+	pxyCfgs map[string]config.ProxyConf,
+	visitorCfgs map[string]config.VisitorConf,
+	err error,
+) {
 	var content []byte
 	content, err = config.GetRenderedConfFromFile(cfgFilePath)
 	if err != nil {
-		return err
+		return
 	}
+	configBuffer := bytes.NewBuffer(nil)
+	configBuffer.Write(content)
 
-	cfg, err := parseClientCommonCfg(CfgFileTypeIni, content)
+	// Parse common section.
+	cfg, err = parseClientCommonCfg(CfgFileTypeIni, content)
 	if err != nil {
-		return err
+		return
 	}
 
-	pxyCfgs, visitorCfgs, err := config.LoadAllProxyConfsFromIni(cfg.User, content, cfg.Start)
+	// Aggregate proxy configs from include files.
+	var buf []byte
+	buf, err = getIncludeContents(cfg.IncludeConfigFiles)
 	if err != nil {
-		return err
+		err = fmt.Errorf("getIncludeContents error: %v", err)
+		return
 	}
+	configBuffer.WriteString("\n")
+	configBuffer.Write(buf)
 
-	return startService(cfg, pxyCfgs, visitorCfgs, cfgFilePath)
+	// Parse all proxy and visitor configs.
+	pxyCfgs, visitorCfgs, err = config.LoadAllProxyConfsFromIni(cfg.User, configBuffer.Bytes(), cfg.Start)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// getIncludeContents renders all configs from paths.
+// files format can be a single file path or directory or regex path.
+func getIncludeContents(paths []string) ([]byte, error) {
+	out := bytes.NewBuffer(nil)
+	for _, path := range paths {
+		absDir, err := filepath.Abs(filepath.Dir(path))
+		if err != nil {
+			return nil, err
+		}
+		if _, err := os.Stat(absDir); os.IsNotExist(err) {
+			return nil, err
+		}
+		files, err := ioutil.ReadDir(absDir)
+		if err != nil {
+			return nil, err
+		}
+		for _, fi := range files {
+			if fi.IsDir() {
+				continue
+			}
+			absFile := filepath.Join(absDir, fi.Name())
+			if matched, _ := filepath.Match(filepath.Join(absDir, filepath.Base(path)), absFile); matched {
+				tmpContent, err := config.GetRenderedConfFromFile(absFile)
+				if err != nil {
+					return nil, fmt.Errorf("render extra config %s error: %v", absFile, err)
+				}
+				out.Write(tmpContent)
+				out.WriteString("\n")
+			}
+		}
+	}
+	return out.Bytes(), nil
 }
 
 func startService(

cmd/frpc/sub/verify.go

@@ -18,8 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/fatedier/frp/pkg/config"
-
 	"github.com/spf13/cobra"
 )
 
@@ -31,18 +29,7 @@ var verifyCmd = &cobra.Command{
 	Use:   "verify",
 	Short: "Verify that the configures is valid",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		iniContent, err := config.GetRenderedConfFromFile(cfgFile)
+		_, _, _, err := parseConfig(cfgFile)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-
-		cfg, err := parseClientCommonCfg(CfgFileTypeIni, iniContent)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-		_, _, err = config.LoadAllProxyConfsFromIni(cfg.User, iniContent, cfg.Start)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)

pkg/config/client.go

@@ -17,6 +17,7 @@ package config
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/fatedier/frp/pkg/auth"
@@ -136,40 +137,43 @@ type ClientCommonConf struct {
 	// UDPPacketSize specifies the udp packet size
 	// By default, this value is 1500
 	UDPPacketSize int64 `ini:"udp_packet_size" json:"udp_packet_size"`
+	// Include other config files for proxies.
+	IncludeConfigFiles []string `ini:"includes" json:"includes"`
 }
 
 // GetDefaultClientConf returns a client configuration with default values.
 func GetDefaultClientConf() ClientCommonConf {
 	return ClientCommonConf{
-		ClientConfig:      auth.GetDefaultClientConf(),
+		ClientConfig:       auth.GetDefaultClientConf(),
-		ServerAddr:        "0.0.0.0",
+		ServerAddr:         "0.0.0.0",
-		ServerPort:        7000,
+		ServerPort:         7000,
-		HTTPProxy:         os.Getenv("http_proxy"),
+		HTTPProxy:          os.Getenv("http_proxy"),
-		LogFile:           "console",
+		LogFile:            "console",
-		LogWay:            "console",
+		LogWay:             "console",
-		LogLevel:          "info",
+		LogLevel:           "info",
-		LogMaxDays:        3,
+		LogMaxDays:         3,
-		DisableLogColor:   false,
+		DisableLogColor:    false,
-		AdminAddr:         "127.0.0.1",
+		AdminAddr:          "127.0.0.1",
-		AdminPort:         0,
+		AdminPort:          0,
-		AdminUser:         "",
+		AdminUser:          "",
-		AdminPwd:          "",
+		AdminPwd:           "",
-		AssetsDir:         "",
+		AssetsDir:          "",
-		PoolCount:         1,
+		PoolCount:          1,
-		TCPMux:            true,
+		TCPMux:             true,
-		User:              "",
+		User:               "",
-		DNSServer:         "",
+		DNSServer:          "",
-		LoginFailExit:     true,
+		LoginFailExit:      true,
-		Start:             make([]string, 0),
+		Start:              make([]string, 0),
-		Protocol:          "tcp",
+		Protocol:           "tcp",
-		TLSEnable:         false,
+		TLSEnable:          false,
-		TLSCertFile:       "",
+		TLSCertFile:        "",
-		TLSKeyFile:        "",
+		TLSKeyFile:         "",
-		TLSTrustedCaFile:  "",
+		TLSTrustedCaFile:   "",
-		HeartbeatInterval: 30,
+		HeartbeatInterval:  30,
-		HeartbeatTimeout:  90,
+		HeartbeatTimeout:   90,
-		Metas:             make(map[string]string),
+		Metas:              make(map[string]string),
-		UDPPacketSize:     1500,
+		UDPPacketSize:      1500,
+		IncludeConfigFiles: make([]string, 0),
 	}
 }
 
@@ -208,6 +212,15 @@ func (cfg *ClientCommonConf) Validate() error {
 		return fmt.Errorf("invalid protocol")
 	}
 
+	for _, f := range cfg.IncludeConfigFiles {
+		absDir, err := filepath.Abs(filepath.Dir(f))
+		if err != nil {
+			return fmt.Errorf("include: parse directory of %s failed: %v", f, absDir)
+		}
+		if _, err := os.Stat(absDir); os.IsNotExist(err) {
+			return fmt.Errorf("include: directory of %s not exist", f)
+		}
+	}
 	return nil
 }
 
@@ -236,7 +249,6 @@ func UnmarshalClientConfFromIni(source interface{}) (ClientCommonConf, error) {
 	}
 
 	common.Metas = GetMapWithoutPrefix(s.KeysHash(), "meta_")
-
 	return common, nil
 }
 

pkg/config/client_test.go

@@ -290,12 +290,13 @@ func Test_LoadClientCommonConf(t *testing.T) {
 			"var1": "123",
 			"var2": "234",
 		},
-		UDPPacketSize: 1509,
+		UDPPacketSize:      1509,
+		IncludeConfigFiles: []string{},
 	}
 
 	common, err := UnmarshalClientConfFromIni(testClientBytesWithFull)
 	assert.NoError(err)
-	assert.Equal(expected, common)
+	assert.EqualValues(expected, common)
 }
 
 func Test_LoadClientBasicConf(t *testing.T) {
@@ -641,5 +642,4 @@ func Test_LoadClientBasicConf(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(proxyExpected, proxyActual)
 	assert.Equal(visitorExpected, visitorActual)
-
 }