frp/server/control.go

556 lines
13 KiB
Go
Raw Normal View History

2017-03-23 02:01:25 +08:00
// Copyright 2017 fatedier, fatedier@gmail.com
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2017-03-09 02:03:47 +08:00
package server
import (
2019-10-12 20:13:12 +08:00
"context"
2017-03-09 02:03:47 +08:00
"fmt"
"io"
2019-10-12 20:13:12 +08:00
"net"
2018-05-11 10:42:57 +08:00
"runtime/debug"
2017-03-09 02:03:47 +08:00
"sync"
"time"
"github.com/fatedier/frp/models/auth"
2017-03-09 02:03:47 +08:00
"github.com/fatedier/frp/models/config"
"github.com/fatedier/frp/models/consts"
2018-05-08 02:13:30 +08:00
frpErr "github.com/fatedier/frp/models/errors"
2017-03-09 02:03:47 +08:00
"github.com/fatedier/frp/models/msg"
2019-12-20 20:28:28 +08:00
plugin "github.com/fatedier/frp/models/plugin/server"
2019-01-15 00:11:08 +08:00
"github.com/fatedier/frp/server/controller"
"github.com/fatedier/frp/server/metrics"
2019-01-15 00:11:08 +08:00
"github.com/fatedier/frp/server/proxy"
"github.com/fatedier/frp/utils/util"
2017-03-09 02:03:47 +08:00
"github.com/fatedier/frp/utils/version"
2019-10-12 20:13:12 +08:00
"github.com/fatedier/frp/utils/xlog"
2018-05-08 02:13:30 +08:00
2018-05-08 23:51:13 +08:00
"github.com/fatedier/golib/control/shutdown"
2018-05-08 02:13:30 +08:00
"github.com/fatedier/golib/crypto"
"github.com/fatedier/golib/errors"
2017-03-09 02:03:47 +08:00
)
2019-01-15 00:11:08 +08:00
type ControlManager struct {
// controls indexed by run id
ctlsByRunId map[string]*Control
mu sync.RWMutex
}
func NewControlManager() *ControlManager {
return &ControlManager{
ctlsByRunId: make(map[string]*Control),
}
}
func (cm *ControlManager) Add(runId string, ctl *Control) (oldCtl *Control) {
cm.mu.Lock()
defer cm.mu.Unlock()
oldCtl, ok := cm.ctlsByRunId[runId]
if ok {
oldCtl.Replaced(ctl)
}
cm.ctlsByRunId[runId] = ctl
return
}
2019-01-26 21:36:24 +08:00
// we should make sure if it's the same control to prevent delete a new one
func (cm *ControlManager) Del(runId string, ctl *Control) {
2019-01-15 00:11:08 +08:00
cm.mu.Lock()
defer cm.mu.Unlock()
2019-01-26 21:36:24 +08:00
if c, ok := cm.ctlsByRunId[runId]; ok && c == ctl {
delete(cm.ctlsByRunId, runId)
}
2019-01-15 00:11:08 +08:00
}
func (cm *ControlManager) GetById(runId string) (ctl *Control, ok bool) {
cm.mu.RLock()
defer cm.mu.RUnlock()
ctl, ok = cm.ctlsByRunId[runId]
return
}
2017-03-09 02:03:47 +08:00
type Control struct {
2019-01-10 20:53:06 +08:00
// all resource managers and controllers
2019-01-15 00:11:08 +08:00
rc *controller.ResourceController
// proxy manager
pxyManager *proxy.ProxyManager
2019-12-20 20:28:28 +08:00
// plugin manager
pluginManager *plugin.Manager
// verifies authentication based on selected method
authVerifier auth.Verifier
2017-03-09 02:03:47 +08:00
// login message
loginMsg *msg.Login
// control connection
conn net.Conn
// put a message in this channel to send it over control connection to client
sendCh chan (msg.Message)
// read from this channel to get the next message sent by client
readCh chan (msg.Message)
// work connections
workConnCh chan net.Conn
// proxies in one client
2019-01-15 00:11:08 +08:00
proxies map[string]proxy.Proxy
2017-03-09 02:03:47 +08:00
// pool count
poolCount int
2018-01-26 14:56:55 +08:00
// ports used, for limitations
portsUsedNum int
2017-03-09 02:03:47 +08:00
// last time got the Ping message
lastPing time.Time
// A new run id will be generated when a new client login.
// If run id got from login message has same run id, it means it's the same client, so we can
// replace old controller instantly.
runId string
// control status
status string
readerShutdown *shutdown.Shutdown
writerShutdown *shutdown.Shutdown
managerShutdown *shutdown.Shutdown
allShutdown *shutdown.Shutdown
mu sync.RWMutex
// Server configuration information
serverCfg config.ServerCommonConf
2019-10-12 20:13:12 +08:00
xl *xlog.Logger
ctx context.Context
2017-03-09 02:03:47 +08:00
}
2019-12-20 20:28:28 +08:00
func NewControl(
ctx context.Context,
rc *controller.ResourceController,
pxyManager *proxy.ProxyManager,
pluginManager *plugin.Manager,
authVerifier auth.Verifier,
2019-12-20 20:28:28 +08:00
ctlConn net.Conn,
loginMsg *msg.Login,
serverCfg config.ServerCommonConf,
) *Control {
2019-01-15 00:11:08 +08:00
2019-08-29 21:13:21 +08:00
poolCount := loginMsg.PoolCount
if poolCount > int(serverCfg.MaxPoolCount) {
poolCount = int(serverCfg.MaxPoolCount)
}
2017-03-09 02:03:47 +08:00
return &Control{
2019-01-10 20:53:06 +08:00
rc: rc,
2019-01-15 00:11:08 +08:00
pxyManager: pxyManager,
2019-12-20 20:28:28 +08:00
pluginManager: pluginManager,
authVerifier: authVerifier,
2017-03-09 02:03:47 +08:00
conn: ctlConn,
loginMsg: loginMsg,
sendCh: make(chan msg.Message, 10),
readCh: make(chan msg.Message, 10),
2019-08-29 21:13:21 +08:00
workConnCh: make(chan net.Conn, poolCount+10),
2019-01-15 00:11:08 +08:00
proxies: make(map[string]proxy.Proxy),
2019-08-29 21:13:21 +08:00
poolCount: poolCount,
2018-01-26 14:56:55 +08:00
portsUsedNum: 0,
2017-03-09 02:03:47 +08:00
lastPing: time.Now(),
runId: loginMsg.RunId,
status: consts.Working,
readerShutdown: shutdown.New(),
writerShutdown: shutdown.New(),
managerShutdown: shutdown.New(),
allShutdown: shutdown.New(),
serverCfg: serverCfg,
2019-10-12 20:13:12 +08:00
xl: xlog.FromContextSafe(ctx),
ctx: ctx,
2017-03-09 02:03:47 +08:00
}
}
// Start send a login success message to client and start working.
func (ctl *Control) Start() {
2017-03-10 01:42:06 +08:00
loginRespMsg := &msg.LoginResp{
2017-10-24 18:20:07 +08:00
Version: version.Full(),
RunId: ctl.runId,
ServerUdpPort: ctl.serverCfg.BindUdpPort,
2017-10-24 18:20:07 +08:00
Error: "",
2017-03-09 02:03:47 +08:00
}
2017-03-10 01:42:06 +08:00
msg.WriteMsg(ctl.conn, loginRespMsg)
2017-03-09 02:03:47 +08:00
2017-03-10 01:42:06 +08:00
go ctl.writer()
2017-03-09 02:03:47 +08:00
for i := 0; i < ctl.poolCount; i++ {
ctl.sendCh <- &msg.ReqWorkConn{}
}
go ctl.manager()
go ctl.reader()
go ctl.stoper()
}
func (ctl *Control) RegisterWorkConn(conn net.Conn) error {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
select {
case ctl.workConnCh <- conn:
2019-10-12 20:13:12 +08:00
xl.Debug("new work connection registered")
return nil
2017-03-09 02:03:47 +08:00
default:
2019-10-12 20:13:12 +08:00
xl.Debug("work connection pool is full, discarding")
return fmt.Errorf("work connection pool is full, discarding")
2017-03-09 02:03:47 +08:00
}
}
// When frps get one user connection, we get one work connection from the pool and return it.
// If no workConn available in the pool, send message to frpc to get one or more
// and wait until it is available.
// return an error if wait timeout
func (ctl *Control) GetWorkConn() (workConn net.Conn, err error) {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
var ok bool
// get a work connection from the pool
select {
case workConn, ok = <-ctl.workConnCh:
if !ok {
2018-05-08 02:13:30 +08:00
err = frpErr.ErrCtlClosed
2017-03-09 02:03:47 +08:00
return
}
2019-10-12 20:13:12 +08:00
xl.Debug("get work connection from pool")
2017-03-09 02:03:47 +08:00
default:
// no work connections available in the poll, send message to frpc to get more
err = errors.PanicToError(func() {
ctl.sendCh <- &msg.ReqWorkConn{}
})
if err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("%v", err)
2017-03-09 02:03:47 +08:00
return
}
select {
case workConn, ok = <-ctl.workConnCh:
if !ok {
2018-05-08 02:13:30 +08:00
err = frpErr.ErrCtlClosed
2019-10-12 20:13:12 +08:00
xl.Warn("no work connections avaiable, %v", err)
2017-03-09 02:03:47 +08:00
return
}
case <-time.After(time.Duration(ctl.serverCfg.UserConnTimeout) * time.Second):
2017-03-09 02:03:47 +08:00
err = fmt.Errorf("timeout trying to get work connection")
2019-10-12 20:13:12 +08:00
xl.Warn("%v", err)
2017-03-09 02:03:47 +08:00
return
}
}
// When we get a work connection from pool, replace it with a new one.
errors.PanicToError(func() {
ctl.sendCh <- &msg.ReqWorkConn{}
})
return
}
func (ctl *Control) Replaced(newCtl *Control) {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
xl.Info("Replaced by client [%s]", newCtl.runId)
2017-03-09 02:03:47 +08:00
ctl.runId = ""
ctl.allShutdown.Start()
}
func (ctl *Control) writer() {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
defer ctl.allShutdown.Start()
defer ctl.writerShutdown.Done()
encWriter, err := crypto.NewWriter(ctl.conn, []byte(ctl.serverCfg.Token))
2017-03-10 01:42:06 +08:00
if err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("crypto new writer error: %v", err)
2017-03-10 01:42:06 +08:00
ctl.allShutdown.Start()
return
}
2017-03-09 02:03:47 +08:00
for {
if m, ok := <-ctl.sendCh; !ok {
2019-10-12 20:13:12 +08:00
xl.Info("control writer is closing")
2017-03-09 02:03:47 +08:00
return
} else {
2017-03-10 01:42:06 +08:00
if err := msg.WriteMsg(encWriter, m); err != nil {
2019-10-12 20:13:12 +08:00
xl.Warn("write message to control connection error: %v", err)
2017-03-09 02:03:47 +08:00
return
}
}
}
}
func (ctl *Control) reader() {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
defer ctl.allShutdown.Start()
defer ctl.readerShutdown.Done()
encReader := crypto.NewReader(ctl.conn, []byte(ctl.serverCfg.Token))
2017-03-09 02:03:47 +08:00
for {
2017-03-10 01:42:06 +08:00
if m, err := msg.ReadMsg(encReader); err != nil {
2017-03-09 02:03:47 +08:00
if err == io.EOF {
2019-10-12 20:13:12 +08:00
xl.Debug("control connection closed")
2017-03-09 02:03:47 +08:00
return
} else {
2019-10-12 20:13:12 +08:00
xl.Warn("read error: %v", err)
2019-04-08 15:39:14 +08:00
ctl.conn.Close()
2017-03-09 02:03:47 +08:00
return
}
} else {
ctl.readCh <- m
}
}
}
func (ctl *Control) stoper() {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
ctl.allShutdown.WaitStart()
close(ctl.readCh)
2018-01-17 01:09:33 +08:00
ctl.managerShutdown.WaitDone()
2017-03-09 02:03:47 +08:00
close(ctl.sendCh)
2018-01-17 01:09:33 +08:00
ctl.writerShutdown.WaitDone()
2017-03-09 02:03:47 +08:00
ctl.conn.Close()
2018-01-17 01:09:33 +08:00
ctl.readerShutdown.WaitDone()
2017-03-09 02:03:47 +08:00
2018-03-19 20:22:15 +08:00
ctl.mu.Lock()
defer ctl.mu.Unlock()
2017-03-09 02:03:47 +08:00
close(ctl.workConnCh)
for workConn := range ctl.workConnCh {
workConn.Close()
}
for _, pxy := range ctl.proxies {
pxy.Close()
2019-01-15 00:11:08 +08:00
ctl.pxyManager.Del(pxy.GetName())
metrics.Server.CloseProxy(pxy.GetName(), pxy.GetConf().GetBaseInfo().ProxyType)
2017-03-09 02:03:47 +08:00
}
ctl.allShutdown.Done()
2019-10-12 20:13:12 +08:00
xl.Info("client exit success")
metrics.Server.CloseClient()
2019-01-15 00:11:08 +08:00
}
// block until Control closed
func (ctl *Control) WaitClosed() {
ctl.allShutdown.WaitDone()
2017-03-09 02:03:47 +08:00
}
func (ctl *Control) manager() {
2019-10-12 20:13:12 +08:00
xl := ctl.xl
2017-03-09 02:03:47 +08:00
defer func() {
if err := recover(); err != nil {
2019-10-12 20:13:12 +08:00
xl.Error("panic error: %v", err)
xl.Error(string(debug.Stack()))
2017-03-09 02:03:47 +08:00
}
}()
defer ctl.allShutdown.Start()
defer ctl.managerShutdown.Done()
heartbeat := time.NewTicker(time.Second)
defer heartbeat.Stop()
for {
select {
case <-heartbeat.C:
if time.Since(ctl.lastPing) > time.Duration(ctl.serverCfg.HeartBeatTimeout)*time.Second {
2019-10-12 20:13:12 +08:00
xl.Warn("heartbeat timeout")
2018-03-19 20:22:15 +08:00
return
2017-03-09 02:03:47 +08:00
}
case rawMsg, ok := <-ctl.readCh:
if !ok {
return
}
switch m := rawMsg.(type) {
case *msg.NewProxy:
2019-12-20 20:28:28 +08:00
content := &plugin.NewProxyContent{
User: plugin.UserInfo{
User: ctl.loginMsg.User,
Metas: ctl.loginMsg.Metas,
RunId: ctl.loginMsg.RunId,
2019-12-20 20:28:28 +08:00
},
NewProxy: *m,
}
var remoteAddr string
retContent, err := ctl.pluginManager.NewProxy(content)
if err == nil {
m = &retContent.NewProxy
remoteAddr, err = ctl.RegisterProxy(m)
}
2017-03-09 02:03:47 +08:00
// register proxy in this control
resp := &msg.NewProxyResp{
ProxyName: m.ProxyName,
}
if err != nil {
2019-10-12 20:13:12 +08:00
xl.Warn("new proxy [%s] error: %v", m.ProxyName, err)
resp.Error = util.GenerateResponseErrorString(fmt.Sprintf("new proxy [%s] error", m.ProxyName), err, ctl.serverCfg.DetailedErrorsToClient)
2017-03-09 02:03:47 +08:00
} else {
2018-01-17 14:40:08 +08:00
resp.RemoteAddr = remoteAddr
2019-10-12 20:13:12 +08:00
xl.Info("new proxy [%s] success", m.ProxyName)
metrics.Server.NewProxy(m.ProxyName, m.ProxyType)
2017-03-09 02:03:47 +08:00
}
ctl.sendCh <- resp
2017-06-11 17:22:05 +08:00
case *msg.CloseProxy:
ctl.CloseProxy(m)
2019-10-12 20:13:12 +08:00
xl.Info("close proxy [%s] success", m.ProxyName)
2017-03-09 02:03:47 +08:00
case *msg.Ping:
content := &plugin.PingContent{
User: plugin.UserInfo{
User: ctl.loginMsg.User,
Metas: ctl.loginMsg.Metas,
RunId: ctl.loginMsg.RunId,
},
Ping: *m,
}
retContent, err := ctl.pluginManager.Ping(content)
if err == nil {
m = &retContent.Ping
err = ctl.authVerifier.VerifyPing(m)
}
if err != nil {
xl.Warn("received invalid ping: %v", err)
ctl.sendCh <- &msg.Pong{
Error: util.GenerateResponseErrorString("invalid ping", err, ctl.serverCfg.DetailedErrorsToClient),
}
return
}
2017-03-09 02:03:47 +08:00
ctl.lastPing = time.Now()
2019-10-12 20:13:12 +08:00
xl.Debug("receive heartbeat")
2017-03-09 02:03:47 +08:00
ctl.sendCh <- &msg.Pong{}
}
}
}
}
2018-01-17 14:40:08 +08:00
func (ctl *Control) RegisterProxy(pxyMsg *msg.NewProxy) (remoteAddr string, err error) {
2017-03-09 02:03:47 +08:00
var pxyConf config.ProxyConf
// Load configures from NewProxy message and check.
pxyConf, err = config.NewProxyConfFromMsg(pxyMsg, ctl.serverCfg)
2017-03-09 02:03:47 +08:00
if err != nil {
2018-01-17 14:40:08 +08:00
return
2017-03-09 02:03:47 +08:00
}
// NewProxy will return a interface Proxy.
// In fact it create different proxies by different proxy type, we just call run() here.
pxy, err := proxy.NewProxy(ctl.ctx, ctl.runId, ctl.rc, ctl.poolCount, ctl.GetWorkConn, pxyConf, ctl.serverCfg)
2017-03-09 02:03:47 +08:00
if err != nil {
2018-01-17 14:40:08 +08:00
return remoteAddr, err
2017-03-09 02:03:47 +08:00
}
2018-01-26 14:56:55 +08:00
// Check ports used number in each client
if ctl.serverCfg.MaxPortsPerClient > 0 {
2018-01-26 14:56:55 +08:00
ctl.mu.Lock()
if ctl.portsUsedNum+pxy.GetUsedPortsNum() > int(ctl.serverCfg.MaxPortsPerClient) {
2018-01-26 14:56:55 +08:00
ctl.mu.Unlock()
err = fmt.Errorf("exceed the max_ports_per_client")
return
}
ctl.portsUsedNum = ctl.portsUsedNum + pxy.GetUsedPortsNum()
ctl.mu.Unlock()
defer func() {
if err != nil {
ctl.mu.Lock()
ctl.portsUsedNum = ctl.portsUsedNum - pxy.GetUsedPortsNum()
ctl.mu.Unlock()
}
}()
}
2018-01-17 14:40:08 +08:00
remoteAddr, err = pxy.Run()
2017-03-09 02:03:47 +08:00
if err != nil {
2018-01-17 14:40:08 +08:00
return
2017-03-09 02:03:47 +08:00
}
defer func() {
if err != nil {
pxy.Close()
}
}()
2019-01-15 00:11:08 +08:00
err = ctl.pxyManager.Add(pxyMsg.ProxyName, pxy)
2017-03-09 02:03:47 +08:00
if err != nil {
2018-01-17 14:40:08 +08:00
return
2017-03-09 02:03:47 +08:00
}
2017-06-11 17:22:05 +08:00
ctl.mu.Lock()
ctl.proxies[pxy.GetName()] = pxy
ctl.mu.Unlock()
2018-01-17 14:40:08 +08:00
return
2017-03-09 02:03:47 +08:00
}
2017-06-11 17:22:05 +08:00
func (ctl *Control) CloseProxy(closeMsg *msg.CloseProxy) (err error) {
ctl.mu.Lock()
pxy, ok := ctl.proxies[closeMsg.ProxyName]
if !ok {
2018-01-26 14:56:55 +08:00
ctl.mu.Unlock()
2017-06-11 17:22:05 +08:00
return
}
if ctl.serverCfg.MaxPortsPerClient > 0 {
2018-01-26 14:56:55 +08:00
ctl.portsUsedNum = ctl.portsUsedNum - pxy.GetUsedPortsNum()
}
2017-06-11 17:22:05 +08:00
pxy.Close()
2019-01-15 00:11:08 +08:00
ctl.pxyManager.Del(pxy.GetName())
2017-06-27 01:59:30 +08:00
delete(ctl.proxies, closeMsg.ProxyName)
2018-01-26 14:56:55 +08:00
ctl.mu.Unlock()
metrics.Server.CloseProxy(pxy.GetName(), pxy.GetConf().GetBaseInfo().ProxyType)
2017-06-11 17:22:05 +08:00
return
}