From 8712ca0bcb69809459ab7fd11603341d90e54de6 Mon Sep 17 00:00:00 2001 From: xhe Date: Sat, 18 Nov 2023 15:32:39 +0800 Subject: [PATCH] conf: use server-http-tls to specify http security | tidb-test=pr/2248 (#403) Signed-off-by: xhe --- conf/proxy.toml | 12 +++++---- lib/config/proxy.go | 13 +++++----- lib/config/proxy_test.go | 4 +-- pkg/manager/cert/manager.go | 44 ++++++++++++++++---------------- pkg/manager/cert/manager_test.go | 32 +++++++++++++---------- pkg/proxy/proxy.go | 2 +- pkg/server/api/config_test.go | 4 +-- pkg/server/api/server.go | 2 +- 8 files changed, 61 insertions(+), 52 deletions(-) diff --git a/conf/proxy.toml b/conf/proxy.toml index 5507d558..b688e429 100644 --- a/conf/proxy.toml +++ b/conf/proxy.toml @@ -87,9 +87,6 @@ graceful-close-conn-timeout = 15 # server object: # 1. requires: cert/key or auto-certs(generate a temporary cert, mostly for testing) # 2. optionally: ca will enable server-side client verification. If skip-ca is true with non-empty ca, server will only verify clients if it can provide any cert. Otherwise, clients must provide a cert. -# peer object: -# 1. requires: cert/key/ca or auto-certs -# 2. useless/forbid: skip-ca # client object [security.cluster-tls] @@ -98,12 +95,17 @@ graceful-close-conn-timeout = 15 # client object [security.sql-tls] - # access to other components like TiDB or PD, will use this + # access to TiDB SQL(4000) port will use this skip-ca = true # server object [security.server-tls] - # proxy SQL or HTTP port will use this + # proxy SQL port will use this + # auto-certs = true + + # server object + [security.server-http-tls] + # proxy HTTP port will use this # auto-certs = true [metrics] diff --git a/lib/config/proxy.go b/lib/config/proxy.go index 556173bc..ea8e221a 100644 --- a/lib/config/proxy.go +++ b/lib/config/proxy.go @@ -113,10 +113,10 @@ func (c TLSConfig) HasCA() bool { } type Security struct { - ServerTLS TLSConfig `yaml:"server-tls,omitempty" toml:"server-tls,omitempty" json:"server-tls,omitempty"` - PeerTLS TLSConfig `yaml:"peer-tls,omitempty" toml:"peer-tls,omitempty" json:"peer-tls,omitempty"` - ClusterTLS TLSConfig `yaml:"cluster-tls,omitempty" toml:"cluster-tls,omitempty" json:"cluster-tls,omitempty"` - SQLTLS TLSConfig `yaml:"sql-tls,omitempty" toml:"sql-tls,omitempty" json:"sql-tls,omitempty"` + ServerSQLTLS TLSConfig `yaml:"server-tls,omitempty" toml:"server-tls,omitempty" json:"server-tls,omitempty"` + ServerHTTPTLS TLSConfig `yaml:"server-http-tls,omitempty" toml:"server-http-tls,omitempty" json:"server-http-tls,omitempty"` + ClusterTLS TLSConfig `yaml:"cluster-tls,omitempty" toml:"cluster-tls,omitempty" json:"cluster-tls,omitempty"` + SQLTLS TLSConfig `yaml:"sql-tls,omitempty" toml:"sql-tls,omitempty" json:"sql-tls,omitempty"` } func DefaultKeepAlive() (frontend, backendHealthy, backendUnhealthy KeepAlive) { @@ -153,8 +153,8 @@ func NewConfig() *Config { cfg.Advance.IgnoreWrongNamespace = true cfg.Security.SQLTLS.MinTLSVersion = "1.1" - cfg.Security.PeerTLS.MinTLSVersion = "1.1" - cfg.Security.ServerTLS.MinTLSVersion = "1.1" + cfg.Security.ServerSQLTLS.MinTLSVersion = "1.1" + cfg.Security.ServerHTTPTLS.MinTLSVersion = "1.1" cfg.Security.ClusterTLS.MinTLSVersion = "1.1" return &cfg @@ -184,6 +184,7 @@ func (cfg *Config) Check() error { if cfg.Proxy.ConnBufferSize > 0 && (cfg.Proxy.ConnBufferSize > 16*1024*1024 || cfg.Proxy.ConnBufferSize < 1024) { return errors.Wrapf(ErrInvalidConfigValue, "conn-buffer-size must be between 1K and 16M") } + return nil } diff --git a/lib/config/proxy_test.go b/lib/config/proxy_test.go index b62ab628..69d822b8 100644 --- a/lib/config/proxy_test.go +++ b/lib/config/proxy_test.go @@ -49,13 +49,13 @@ var testProxyConfig = Config{ }, }, Security: Security{ - ServerTLS: TLSConfig{ + ServerSQLTLS: TLSConfig{ CA: "a", Cert: "b", Key: "c", AutoCerts: true, }, - PeerTLS: TLSConfig{ + ServerHTTPTLS: TLSConfig{ CA: "a", Cert: "b", Key: "c", diff --git a/pkg/manager/cert/manager.go b/pkg/manager/cert/manager.go index 1acadba3..d741401c 100644 --- a/pkg/manager/cert/manager.go +++ b/pkg/manager/cert/manager.go @@ -25,14 +25,14 @@ const ( // Currently, all the namespaces share the same certs but there might be per-namespace // certs in the future. type CertManager struct { - serverTLS *security.CertInfo // client / proxyctl -> proxy - serverTLSConfig atomic.Pointer[tls.Config] - peerTLS *security.CertInfo // proxy -> proxy - peerTLSConfig atomic.Pointer[tls.Config] - clusterTLS *security.CertInfo // proxy -> pd / tidb status port - clusterTLSConfig atomic.Pointer[tls.Config] - sqlTLS *security.CertInfo // proxy -> tidb sql port - sqlTLSConfig atomic.Pointer[tls.Config] + serverSQLTLS *security.CertInfo // client -> proxy + serverSQLTLSConfig atomic.Pointer[tls.Config] + serverHTTPTLS *security.CertInfo // proxyctl -> proxy + serverHTTPTLSConfig atomic.Pointer[tls.Config] + clusterTLS *security.CertInfo // proxy -> pd / tidb status port + clusterTLSConfig atomic.Pointer[tls.Config] + sqlTLS *security.CertInfo // proxy -> tidb sql port + sqlTLSConfig atomic.Pointer[tls.Config] cancel context.CancelFunc wg waitgroup.WaitGroup @@ -51,8 +51,8 @@ func NewCertManager() *CertManager { // cfgch can be set to nil for the serverless tier because it has no config manager. func (cm *CertManager) Init(cfg *config.Config, logger *zap.Logger, cfgch <-chan *config.Config) error { cm.logger = logger - cm.serverTLS = security.NewCert(true) - cm.peerTLS = security.NewCert(false) + cm.serverSQLTLS = security.NewCert(true) + cm.serverHTTPTLS = security.NewCert(true) cm.clusterTLS = security.NewCert(false) cm.sqlTLS = security.NewCert(false) cm.setConfig(cfg) @@ -67,8 +67,8 @@ func (cm *CertManager) Init(cfg *config.Config, logger *zap.Logger, cfgch <-chan } func (cm *CertManager) setConfig(cfg *config.Config) { - cm.serverTLS.SetConfig(cfg.Security.ServerTLS) - cm.peerTLS.SetConfig(cfg.Security.PeerTLS) + cm.serverSQLTLS.SetConfig(cfg.Security.ServerSQLTLS) + cm.serverHTTPTLS.SetConfig(cfg.Security.ServerHTTPTLS) cm.clusterTLS.SetConfig(cfg.Security.ClusterTLS) cm.sqlTLS.SetConfig(cfg.Security.SQLTLS) } @@ -77,16 +77,16 @@ func (cm *CertManager) SetRetryInterval(interval time.Duration) { cm.retryInterval.Store(int64(interval)) } -func (cm *CertManager) ServerTLS() *tls.Config { - return cm.serverTLSConfig.Load() +func (cm *CertManager) ServerSQLTLS() *tls.Config { + return cm.serverSQLTLSConfig.Load() } -func (cm *CertManager) ClusterTLS() *tls.Config { - return cm.clusterTLSConfig.Load() +func (cm *CertManager) ServerHTTPTLS() *tls.Config { + return cm.serverHTTPTLSConfig.Load() } -func (cm *CertManager) PeerTLS() *tls.Config { - return cm.peerTLSConfig.Load() +func (cm *CertManager) ClusterTLS() *tls.Config { + return cm.clusterTLSConfig.Load() } func (cm *CertManager) SQLTLS() *tls.Config { @@ -122,15 +122,15 @@ func (cm *CertManager) reloadLoop(ctx context.Context, cfgch <-chan *config.Conf // If any error happens, we still continue and use the old cert. func (cm *CertManager) reload() error { errs := make([]error, 0, 4) - if tlsConfig, err := cm.serverTLS.Reload(cm.logger); err != nil { + if tlsConfig, err := cm.serverSQLTLS.Reload(cm.logger); err != nil { errs = append(errs, err) } else { - cm.serverTLSConfig.Store(tlsConfig) + cm.serverSQLTLSConfig.Store(tlsConfig) } - if tlsConfig, err := cm.peerTLS.Reload(cm.logger); err != nil { + if tlsConfig, err := cm.serverHTTPTLS.Reload(cm.logger); err != nil { errs = append(errs, err) } else { - cm.peerTLSConfig.Store(tlsConfig) + cm.serverHTTPTLSConfig.Store(tlsConfig) } if tlsConfig, err := cm.clusterTLS.Reload(cm.logger); err != nil { errs = append(errs, err) diff --git a/pkg/manager/cert/manager_test.go b/pkg/manager/cert/manager_test.go index bdb5921f..35352cfb 100644 --- a/pkg/manager/cert/manager_test.go +++ b/pkg/manager/cert/manager_test.go @@ -73,9 +73,9 @@ func TestInit(t *testing.T) { { name: "empty", check: func(t *testing.T, cm *CertManager) { - require.Nil(t, cm.ServerTLS()) + require.Nil(t, cm.ServerSQLTLS()) require.Nil(t, cm.ClusterTLS()) - require.Nil(t, cm.PeerTLS()) + require.Nil(t, cm.ServerHTTPTLS()) require.Nil(t, cm.SQLTLS()) }, }, @@ -83,28 +83,34 @@ func TestInit(t *testing.T) { name: "server config", cfg: config.Config{ Security: config.Security{ - ServerTLS: config.TLSConfig{AutoCerts: true}, + ServerSQLTLS: config.TLSConfig{AutoCerts: true}, + ServerHTTPTLS: config.TLSConfig{AutoCerts: true}, + ClusterTLS: config.TLSConfig{AutoCerts: true}, + SQLTLS: config.TLSConfig{AutoCerts: true}, }, }, check: func(t *testing.T, cm *CertManager) { require.Nil(t, cm.ClusterTLS()) - require.Nil(t, cm.PeerTLS()) require.Nil(t, cm.SQLTLS()) - require.NotNil(t, cm.ServerTLS()) + require.NotNil(t, cm.ServerHTTPTLS()) + require.NotNil(t, cm.ServerSQLTLS()) }, }, { name: "client config", cfg: config.Config{ Security: config.Security{ - SQLTLS: config.TLSConfig{SkipCA: true}, + ServerSQLTLS: config.TLSConfig{SkipCA: true}, + ServerHTTPTLS: config.TLSConfig{SkipCA: true}, + ClusterTLS: config.TLSConfig{SkipCA: true}, + SQLTLS: config.TLSConfig{SkipCA: true}, }, }, check: func(t *testing.T, cm *CertManager) { - require.Nil(t, cm.ClusterTLS()) - require.Nil(t, cm.PeerTLS()) - require.Nil(t, cm.ServerTLS()) + require.NotNil(t, cm.ClusterTLS()) require.NotNil(t, cm.SQLTLS()) + require.Nil(t, cm.ServerHTTPTLS()) + require.Nil(t, cm.ServerSQLTLS()) }, }, { @@ -159,7 +165,7 @@ func TestRotate(t *testing.T) { cfg := &config.Config{ Workdir: tmpdir, Security: config.Security{ - ServerTLS: config.TLSConfig{ + ServerSQLTLS: config.TLSConfig{ Cert: certPath, Key: keyPath, }, @@ -270,7 +276,7 @@ func TestRotate(t *testing.T) { } require.NoError(t, certMgr.Init(cfg, lg, nil)) - stls := certMgr.ServerTLS() + stls := certMgr.ServerSQLTLS() ctls := certMgr.SQLTLS() // pre reloading test @@ -335,7 +341,7 @@ func TestBidirectional(t *testing.T) { cfg := &config.Config{ Workdir: tmpdir, Security: config.Security{ - ServerTLS: config.TLSConfig{ + ServerSQLTLS: config.TLSConfig{ Cert: certPath1, Key: keyPath1, CA: caPath2, @@ -350,7 +356,7 @@ func TestBidirectional(t *testing.T) { certMgr := NewCertManager() require.NoError(t, certMgr.Init(cfg, lg, nil)) - stls := certMgr.ServerTLS() + stls := certMgr.ServerSQLTLS() ctls := certMgr.SQLTLS() clientErr, serverErr := connectWithTLS(ctls, stls) require.NoError(t, clientErr) diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index 7176ac1a..5c6baa8f 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -167,7 +167,7 @@ func (s *SQLServer) onConn(ctx context.Context, conn net.Conn, addr string) { s.mu.connID++ logger := s.logger.With(zap.Uint64("connID", connID), zap.String("client_addr", conn.RemoteAddr().String()), zap.String("addr", addr)) - clientConn := client.NewClientConnection(logger.Named("conn"), conn, s.certMgr.ServerTLS(), s.certMgr.SQLTLS(), + clientConn := client.NewClientConnection(logger.Named("conn"), conn, s.certMgr.ServerSQLTLS(), s.certMgr.SQLTLS(), s.hsHandler, connID, addr, &backend.BCConfig{ ProxyProtocol: s.mu.proxyProtocol, RequireBackendTLS: s.mu.requireBackendTLS, diff --git a/pkg/server/api/config_test.go b/pkg/server/api/config_test.go index fd957b0f..db678b4c 100644 --- a/pkg/server/api/config_test.go +++ b/pkg/server/api/config_test.go @@ -53,7 +53,7 @@ ignore-wrong-namespace = true [security.server-tls] min-tls-version = '1.1' -[security.peer-tls] +[security.server-http-tls] min-tls-version = '1.1' [security.cluster-tls] @@ -76,7 +76,7 @@ max-backups = 3 doHTTP(t, http.MethodGet, "/api/admin/config?format=json", nil, func(t *testing.T, r *http.Response) { all, err := io.ReadAll(r.Body) require.NoError(t, err) - require.Equal(t, `{"proxy":{"addr":"0.0.0.0:6000","pd-addrs":"127.0.0.1:2379","require-backend-tls":true,"frontend-keepalive":{"enabled":true},"backend-healthy-keepalive":{"enabled":true,"idle":60000000000,"cnt":5,"intvl":3000000000,"timeout":15000000000},"backend-unhealthy-keepalive":{"enabled":true,"idle":10000000000,"cnt":5,"intvl":1000000000,"timeout":5000000000},"graceful-close-conn-timeout":15},"api":{"addr":"0.0.0.0:3080"},"advance":{"ignore-wrong-namespace":true},"security":{"server-tls":{"min-tls-version":"1.1"},"peer-tls":{"min-tls-version":"1.1"},"cluster-tls":{"min-tls-version":"1.1"},"sql-tls":{"min-tls-version":"1.1"}},"metrics":{"metrics-addr":"","metrics-interval":0},"log":{"encoder":"tidb","level":"info","log-file":{"max-size":300,"max-days":3,"max-backups":3}}}`, + require.Equal(t, `{"proxy":{"addr":"0.0.0.0:6000","pd-addrs":"127.0.0.1:2379","require-backend-tls":true,"frontend-keepalive":{"enabled":true},"backend-healthy-keepalive":{"enabled":true,"idle":60000000000,"cnt":5,"intvl":3000000000,"timeout":15000000000},"backend-unhealthy-keepalive":{"enabled":true,"idle":10000000000,"cnt":5,"intvl":1000000000,"timeout":5000000000},"graceful-close-conn-timeout":15},"api":{"addr":"0.0.0.0:3080"},"advance":{"ignore-wrong-namespace":true},"security":{"server-tls":{"min-tls-version":"1.1"},"server-http-tls":{"min-tls-version":"1.1"},"cluster-tls":{"min-tls-version":"1.1"},"sql-tls":{"min-tls-version":"1.1"}},"metrics":{"metrics-addr":"","metrics-interval":0},"log":{"encoder":"tidb","level":"info","log-file":{"max-size":300,"max-days":3,"max-backups":3}}}`, string(regexp.MustCompile(`"workdir":"[^"]+",`).ReplaceAll(all, nil))) require.Equal(t, http.StatusOK, r.StatusCode) }) diff --git a/pkg/server/api/server.go b/pkg/server/api/server.go index c33384e6..daae792f 100644 --- a/pkg/server/api/server.go +++ b/pkg/server/api/server.go @@ -117,7 +117,7 @@ func NewServer(cfg config.API, lg *zap.Logger, } } - if tlscfg := crtmgr.ServerTLS(); tlscfg != nil { + if tlscfg := crtmgr.ServerHTTPTLS(); tlscfg != nil { h.listener = tls.NewListener(h.listener, tlscfg) }