From 79b0790479fb48672414f990feda3ba1fecfb78b Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 20 Sep 2023 11:25:31 +0800 Subject: [PATCH 01/15] support compress --- go.mod | 1 + go.sum | 3 +- pkg/proxy/backend/authenticator.go | 30 ++++- pkg/proxy/net/compress.go | 203 +++++++++++++++++++++++++++++ pkg/proxy/net/mysql.go | 17 ++- pkg/proxy/net/packetio.go | 171 ++++++++++++------------ pkg/proxy/net/packetio_options.go | 2 +- pkg/proxy/net/proxy.go | 118 ++++++++++++++--- pkg/proxy/net/proxy_test.go | 2 +- pkg/proxy/net/tls.go | 70 ++++++++-- 10 files changed, 495 insertions(+), 122 deletions(-) create mode 100644 pkg/proxy/net/compress.go diff --git a/go.mod b/go.mod index b514da75..19390e9a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/gin-gonic/gin v1.8.1 github.com/go-mysql-org/go-mysql v1.6.0 github.com/go-sql-driver/mysql v1.7.0 + github.com/klauspost/compress v1.16.5 github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc github.com/pingcap/tiproxy/lib v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index 2f2ce436..3539c9e2 100644 --- a/go.sum +++ b/go.sum @@ -415,7 +415,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/pkg/proxy/backend/authenticator.go b/pkg/proxy/backend/authenticator.go index 3841a09c..2616267f 100644 --- a/pkg/proxy/backend/authenticator.go +++ b/pkg/proxy/backend/authenticator.go @@ -29,11 +29,12 @@ const defRequiredBackendCaps = pnet.ClientDeprecateEOF // SupportedServerCapabilities is the default supported capabilities. Other server capabilities are not supported. // TiDB supports ClientDeprecateEOF since v6.3.0. +// TiDB supports ClientCompress and ClientZstdCompressionAlgorithm since v7.2.0. const SupportedServerCapabilities = pnet.ClientLongPassword | pnet.ClientFoundRows | pnet.ClientConnectWithDB | pnet.ClientODBC | pnet.ClientLocalFiles | pnet.ClientInteractive | pnet.ClientLongFlag | pnet.ClientSSL | pnet.ClientTransactions | pnet.ClientReserved | pnet.ClientSecureConnection | pnet.ClientMultiStatements | pnet.ClientMultiResults | pnet.ClientPluginAuth | pnet.ClientConnectAttrs | pnet.ClientPluginAuthLenencClientData | - requiredFrontendCaps | defRequiredBackendCaps + pnet.ClientCompress | pnet.ClientZstdCompressionAlgorithm | requiredFrontendCaps | defRequiredBackendCaps // Authenticator handshakes with the client and the backend. type Authenticator struct { @@ -42,6 +43,7 @@ type Authenticator struct { attrs map[string]string salt []byte capability pnet.Capability + zstdLevel int collation uint8 proxyProtocol bool requireBackendTLS bool @@ -64,9 +66,7 @@ func (auth *Authenticator) writeProxyProtocol(clientIO, backendIO *pnet.PacketIO } // either from another proxy or directly from clients, we are acting as a proxy proxy.Command = proxyprotocol.ProxyCommandProxy - if err := backendIO.WriteProxyV2(proxy); err != nil { - return err - } + backendIO.EnableProxyClient(proxy) } return nil } @@ -157,6 +157,7 @@ func (auth *Authenticator) handshakeFirstTime(logger *zap.Logger, cctx ConnConte auth.dbname = clientResp.DB auth.collation = clientResp.Collation auth.attrs = clientResp.Attrs + auth.zstdLevel = clientResp.ZstdLevel // In case of testing, backendIO is passed manually that we don't want to bother with the routing logic. backendIO, err := getBackendIO(cctx, auth, clientResp, 15*time.Second) @@ -225,6 +226,12 @@ loop: pktIdx++ switch serverPkt[0] { case pnet.OKHeader.Byte(): + if err := auth.setCompress(clientIO, auth.capability); err != nil { + return err + } + if err := auth.setCompress(backendIO, auth.capability&backendCapability); err != nil { + return err + } return nil case pnet.ErrHeader.Byte(): return pnet.ParseErrorPacket(serverPkt) @@ -277,7 +284,10 @@ func (auth *Authenticator) handshakeSecondTime(logger *zap.Logger, clientIO, bac return err } - return auth.handleSecondAuthResult(backendIO) + if err = auth.handleSecondAuthResult(backendIO); err == nil { + return auth.setCompress(backendIO, auth.capability&backendCapability) + } + return err } func (auth *Authenticator) readInitialHandshake(backendIO *pnet.PacketIO) (serverPkt []byte, capability pnet.Capability, err error) { @@ -370,6 +380,16 @@ func (auth *Authenticator) handleSecondAuthResult(backendIO *pnet.PacketIO) erro } } +func (auth *Authenticator) setCompress(clientIO *pnet.PacketIO, capability pnet.Capability) error { + algorithm := pnet.CompressionNone + if capability&pnet.ClientCompress > 0 { + algorithm = pnet.CompressionZlib + } else if capability&pnet.ClientZstdCompressionAlgorithm > 0 { + algorithm = pnet.CompressionZstd + } + return clientIO.SetCompressionAlgorithm(algorithm, auth.zstdLevel) +} + // changeUser is called once the client sends COM_CHANGE_USER. func (auth *Authenticator) changeUser(req *pnet.ChangeUserReq) { auth.user = req.User diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go new file mode 100644 index 00000000..f883c8af --- /dev/null +++ b/pkg/proxy/net/compress.go @@ -0,0 +1,203 @@ +// Copyright 2023 PingCAP, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "bytes" + "compress/zlib" + "io" + + "github.com/klauspost/compress/zstd" + "github.com/pingcap/tiproxy/lib/util/errors" +) + +// CompressAlgorithm is the algorithm for MySQL compressed protocol. +type CompressAlgorithm int + +const ( + // CompressionNone indicates no compression in use. + CompressionNone CompressAlgorithm = iota + // CompressionZlib is zlib/deflate. + CompressionZlib + // CompressionZstd is Facebook's Zstandard. + CompressionZstd +) + +const ( + // maxCompressedSize is the max size for compressed data. + // MySQL starts with `net_buffer_length` (default 16384) and larger packets after that. + // The length itself must fit in the 3 byte field in the header. + // Can't be bigger then the max value for `net_buffer_length` (1048576) + maxCompressedSize = 1024 * 1024 + // minCompressSize is the min size for compressed data. + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_compression_packet.html + // suggests a MIN_COMPRESS_LENGTH of 50. + minCompressSize = 50 +) + +func (p *PacketIO) SetCompressionAlgorithm(algorithm CompressAlgorithm, zstdLevel int) error { + switch algorithm { + case CompressionZlib, CompressionZstd: + case CompressionNone: + return nil + default: + return errors.Errorf("Unknown compression algorithm %d", algorithm) + } + rw, err := newCompressedReadWriter(p.readWriter, algorithm, zstdLevel) + if err != nil { + return err + } + p.readWriter = rw + return nil +} + +var _ packetReadWriter = (*compressedReadWriter)(nil) + +type compressedReadWriter struct { + packetReadWriter + compressedReader io.ReadCloser + buf *bytes.Buffer + algorithm CompressAlgorithm + zstdLevel zstd.EncoderLevel + sequence uint8 +} + +func newCompressedReadWriter(rw packetReadWriter, algorithm CompressAlgorithm, zstdLevel int) (*compressedReadWriter, error) { + if err := rw.Flush(); err != nil { + return nil, err + } + return &compressedReadWriter{ + packetReadWriter: rw, + buf: new(bytes.Buffer), + algorithm: algorithm, + zstdLevel: zstd.EncoderLevelFromZstd(zstdLevel), + }, nil +} + +func (crw *compressedReadWriter) reset() { + crw.sequence = 0 +} + +func (crw *compressedReadWriter) afterRead() { + if crw.compressedReader != nil { + _ = crw.compressedReader.Close() + crw.compressedReader = nil + } +} + +func (crw *compressedReadWriter) Read(p []byte) (n int, err error) { + if crw.compressedReader == nil { + var compressedHeader [7]byte + if _, err := io.ReadFull(crw.packetReadWriter, compressedHeader[:]); err != nil { + return 0, errors.WithStack(err) + } + compressedSequence := compressedHeader[3] + if compressedSequence != crw.sequence { + return 0, ErrInvalidSequence.GenWithStack( + "invalid compressed sequence, expected %d, actual %d", crw.sequence, compressedSequence) + } + crw.sequence++ + uncompressedLength := int(uint32(compressedHeader[4]) | uint32(compressedHeader[5])<<8 | uint32(compressedHeader[6])<<16) + if uncompressedLength <= 0 { + return crw.packetReadWriter.Read(p) + } + + switch crw.algorithm { + case CompressionZlib: + crw.compressedReader, err = zlib.NewReader(crw.packetReadWriter) + case CompressionZstd: + var zstdReader *zstd.Decoder + if zstdReader, err = zstd.NewReader(crw.packetReadWriter, zstd.WithDecoderConcurrency(1)); err == nil { + crw.compressedReader = zstdReader.IOReadCloser() + } + } + if err != nil { + return 0, errors.WithStack(err) + } + } + return crw.compressedReader.Read(p) +} + +func (crw *compressedReadWriter) Write(data []byte) (n int, err error) { + for { + remainingLen := maxCompressedSize - crw.buf.Len() + if len(data) <= remainingLen { + written, err := crw.buf.Write(data) + if err != nil { + return 0, err + } + return n + written, nil + } + written, err := crw.buf.Write(data[:remainingLen]) + if err != nil { + return 0, err + } + n += written + data = data[remainingLen:] + if err = crw.flush(); err != nil { + return 0, err + } + } +} + +func (crw *compressedReadWriter) flush() error { + var payload, compressedPacket bytes.Buffer + var w io.WriteCloser + var err error + + data := crw.buf.Bytes() + crw.buf.Reset() + + switch crw.algorithm { + case CompressionZlib: + w, err = zlib.NewWriterLevel(&payload, zlib.HuffmanOnly) + case CompressionZstd: + w, err = zstd.NewWriter(&payload, zstd.WithEncoderLevel(crw.zstdLevel)) + } + if err != nil { + return errors.WithStack(err) + } + + uncompressedLength := 0 + compressedLength := len(data) + compressedHeader := make([]byte, 7) + + if len(data) > minCompressSize { + uncompressedLength = len(data) + if _, err = w.Write(data); err != nil { + return errors.WithStack(err) + } + if err = w.Close(); err != nil { + return errors.WithStack(err) + } + compressedLength = len(payload.Bytes()) + } + compressedHeader[0] = byte(compressedLength) + compressedHeader[1] = byte(compressedLength >> 8) + compressedHeader[2] = byte(compressedLength >> 16) + compressedHeader[3] = crw.sequence + compressedHeader[4] = byte(uncompressedLength) + compressedHeader[5] = byte(uncompressedLength >> 8) + compressedHeader[6] = byte(uncompressedLength >> 16) + if _, err = compressedPacket.Write(compressedHeader); err != nil { + return errors.WithStack(err) + } + crw.sequence++ + + if len(data) > minCompressSize { + _, err = compressedPacket.Write(payload.Bytes()) + } else { + _, err = compressedPacket.Write(data) + } + if err != nil { + return errors.WithStack(err) + } + if err = w.Close(); err != nil { + return errors.WithStack(err) + } + if _, err = crw.packetReadWriter.Write(compressedPacket.Bytes()); err != nil { + return errors.WithStack(err) + } + return nil +} diff --git a/pkg/proxy/net/mysql.go b/pkg/proxy/net/mysql.go index c128bada..1cf0fc83 100644 --- a/pkg/proxy/net/mysql.go +++ b/pkg/proxy/net/mysql.go @@ -62,6 +62,7 @@ type HandshakeResp struct { AuthPlugin string AuthData []byte Capability Capability + ZstdLevel int Collation uint8 } @@ -132,11 +133,19 @@ func ParseHandshakeResponse(data []byte) (*HandshakeResp, error) { pos += off row := data[pos : pos+int(num)] resp.Attrs, err = parseAttrs(row) + // Some clients have known bugs, but we should be compatible with them. + // E.g. https://bugs.mysql.com/bug.php?id=79612. if err != nil { err = &errors.Warning{Err: errors.Wrapf(err, "parse attrs failed")} } + pos += int(num) } } + + // zstd compress level + if resp.Capability&ClientZstdCompressionAlgorithm > 0 { + resp.ZstdLevel = int(data[pos]) + } return resp, err } @@ -192,7 +201,7 @@ func MakeHandshakeResponse(resp *HandshakeResp) []byte { attrBuf = DumpLengthEncodedInt(attrLenBuf[:0], uint64(len(attrs))) } - length := 4 + 4 + 1 + 23 + len(resp.User) + 1 + len(authResp) + len(resp.AuthData) + len(resp.DB) + 1 + len(resp.AuthPlugin) + 1 + len(attrBuf) + len(attrs) + length := 4 + 4 + 1 + 23 + len(resp.User) + 1 + len(authResp) + len(resp.AuthData) + len(resp.DB) + 1 + len(resp.AuthPlugin) + 1 + len(attrBuf) + len(attrs) + 1 data := make([]byte, length) pos := 0 // capability [32 bit] @@ -244,6 +253,12 @@ func MakeHandshakeResponse(resp *HandshakeResp) []byte { pos += copy(data[pos:], attrBuf) pos += copy(data[pos:], attrs) } + + // compress level + if capability&ClientZstdCompressionAlgorithm > 0 { + data[pos] = byte(resp.ZstdLevel) + pos++ + } return data[:pos] } diff --git a/pkg/proxy/net/packetio.go b/pkg/proxy/net/packetio.go index 5b151698..32d95b15 100644 --- a/pkg/proxy/net/packetio.go +++ b/pkg/proxy/net/packetio.go @@ -30,7 +30,6 @@ import ( "crypto/tls" "io" "net" - "sync/atomic" "time" "github.com/pingcap/tidb/errno" @@ -51,29 +50,80 @@ const ( defaultReaderSize = 16 * 1024 ) -// rdbufConn will buffer read for non-TLS connections. -// While TLS connections have internal buffering, we still need to pass *rdbufConn to `tls.XXX()`. -// Because TLS handshake data may already be buffered in `*rdbufConn`. -// TODO: only enable writer buffering for TLS connections, otherwise enable read/write buffering. +type buffer interface { + Peek(n int) ([]byte, error) + Discard(n int) (int, error) + Flush() error +} + +type packetReadWriter interface { + net.Conn + buffer + afterRead() + reset() + getProxy() *proxyprotocol.Proxy + tlsConnectionState() tls.ConnectionState + getInBytes() uint64 + getOutBytes() uint64 +} + +// rdbufConn buffers read for non-TLS connections. type rdbufConn struct { net.Conn - *bufio.Reader + *bufio.ReadWriter + inBytes uint64 + outBytes uint64 +} + +func newRdbufConn(conn net.Conn) *rdbufConn { + return &rdbufConn{ + Conn: conn, + ReadWriter: bufio.NewReadWriter(bufio.NewReaderSize(conn, defaultReaderSize), bufio.NewWriterSize(conn, defaultWriterSize)), + } +} + +func (f *rdbufConn) Read(b []byte) (n int, err error) { + n, err = f.ReadWriter.Read(b) + f.inBytes += uint64(n) + return n, err +} + +func (f *rdbufConn) Write(p []byte) (n int, err error) { + n, err = f.ReadWriter.Write(p) + f.outBytes += uint64(n) + return n, err +} + +func (f *rdbufConn) afterRead() { +} + +func (f *rdbufConn) reset() { +} + +func (f *rdbufConn) getProxy() *proxyprotocol.Proxy { + return nil +} + +func (f *rdbufConn) getInBytes() uint64 { + return f.inBytes +} + +func (f *rdbufConn) getOutBytes() uint64 { + return f.outBytes } -func (f *rdbufConn) Read(b []byte) (int, error) { - return f.Reader.Read(b) +func (f *rdbufConn) tlsConnectionState() tls.ConnectionState { + if conn, ok := f.Conn.(*tls.Conn); ok { + return conn.ConnectionState() + } + return tls.ConnectionState{} } // PacketIO is a helper to read and write sql and proxy protocol. type PacketIO struct { lastKeepAlive config.KeepAlive - inBytes uint64 - outBytes uint64 - conn net.Conn rawConn net.Conn - buf *bufio.ReadWriter - proxyInited atomic.Bool - proxy *proxyprotocol.Proxy + readWriter packetReadWriter logger *zap.Logger remoteAddr net.Addr wrap error @@ -81,21 +131,12 @@ type PacketIO struct { } func NewPacketIO(conn net.Conn, lg *zap.Logger, opts ...PacketIOption) *PacketIO { - buf := bufio.NewReadWriter( - bufio.NewReaderSize(conn, defaultReaderSize), - bufio.NewWriterSize(conn, defaultWriterSize), - ) p := &PacketIO{ - rawConn: conn, - conn: &rdbufConn{ - conn, - buf.Reader, - }, - logger: lg, - sequence: 0, - buf: buf, + rawConn: conn, + logger: lg, + sequence: 0, + readWriter: newRdbufConn(conn), } - p.proxyInited.Store(true) p.ApplyOpts(opts...) return p } @@ -110,24 +151,20 @@ func (p *PacketIO) wrapErr(err error) error { return errors.WithStack(errors.Wrap(p.wrap, err)) } -// Proxy returned parsed proxy header from clients if any. -func (p *PacketIO) Proxy() *proxyprotocol.Proxy { - return p.proxy -} - func (p *PacketIO) LocalAddr() net.Addr { - return p.conn.LocalAddr() + return p.readWriter.LocalAddr() } func (p *PacketIO) RemoteAddr() net.Addr { if p.remoteAddr != nil { return p.remoteAddr } - return p.conn.RemoteAddr() + return p.readWriter.RemoteAddr() } func (p *PacketIO) ResetSequence() { p.sequence = 0 + p.readWriter.reset() } // GetSequence is used in tests to assert that the sequences on the client and server are equal. @@ -137,48 +174,21 @@ func (p *PacketIO) GetSequence() uint8 { func (p *PacketIO) readOnePacket() ([]byte, bool, error) { var header [4]byte - - if _, err := io.ReadFull(p.buf, header[:]); err != nil { + defer p.readWriter.afterRead() + if _, err := io.ReadFull(p.readWriter, header[:]); err != nil { return nil, false, errors.Wrap(ErrReadConn, err) } - p.inBytes += 4 - - // probe proxy V2 - refill := false - if !p.proxyInited.Load() { - if bytes.Equal(header[:], proxyprotocol.MagicV2[:4]) { - proxyHeader, err := p.parseProxyV2() - if err != nil { - return nil, false, errors.Wrap(ErrReadConn, err) - } - if proxyHeader != nil { - p.proxy = proxyHeader - refill = true - } - } - p.proxyInited.Store(true) - } - - // refill mysql headers - if refill { - if _, err := io.ReadFull(p.buf, header[:]); err != nil { - return nil, false, errors.Wrap(ErrReadConn, err) - } - p.inBytes += 4 - } - sequence := header[3] if sequence != p.sequence { - return nil, false, ErrInvalidSequence.GenWithStack("invalid sequence %d != %d", sequence, p.sequence) + return nil, false, ErrInvalidSequence.GenWithStack("invalid sequence, expected %d, actual %d", p.sequence, sequence) } p.sequence++ - length := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) + length := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) data := make([]byte, length) - if _, err := io.ReadFull(p.buf, data); err != nil { + if _, err := io.ReadFull(p.readWriter, data); err != nil { return nil, false, errors.Wrap(ErrReadConn, err) } - p.inBytes += uint64(length) return data, length == MaxPayloadLen, nil } @@ -213,15 +223,13 @@ func (p *PacketIO) writeOnePacket(data []byte) (int, bool, error) { header[3] = p.sequence p.sequence++ - if _, err := io.Copy(p.buf, bytes.NewReader(header[:])); err != nil { + if _, err := io.Copy(p.readWriter, bytes.NewReader(header[:])); err != nil { return 0, more, errors.Wrap(ErrWriteConn, err) } - p.outBytes += 4 - if _, err := io.Copy(p.buf, bytes.NewReader(data[:length])); err != nil { + if _, err := io.Copy(p.readWriter, bytes.NewReader(data[:length])); err != nil { return 0, more, errors.Wrap(ErrWriteConn, err) } - p.outBytes += uint64(length) return length, more, nil } @@ -244,22 +252,15 @@ func (p *PacketIO) WritePacket(data []byte, flush bool) (err error) { } func (p *PacketIO) InBytes() uint64 { - return p.inBytes + return p.readWriter.getInBytes() } func (p *PacketIO) OutBytes() uint64 { - return p.outBytes -} - -func (p *PacketIO) TLSConnectionState() tls.ConnectionState { - if tlsConn, ok := p.conn.(*tls.Conn); ok { - return tlsConn.ConnectionState() - } - return tls.ConnectionState{} + return p.readWriter.getOutBytes() } func (p *PacketIO) Flush() error { - if err := p.buf.Flush(); err != nil { + if err := p.readWriter.Flush(); err != nil { return p.wrapErr(errors.Wrap(ErrFlushConn, err)) } return nil @@ -270,14 +271,14 @@ func (p *PacketIO) Flush() error { // This function normally costs 1ms, so don't call it too frequently. // This function may incorrectly return true if the system is extremely slow. func (p *PacketIO) IsPeerActive() bool { - if err := p.conn.SetReadDeadline(time.Now().Add(time.Millisecond)); err != nil { + if err := p.readWriter.SetReadDeadline(time.Now().Add(time.Millisecond)); err != nil { return false } active := true - if _, err := p.buf.Peek(1); err != nil { + if _, err := p.readWriter.Peek(1); err != nil { active = !errors.Is(err, io.EOF) } - if err := p.conn.SetReadDeadline(time.Time{}); err != nil { + if err := p.readWriter.SetReadDeadline(time.Time{}); err != nil { return false } return active @@ -297,7 +298,7 @@ func (p *PacketIO) LastKeepAlive() config.KeepAlive { } func (p *PacketIO) GracefulClose() error { - if err := p.conn.SetDeadline(time.Now()); err != nil && !errors.Is(err, net.ErrClosed) { + if err := p.readWriter.SetDeadline(time.Now()); err != nil && !errors.Is(err, net.ErrClosed) { return err } return nil @@ -311,7 +312,7 @@ func (p *PacketIO) Close() error { errs = append(errs, err) } */ - if err := p.conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + if err := p.readWriter.Close(); err != nil && !errors.Is(err, net.ErrClosed) { errs = append(errs, err) } return p.wrapErr(errors.Collect(ErrCloseConn, errs...)) diff --git a/pkg/proxy/net/packetio_options.go b/pkg/proxy/net/packetio_options.go index 83a0def0..0a815642 100644 --- a/pkg/proxy/net/packetio_options.go +++ b/pkg/proxy/net/packetio_options.go @@ -12,7 +12,7 @@ import ( type PacketIOption = func(*PacketIO) func WithProxy(pi *PacketIO) { - pi.proxyInited.Store(false) + pi.EnableProxyServer() } func WithWrapError(err error) func(pi *PacketIO) { diff --git a/pkg/proxy/net/proxy.go b/pkg/proxy/net/proxy.go index c86d5a22..14519947 100644 --- a/pkg/proxy/net/proxy.go +++ b/pkg/proxy/net/proxy.go @@ -6,13 +6,100 @@ package net import ( "bytes" "io" + "net" + "sync/atomic" "github.com/pingcap/tiproxy/lib/util/errors" "github.com/pingcap/tiproxy/pkg/proxy/proxyprotocol" ) -func (p *PacketIO) parseProxyV2() (*proxyprotocol.Proxy, error) { - rem, err := p.buf.Peek(8) +func (p *PacketIO) EnableProxyClient(proxy *proxyprotocol.Proxy) { + p.readWriter = newProxyClient(p.readWriter, proxy) +} + +func (p *PacketIO) EnableProxyServer() { + p.readWriter = newProxyServer(p.readWriter) +} + +// Proxy returned parsed proxy header from clients if any. +func (p *PacketIO) Proxy() *proxyprotocol.Proxy { + return p.readWriter.getProxy() +} + +var _ packetReadWriter = (*proxyReadWriter)(nil) + +type proxyReadWriter struct { + packetReadWriter + proxyInited atomic.Bool + proxy *proxyprotocol.Proxy + addr net.Addr + client bool +} + +func newProxyClient(rw packetReadWriter, proxy *proxyprotocol.Proxy) *proxyReadWriter { + prw := &proxyReadWriter{ + packetReadWriter: rw, + proxy: proxy, + client: true, + } + return prw +} + +func newProxyServer(rw packetReadWriter) *proxyReadWriter { + prw := &proxyReadWriter{ + packetReadWriter: rw, + client: false, + } + return prw +} + +func (prw *proxyReadWriter) Read(b []byte) (int, error) { + // probe proxy V2 + if !prw.client && !prw.proxyInited.Load() { + // We don't know whether the client has enabled proxy protocol. + // If it doesn't, reading data of len(MagicV2) may block forever. + header, err := prw.Peek(4) + if err != nil { + return 0, errors.Wrap(ErrReadConn, err) + } + if bytes.Equal(header[:], proxyprotocol.MagicV2[:4]) { + if _, err = prw.Discard(4); err != nil { + return 0, err + } + proxyHeader, err := prw.parseProxyV2() + if err != nil { + return 0, errors.Wrap(ErrReadConn, err) + } + if proxyHeader != nil { + prw.proxy = proxyHeader + } + } + prw.proxyInited.Store(true) + } + return prw.packetReadWriter.Read(b) +} + +func (prw *proxyReadWriter) Write(p []byte) (n int, err error) { + // The proxy header should be written at the beginning of connection, before any write operations. + if prw.client && !prw.proxyInited.Load() { + buf, err := prw.proxy.ToBytes() + if err != nil { + return 0, errors.Wrap(ErrWriteConn, err) + } + if _, err := io.Copy(prw.packetReadWriter, bytes.NewReader(buf)); err != nil { + return 0, errors.Wrap(ErrWriteConn, err) + } + // according to the spec, we better flush to avoid server hanging + if err := prw.packetReadWriter.Flush(); err != nil { + return 0, err + } + prw.proxyInited.Store(true) + } + return prw.Write(p) +} + +func (prw *proxyReadWriter) parseProxyV2() (*proxyprotocol.Proxy, error) { + rem, err := prw.packetReadWriter.Peek(8) if err != nil { return nil, errors.WithStack(errors.Wrap(ErrReadConn, err)) } @@ -21,31 +108,26 @@ func (p *PacketIO) parseProxyV2() (*proxyprotocol.Proxy, error) { } // yes, it is proxyV2 - _, err = p.buf.Discard(8) + _, err = prw.packetReadWriter.Discard(8) if err != nil { return nil, errors.WithStack(errors.Wrap(ErrReadConn, err)) } - p.inBytes += 8 - m, n, err := proxyprotocol.ParseProxyV2(p.buf) - p.inBytes += uint64(n) + m, _, err := proxyprotocol.ParseProxyV2(prw.packetReadWriter) if err == nil { // set RemoteAddr in case of proxy. - p.remoteAddr = m.SrcAddress + prw.addr = m.SrcAddress } return m, err } -// WriteProxyV2 should only be called at the beginning of connection, before any write operations. -func (p *PacketIO) WriteProxyV2(m *proxyprotocol.Proxy) error { - buf, err := m.ToBytes() - if err != nil { - return errors.Wrap(ErrWriteConn, err) +func (prw *proxyReadWriter) RemoteAddr() net.Addr { + if prw.addr != nil { + return prw.addr } - if _, err := io.Copy(p.buf, bytes.NewReader(buf)); err != nil { - return errors.Wrap(ErrWriteConn, err) - } - p.outBytes += uint64(len(buf)) - // according to the spec, we better flush to avoid server hanging - return p.Flush() + return prw.RemoteAddr() +} + +func (prw *proxyReadWriter) getProxy() *proxyprotocol.Proxy { + return prw.proxy } diff --git a/pkg/proxy/net/proxy_test.go b/pkg/proxy/net/proxy_test.go index 15aaa545..7fc61ae5 100644 --- a/pkg/proxy/net/proxy_test.go +++ b/pkg/proxy/net/proxy_test.go @@ -37,7 +37,7 @@ func TestProxyParse(t *testing.T) { } b, err := p.ToBytes() require.NoError(t, err) - _, err = io.Copy(cli.conn, bytes.NewReader(b)) + _, err = io.Copy(cli.readWriter, bytes.NewReader(b)) require.NoError(t, err) err = cli.WritePacket([]byte("hello"), true) require.NoError(t, err) diff --git a/pkg/proxy/net/tls.go b/pkg/proxy/net/tls.go index 226280f5..e21032b8 100644 --- a/pkg/proxy/net/tls.go +++ b/pkg/proxy/net/tls.go @@ -6,32 +6,82 @@ package net import ( "bufio" "crypto/tls" + "net" "github.com/pingcap/tiproxy/lib/util/errors" + "github.com/pingcap/tiproxy/pkg/proxy/proxyprotocol" ) +// tlsHandshakeConn is only used for TLS handshake. +// TLS handshake must read from the buffered reader because the handshake data may be already buffered in the reader. +// TLS handshake can not use the buffered writer directly because it assumes the data is always flushed immediately. +type tlsHandshakeConn struct { + packetReadWriter +} + +func (br *tlsHandshakeConn) Write(p []byte) (n int, err error) { + if n, err = br.packetReadWriter.Write(p); err != nil { + return + } + return n, br.packetReadWriter.Flush() +} + func (p *PacketIO) ServerTLSHandshake(tlsConfig *tls.Config) (tls.ConnectionState, error) { tlsConfig = tlsConfig.Clone() - tlsConn := tls.Server(p.conn, tlsConfig) + conn := &tlsHandshakeConn{p.readWriter} + tlsConn := tls.Server(conn, tlsConfig) if err := tlsConn.Handshake(); err != nil { return tls.ConnectionState{}, p.wrapErr(errors.Wrap(ErrHandshakeTLS, err)) } - p.conn = tlsConn - p.buf.Writer.Reset(tlsConn) - // Wrap it with another buffer to enable Peek. - p.buf = bufio.NewReadWriter(bufio.NewReaderSize(tlsConn, defaultReaderSize), p.buf.Writer) + p.readWriter = newTLSReadWriter(p.readWriter, tlsConn) return tlsConn.ConnectionState(), nil } func (p *PacketIO) ClientTLSHandshake(tlsConfig *tls.Config) error { tlsConfig = tlsConfig.Clone() - tlsConn := tls.Client(p.conn, tlsConfig) + conn := &tlsHandshakeConn{p.readWriter} + tlsConn := tls.Client(conn, tlsConfig) if err := tlsConn.Handshake(); err != nil { return errors.WithStack(errors.Wrap(ErrHandshakeTLS, err)) } - p.conn = tlsConn - p.buf.Writer.Reset(tlsConn) - // Wrap it with another buffer to enable Peek. - p.buf = bufio.NewReadWriter(bufio.NewReaderSize(tlsConn, defaultReaderSize), p.buf.Writer) + p.readWriter = newTLSReadWriter(p.readWriter, tlsConn) return nil } + +func (p *PacketIO) TLSConnectionState() tls.ConnectionState { + return p.readWriter.tlsConnectionState() +} + +var _ packetReadWriter = (*tlsReadWriter)(nil) + +type tlsReadWriter struct { + rdbufConn + rw packetReadWriter +} + +func newTLSReadWriter(rw packetReadWriter, tlsConn net.Conn) *tlsReadWriter { + // Can not modify rw and reuse it because tlsConn is using rw internally. + // So we must create another buffer. + buf := bufio.NewReadWriter(bufio.NewReaderSize(tlsConn, defaultReaderSize), bufio.NewWriterSize(tlsConn, defaultWriterSize)) + return &tlsReadWriter{ + rdbufConn: rdbufConn{ + Conn: tlsConn, + ReadWriter: buf, + inBytes: rw.getInBytes(), + outBytes: rw.getOutBytes(), + }, + rw: rw, + } +} + +func (trw *tlsReadWriter) afterRead() { + trw.rw.afterRead() +} + +func (trw *tlsReadWriter) reset() { + trw.rw.reset() +} + +func (trw *tlsReadWriter) getProxy() *proxyprotocol.Proxy { + return trw.rw.getProxy() +} From 43ec69c6118f70d577c274878db6267add4e2576 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Sat, 23 Sep 2023 10:58:37 +0800 Subject: [PATCH 02/15] set uncompressed sequence --- pkg/proxy/net/compress.go | 299 +++++++++++++++++++++++++------------- pkg/proxy/net/packetio.go | 138 ++++++++++-------- pkg/proxy/net/proxy.go | 4 +- pkg/proxy/net/tls.go | 67 +++++---- 4 files changed, 317 insertions(+), 191 deletions(-) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index f883c8af..20f71bb8 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -10,6 +10,7 @@ import ( "github.com/klauspost/compress/zstd" "github.com/pingcap/tiproxy/lib/util/errors" + "go.uber.org/zap" ) // CompressAlgorithm is the algorithm for MySQL compressed protocol. @@ -24,31 +25,34 @@ const ( CompressionZstd ) +type rwStatus int + const ( - // maxCompressedSize is the max size for compressed data. - // MySQL starts with `net_buffer_length` (default 16384) and larger packets after that. + rwNone rwStatus = iota + rwRead + rwWrite +) + +const ( + // maxCompressedSize is the max uncompressed data size for a compressed packet. + // Packets bigger than maxCompressedSize will be split into multiple compressed packets. + // MySQL is 16K for the first packet and the rest for the second, MySQL Connector/J is 16M. // The length itself must fit in the 3 byte field in the header. - // Can't be bigger then the max value for `net_buffer_length` (1048576) - maxCompressedSize = 1024 * 1024 - // minCompressSize is the min size for compressed data. - // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_compression_packet.html - // suggests a MIN_COMPRESS_LENGTH of 50. + maxCompressedSize = 1 << 22 + // minCompressSize is the min uncompressed data size for compressed data. + // Packets smaller than minCompressSize won't be compressed. + // MySQL and MySQL Connector/J are both 50. minCompressSize = 50 ) func (p *PacketIO) SetCompressionAlgorithm(algorithm CompressAlgorithm, zstdLevel int) error { switch algorithm { case CompressionZlib, CompressionZstd: + p.readWriter = newCompressedReadWriter(p.readWriter, algorithm, zstdLevel, p.logger) case CompressionNone: - return nil default: return errors.Errorf("Unknown compression algorithm %d", algorithm) } - rw, err := newCompressedReadWriter(p.readWriter, algorithm, zstdLevel) - if err != nil { - return err - } - p.readWriter = rw return nil } @@ -56,123 +60,142 @@ var _ packetReadWriter = (*compressedReadWriter)(nil) type compressedReadWriter struct { packetReadWriter - compressedReader io.ReadCloser - buf *bytes.Buffer - algorithm CompressAlgorithm - zstdLevel zstd.EncoderLevel - sequence uint8 + readBuffer []byte + writeBuffer bytes.Buffer + algorithm CompressAlgorithm + zstdLevel zstd.EncoderLevel + logger *zap.Logger + rwStatus rwStatus + sequence uint8 } -func newCompressedReadWriter(rw packetReadWriter, algorithm CompressAlgorithm, zstdLevel int) (*compressedReadWriter, error) { - if err := rw.Flush(); err != nil { - return nil, err - } +func newCompressedReadWriter(rw packetReadWriter, algorithm CompressAlgorithm, zstdLevel int, logger *zap.Logger) *compressedReadWriter { return &compressedReadWriter{ packetReadWriter: rw, - buf: new(bytes.Buffer), algorithm: algorithm, zstdLevel: zstd.EncoderLevelFromZstd(zstdLevel), - }, nil + logger: logger, + rwStatus: rwNone, + } } -func (crw *compressedReadWriter) reset() { - crw.sequence = 0 +func (crw *compressedReadWriter) SetSequence(sequence uint8) { + crw.packetReadWriter.SetSequence(sequence) + // Reset the compressed sequence before the next command. + if sequence == 0 { + crw.sequence = 0 + crw.rwStatus = rwNone + } } -func (crw *compressedReadWriter) afterRead() { - if crw.compressedReader != nil { - _ = crw.compressedReader.Close() - crw.compressedReader = nil +// Uncompressed sequence of MySQL doesn't follow the spec: it's set to the compressed sequence when +// the client/server begins reading or writing. +func (crw *compressedReadWriter) beginRW(status rwStatus) { + if crw.rwStatus != status { + crw.packetReadWriter.SetSequence(crw.sequence) + crw.rwStatus = status } } func (crw *compressedReadWriter) Read(p []byte) (n int, err error) { - if crw.compressedReader == nil { - var compressedHeader [7]byte - if _, err := io.ReadFull(crw.packetReadWriter, compressedHeader[:]); err != nil { - return 0, errors.WithStack(err) - } - compressedSequence := compressedHeader[3] - if compressedSequence != crw.sequence { - return 0, ErrInvalidSequence.GenWithStack( - "invalid compressed sequence, expected %d, actual %d", crw.sequence, compressedSequence) - } - crw.sequence++ - uncompressedLength := int(uint32(compressedHeader[4]) | uint32(compressedHeader[5])<<8 | uint32(compressedHeader[6])<<16) - if uncompressedLength <= 0 { - return crw.packetReadWriter.Read(p) + crw.beginRW(rwRead) + // Read from the connection to fill the buffer if the buffer is empty. + if len(crw.readBuffer) == 0 { + if err = crw.readFromConn(); err != nil { + return } + } + n = copy(p, crw.readBuffer) + if n == len(crw.readBuffer) { + // Free the buffer. Too many idle connections may hold too much unnecessary memory. + crw.readBuffer = nil + } else { + crw.readBuffer = crw.readBuffer[n:] + } + return +} - switch crw.algorithm { - case CompressionZlib: - crw.compressedReader, err = zlib.NewReader(crw.packetReadWriter) - case CompressionZstd: - var zstdReader *zstd.Decoder - if zstdReader, err = zstd.NewReader(crw.packetReadWriter, zstd.WithDecoderConcurrency(1)); err == nil { - crw.compressedReader = zstdReader.IOReadCloser() - } +// Read and uncompress the data into readBuffer. +// The format of the protocol: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_compression_packet.html +func (crw *compressedReadWriter) readFromConn() error { + var err error + var header [7]byte + if _, err = io.ReadFull(crw.packetReadWriter, header[:]); err != nil { + return err + } + compressedSequence := header[3] + if compressedSequence != crw.sequence { + return ErrInvalidSequence.GenWithStack( + "invalid compressed sequence, expected %d, actual %d", crw.sequence, compressedSequence) + } + crw.sequence++ + compressedLength := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) + uncompressedLength := int(uint32(header[4]) | uint32(header[5])<<8 | uint32(header[6])<<16) + + if uncompressedLength == 0 { + // If the data is uncompressed, the uncompressed length is 0 and compressed length is the data length + // after the compressed header. + crw.readBuffer = make([]byte, compressedLength) + if _, err = io.ReadFull(crw.packetReadWriter, crw.readBuffer); err != nil { + return err } - if err != nil { - return 0, errors.WithStack(err) + } else { + // If the data is compressed, the compressed length is the length of data after the compressed header and + // the uncompressed length is the length of data after uncompression. + data := make([]byte, compressedLength) + if _, err = io.ReadFull(crw.packetReadWriter, data); err != nil { + return err + } + if crw.readBuffer, err = crw.uncompress(data, uncompressedLength); err != nil { + return err } } - return crw.compressedReader.Read(p) + return nil } func (crw *compressedReadWriter) Write(data []byte) (n int, err error) { + crw.beginRW(rwWrite) for { - remainingLen := maxCompressedSize - crw.buf.Len() + remainingLen := maxCompressedSize - crw.writeBuffer.Len() if len(data) <= remainingLen { - written, err := crw.buf.Write(data) + written, err := crw.writeBuffer.Write(data) if err != nil { - return 0, err + return n, err } return n + written, nil } - written, err := crw.buf.Write(data[:remainingLen]) + written, err := crw.writeBuffer.Write(data[:remainingLen]) if err != nil { - return 0, err + return n, err } n += written data = data[remainingLen:] - if err = crw.flush(); err != nil { - return 0, err + if err = crw.Flush(); err != nil { + return n, err } } } -func (crw *compressedReadWriter) flush() error { - var payload, compressedPacket bytes.Buffer - var w io.WriteCloser +func (crw *compressedReadWriter) Flush() error { var err error + data := crw.writeBuffer.Bytes() + crw.writeBuffer.Reset() - data := crw.buf.Bytes() - crw.buf.Reset() - - switch crw.algorithm { - case CompressionZlib: - w, err = zlib.NewWriterLevel(&payload, zlib.HuffmanOnly) - case CompressionZstd: - w, err = zstd.NewWriter(&payload, zstd.WithEncoderLevel(crw.zstdLevel)) - } - if err != nil { - return errors.WithStack(err) - } - + // If the data is uncompressed, the uncompressed length is 0 and compressed length is the data length + // after the compressed header. uncompressedLength := 0 compressedLength := len(data) - compressedHeader := make([]byte, 7) - - if len(data) > minCompressSize { + if len(data) >= minCompressSize { + // If the data is compressed, the compressed length is the length of data after the compressed header and + // the uncompressed length is the length of data after uncompression. uncompressedLength = len(data) - if _, err = w.Write(data); err != nil { - return errors.WithStack(err) + if data, err = crw.compress(data); err != nil { + return err } - if err = w.Close(); err != nil { - return errors.WithStack(err) - } - compressedLength = len(payload.Bytes()) + compressedLength = len(data) } + + var compressedHeader [7]byte compressedHeader[0] = byte(compressedLength) compressedHeader[1] = byte(compressedLength >> 8) compressedHeader[2] = byte(compressedLength >> 16) @@ -180,24 +203,102 @@ func (crw *compressedReadWriter) flush() error { compressedHeader[4] = byte(uncompressedLength) compressedHeader[5] = byte(uncompressedLength >> 8) compressedHeader[6] = byte(uncompressedLength >> 16) - if _, err = compressedPacket.Write(compressedHeader); err != nil { + crw.sequence++ + if _, err = crw.packetReadWriter.Write(compressedHeader[:]); err != nil { return errors.WithStack(err) } - crw.sequence++ + if _, err = crw.packetReadWriter.Write(data); err != nil { + return errors.WithStack(err) + } + return crw.packetReadWriter.Flush() +} - if len(data) > minCompressSize { - _, err = compressedPacket.Write(payload.Bytes()) - } else { - _, err = compressedPacket.Write(data) +// Peek won't be used. +// Notice: the peeked data may be discarded if an error is returned. +func (crw *compressedReadWriter) Peek(n int) (data []byte, err error) { + crw.beginRW(rwRead) + var readBuffer []byte + for len(readBuffer) < n { + if len(crw.readBuffer) == 0 { + if err = crw.readFromConn(); err != nil { + return + } + } + readBuffer = append(readBuffer, crw.readBuffer...) + crw.readBuffer = nil + } + data = make([]byte, 0, n) + copy(data, readBuffer) + crw.readBuffer = readBuffer + return +} + +// Discard won't be used. +func (crw *compressedReadWriter) Discard(n int) (d int, err error) { + crw.beginRW(rwRead) + for left := n; left > 0; { + if len(crw.readBuffer) == 0 { + if err = crw.readFromConn(); err != nil { + return + } + } + if left >= len(crw.readBuffer) { + left -= len(crw.readBuffer) + crw.readBuffer = nil + d += len(crw.readBuffer) + } else { + left = 0 + crw.readBuffer = crw.readBuffer[left:] + d += left + } + } + return +} + +func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { + var err error + var compressedPacket bytes.Buffer + var compressWriter io.WriteCloser + switch crw.algorithm { + case CompressionZlib: + compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.HuffmanOnly) + case CompressionZstd: + compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(crw.zstdLevel)) } if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } - if err = w.Close(); err != nil { - return errors.WithStack(err) + if _, err = compressWriter.Write(data); err != nil { + return nil, errors.WithStack(err) } - if _, err = crw.packetReadWriter.Write(compressedPacket.Bytes()); err != nil { - return errors.WithStack(err) + if err = compressWriter.Close(); err != nil { + return nil, errors.WithStack(err) } - return nil + return compressedPacket.Bytes(), nil +} + +func (crw *compressedReadWriter) uncompress(data []byte, uncompressedLength int) ([]byte, error) { + var err error + var compressedReader io.ReadCloser + switch crw.algorithm { + case CompressionZlib: + compressedReader, err = zlib.NewReader(bytes.NewReader(data)) + case CompressionZstd: + var zstdReader *zstd.Decoder + if zstdReader, err = zstd.NewReader(bytes.NewReader(data), zstd.WithDecoderConcurrency(1)); err == nil { + compressedReader = zstdReader.IOReadCloser() + } + } + if err != nil { + return nil, errors.WithStack(err) + } + + uncompressed := make([]byte, uncompressedLength) + if _, err = io.ReadFull(compressedReader, uncompressed); err != nil { + return nil, errors.WithStack(err) + } + if err = compressedReader.Close(); err != nil { + return nil, errors.WithStack(err) + } + return uncompressed, nil } diff --git a/pkg/proxy/net/packetio.go b/pkg/proxy/net/packetio.go index 32d95b15..ef7cd210 100644 --- a/pkg/proxy/net/packetio.go +++ b/pkg/proxy/net/packetio.go @@ -50,75 +50,108 @@ const ( defaultReaderSize = 16 * 1024 ) -type buffer interface { +// The functions in bufferedIO are implemented by bufio.ReadWriter. +type bufferedIO interface { Peek(n int) ([]byte, error) Discard(n int) (int, error) Flush() error } +// packetReadWriter acts like a net.Conn with read and write buffer. type packetReadWriter interface { net.Conn - buffer - afterRead() - reset() - getProxy() *proxyprotocol.Proxy - tlsConnectionState() tls.ConnectionState - getInBytes() uint64 - getOutBytes() uint64 + bufferedIO + DirectWrite(p []byte) (int, error) + Proxy() *proxyprotocol.Proxy + TLSConnectionState() tls.ConnectionState + InBytes() uint64 + OutBytes() uint64 + IsPeerActive() bool + SetSequence(uint8) + Sequence() uint8 } -// rdbufConn buffers read for non-TLS connections. -type rdbufConn struct { +var _ packetReadWriter = (*basicReadWriter)(nil) + +// basicReadWriter is used for raw connections. +type basicReadWriter struct { net.Conn *bufio.ReadWriter inBytes uint64 outBytes uint64 + sequence uint8 } -func newRdbufConn(conn net.Conn) *rdbufConn { - return &rdbufConn{ +func newBasicBufConn(conn net.Conn) *basicReadWriter { + return &basicReadWriter{ Conn: conn, ReadWriter: bufio.NewReadWriter(bufio.NewReaderSize(conn, defaultReaderSize), bufio.NewWriterSize(conn, defaultWriterSize)), } } -func (f *rdbufConn) Read(b []byte) (n int, err error) { - n, err = f.ReadWriter.Read(b) - f.inBytes += uint64(n) - return n, err +func (brw *basicReadWriter) Read(b []byte) (n int, err error) { + n, err = brw.ReadWriter.Read(b) + brw.inBytes += uint64(n) + return n, errors.WithStack(err) +} + +func (brw *basicReadWriter) Write(p []byte) (int, error) { + n, err := brw.ReadWriter.Write(p) + brw.outBytes += uint64(n) + return n, errors.WithStack(err) } -func (f *rdbufConn) Write(p []byte) (n int, err error) { - n, err = f.ReadWriter.Write(p) - f.outBytes += uint64(n) - return n, err +func (brw *basicReadWriter) DirectWrite(p []byte) (int, error) { + n, err := brw.Conn.Write(p) + brw.outBytes += uint64(n) + return n, errors.WithStack(err) } -func (f *rdbufConn) afterRead() { +func (brw *basicReadWriter) SetSequence(sequence uint8) { + brw.sequence = sequence } -func (f *rdbufConn) reset() { +func (brw *basicReadWriter) Sequence() uint8 { + return brw.sequence } -func (f *rdbufConn) getProxy() *proxyprotocol.Proxy { +func (brw *basicReadWriter) Proxy() *proxyprotocol.Proxy { return nil } -func (f *rdbufConn) getInBytes() uint64 { - return f.inBytes +func (brw *basicReadWriter) InBytes() uint64 { + return brw.inBytes } -func (f *rdbufConn) getOutBytes() uint64 { - return f.outBytes +func (brw *basicReadWriter) OutBytes() uint64 { + return brw.outBytes } -func (f *rdbufConn) tlsConnectionState() tls.ConnectionState { - if conn, ok := f.Conn.(*tls.Conn); ok { - return conn.ConnectionState() - } +func (brw *basicReadWriter) TLSConnectionState() tls.ConnectionState { return tls.ConnectionState{} } +// IsPeerActive checks if the peer connection is still active. +// If the backend disconnects, the client should also be disconnected (required by serverless). +// We have no other way than reading from the connection. +// +// This function cannot be called concurrently with other functions of packetReadWriter. +// This function normally costs 1ms, so don't call it too frequently. +// This function may incorrectly return true if the system is extremely slow. +func (brw *basicReadWriter) IsPeerActive() bool { + if err := brw.Conn.SetReadDeadline(time.Now().Add(time.Millisecond)); err != nil { + return false + } + active := true + if _, err := brw.ReadWriter.Peek(1); err != nil { + active = !errors.Is(err, io.EOF) + } + if err := brw.Conn.SetReadDeadline(time.Time{}); err != nil { + return false + } + return active +} + // PacketIO is a helper to read and write sql and proxy protocol. type PacketIO struct { lastKeepAlive config.KeepAlive @@ -127,15 +160,13 @@ type PacketIO struct { logger *zap.Logger remoteAddr net.Addr wrap error - sequence uint8 } func NewPacketIO(conn net.Conn, lg *zap.Logger, opts ...PacketIOption) *PacketIO { p := &PacketIO{ rawConn: conn, logger: lg, - sequence: 0, - readWriter: newRdbufConn(conn), + readWriter: newBasicBufConn(conn), } p.ApplyOpts(opts...) return p @@ -163,26 +194,24 @@ func (p *PacketIO) RemoteAddr() net.Addr { } func (p *PacketIO) ResetSequence() { - p.sequence = 0 - p.readWriter.reset() + p.readWriter.SetSequence(0) } // GetSequence is used in tests to assert that the sequences on the client and server are equal. func (p *PacketIO) GetSequence() uint8 { - return p.sequence + return p.readWriter.Sequence() } func (p *PacketIO) readOnePacket() ([]byte, bool, error) { var header [4]byte - defer p.readWriter.afterRead() if _, err := io.ReadFull(p.readWriter, header[:]); err != nil { return nil, false, errors.Wrap(ErrReadConn, err) } - sequence := header[3] - if sequence != p.sequence { - return nil, false, ErrInvalidSequence.GenWithStack("invalid sequence, expected %d, actual %d", p.sequence, sequence) + sequence, pktSequence := header[3], p.readWriter.Sequence() + if sequence != pktSequence { + return nil, false, ErrInvalidSequence.GenWithStack("invalid sequence, expected %d, actual %d", pktSequence, sequence) } - p.sequence++ + p.readWriter.SetSequence(sequence + 1) length := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) data := make([]byte, length) @@ -217,11 +246,12 @@ func (p *PacketIO) writeOnePacket(data []byte) (int, bool, error) { } var header [4]byte + sequence := p.readWriter.Sequence() header[0] = byte(length) header[1] = byte(length >> 8) header[2] = byte(length >> 16) - header[3] = p.sequence - p.sequence++ + header[3] = sequence + p.readWriter.SetSequence(sequence + 1) if _, err := io.Copy(p.readWriter, bytes.NewReader(header[:])); err != nil { return 0, more, errors.Wrap(ErrWriteConn, err) @@ -252,11 +282,11 @@ func (p *PacketIO) WritePacket(data []byte, flush bool) (err error) { } func (p *PacketIO) InBytes() uint64 { - return p.readWriter.getInBytes() + return p.readWriter.InBytes() } func (p *PacketIO) OutBytes() uint64 { - return p.readWriter.getOutBytes() + return p.readWriter.OutBytes() } func (p *PacketIO) Flush() error { @@ -266,22 +296,8 @@ func (p *PacketIO) Flush() error { return nil } -// IsPeerActive checks if the peer connection is still active. -// This function cannot be called concurrently with other functions of PacketIO. -// This function normally costs 1ms, so don't call it too frequently. -// This function may incorrectly return true if the system is extremely slow. func (p *PacketIO) IsPeerActive() bool { - if err := p.readWriter.SetReadDeadline(time.Now().Add(time.Millisecond)); err != nil { - return false - } - active := true - if _, err := p.readWriter.Peek(1); err != nil { - active = !errors.Is(err, io.EOF) - } - if err := p.readWriter.SetReadDeadline(time.Time{}); err != nil { - return false - } - return active + return p.readWriter.IsPeerActive() } func (p *PacketIO) SetKeepalive(cfg config.KeepAlive) error { diff --git a/pkg/proxy/net/proxy.go b/pkg/proxy/net/proxy.go index 14519947..b746e285 100644 --- a/pkg/proxy/net/proxy.go +++ b/pkg/proxy/net/proxy.go @@ -23,7 +23,7 @@ func (p *PacketIO) EnableProxyServer() { // Proxy returned parsed proxy header from clients if any. func (p *PacketIO) Proxy() *proxyprotocol.Proxy { - return p.readWriter.getProxy() + return p.readWriter.Proxy() } var _ packetReadWriter = (*proxyReadWriter)(nil) @@ -128,6 +128,6 @@ func (prw *proxyReadWriter) RemoteAddr() net.Addr { return prw.RemoteAddr() } -func (prw *proxyReadWriter) getProxy() *proxyprotocol.Proxy { +func (prw *proxyReadWriter) Proxy() *proxyprotocol.Proxy { return prw.proxy } diff --git a/pkg/proxy/net/tls.go b/pkg/proxy/net/tls.go index e21032b8..4170af7b 100644 --- a/pkg/proxy/net/tls.go +++ b/pkg/proxy/net/tls.go @@ -6,29 +6,24 @@ package net import ( "bufio" "crypto/tls" - "net" "github.com/pingcap/tiproxy/lib/util/errors" - "github.com/pingcap/tiproxy/pkg/proxy/proxyprotocol" ) -// tlsHandshakeConn is only used for TLS handshake. +// tlsHandshakeConn is only used as the underlying connection in tls.Conn. // TLS handshake must read from the buffered reader because the handshake data may be already buffered in the reader. // TLS handshake can not use the buffered writer directly because it assumes the data is always flushed immediately. -type tlsHandshakeConn struct { +type tlsInternalConn struct { packetReadWriter } -func (br *tlsHandshakeConn) Write(p []byte) (n int, err error) { - if n, err = br.packetReadWriter.Write(p); err != nil { - return - } - return n, br.packetReadWriter.Flush() +func (br *tlsInternalConn) Write(p []byte) (n int, err error) { + return br.packetReadWriter.DirectWrite(p) } func (p *PacketIO) ServerTLSHandshake(tlsConfig *tls.Config) (tls.ConnectionState, error) { tlsConfig = tlsConfig.Clone() - conn := &tlsHandshakeConn{p.readWriter} + conn := &tlsInternalConn{p.readWriter} tlsConn := tls.Server(conn, tlsConfig) if err := tlsConn.Handshake(); err != nil { return tls.ConnectionState{}, p.wrapErr(errors.Wrap(ErrHandshakeTLS, err)) @@ -39,7 +34,7 @@ func (p *PacketIO) ServerTLSHandshake(tlsConfig *tls.Config) (tls.ConnectionStat func (p *PacketIO) ClientTLSHandshake(tlsConfig *tls.Config) error { tlsConfig = tlsConfig.Clone() - conn := &tlsHandshakeConn{p.readWriter} + conn := &tlsInternalConn{p.readWriter} tlsConn := tls.Client(conn, tlsConfig) if err := tlsConn.Handshake(); err != nil { return errors.WithStack(errors.Wrap(ErrHandshakeTLS, err)) @@ -49,39 +44,53 @@ func (p *PacketIO) ClientTLSHandshake(tlsConfig *tls.Config) error { } func (p *PacketIO) TLSConnectionState() tls.ConnectionState { - return p.readWriter.tlsConnectionState() + return p.readWriter.TLSConnectionState() } var _ packetReadWriter = (*tlsReadWriter)(nil) type tlsReadWriter struct { - rdbufConn - rw packetReadWriter + packetReadWriter + buf *bufio.ReadWriter + conn *tls.Conn } -func newTLSReadWriter(rw packetReadWriter, tlsConn net.Conn) *tlsReadWriter { +func newTLSReadWriter(rw packetReadWriter, tlsConn *tls.Conn) *tlsReadWriter { // Can not modify rw and reuse it because tlsConn is using rw internally. - // So we must create another buffer. + // We must create another buffer. buf := bufio.NewReadWriter(bufio.NewReaderSize(tlsConn, defaultReaderSize), bufio.NewWriterSize(tlsConn, defaultWriterSize)) return &tlsReadWriter{ - rdbufConn: rdbufConn{ - Conn: tlsConn, - ReadWriter: buf, - inBytes: rw.getInBytes(), - outBytes: rw.getOutBytes(), - }, - rw: rw, + packetReadWriter: rw, + buf: buf, + conn: tlsConn, } } -func (trw *tlsReadWriter) afterRead() { - trw.rw.afterRead() +func (trw *tlsReadWriter) Read(b []byte) (n int, err error) { + // inBytes and outBytes are updated internally in trw.packetReadWriter. + return trw.buf.Read(b) +} + +func (trw *tlsReadWriter) Write(p []byte) (int, error) { + return trw.buf.Write(p) +} + +func (trw *tlsReadWriter) DirectWrite(p []byte) (int, error) { + return trw.conn.Write(p) +} + +func (trw *tlsReadWriter) Peek(n int) ([]byte, error) { + return trw.buf.Peek(n) +} + +func (trw *tlsReadWriter) Discard(n int) (int, error) { + return trw.buf.Discard(n) } -func (trw *tlsReadWriter) reset() { - trw.rw.reset() +func (trw *tlsReadWriter) Flush() error { + return trw.buf.Flush() } -func (trw *tlsReadWriter) getProxy() *proxyprotocol.Proxy { - return trw.rw.getProxy() +func (trw *tlsReadWriter) TLSConnectionState() tls.ConnectionState { + return trw.conn.ConnectionState() } From 4ba508e84ac68d8296e196769d8971e525bce28e Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Tue, 26 Sep 2023 11:12:18 +0800 Subject: [PATCH 03/15] use datadog/zstd --- go.mod | 2 +- go.sum | 6 +++--- pkg/proxy/backend/authenticator.go | 1 + pkg/proxy/net/compress.go | 30 ++++++++++++------------------ 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 19390e9a..a26a5410 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/BurntSushi/toml v1.2.1 + github.com/DataDog/zstd v1.5.5 github.com/bahlo/generic-list-go v0.2.0 github.com/cenkalti/backoff/v4 v4.2.1 github.com/fsnotify/fsnotify v1.6.0 @@ -11,7 +12,6 @@ require ( github.com/gin-gonic/gin v1.8.1 github.com/go-mysql-org/go-mysql v1.6.0 github.com/go-sql-driver/mysql v1.7.0 - github.com/klauspost/compress v1.16.5 github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc github.com/pingcap/tiproxy/lib v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index 3539c9e2..877a3507 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,8 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -415,8 +416,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/pkg/proxy/backend/authenticator.go b/pkg/proxy/backend/authenticator.go index 2616267f..e042ca45 100644 --- a/pkg/proxy/backend/authenticator.go +++ b/pkg/proxy/backend/authenticator.go @@ -319,6 +319,7 @@ func (auth *Authenticator) writeAuthHandshake( AuthData: authData, Capability: auth.capability | authCap, AuthPlugin: authPlugin, + ZstdLevel: auth.zstdLevel, } if len(resp.Attrs) > 0 { diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 20f71bb8..3dc7d57e 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -8,7 +8,7 @@ import ( "compress/zlib" "io" - "github.com/klauspost/compress/zstd" + "github.com/DataDog/zstd" "github.com/pingcap/tiproxy/lib/util/errors" "go.uber.org/zap" ) @@ -38,7 +38,7 @@ const ( // Packets bigger than maxCompressedSize will be split into multiple compressed packets. // MySQL is 16K for the first packet and the rest for the second, MySQL Connector/J is 16M. // The length itself must fit in the 3 byte field in the header. - maxCompressedSize = 1 << 22 + maxCompressedSize = 1<<24 - 1 // minCompressSize is the min uncompressed data size for compressed data. // Packets smaller than minCompressSize won't be compressed. // MySQL and MySQL Connector/J are both 50. @@ -63,9 +63,9 @@ type compressedReadWriter struct { readBuffer []byte writeBuffer bytes.Buffer algorithm CompressAlgorithm - zstdLevel zstd.EncoderLevel logger *zap.Logger rwStatus rwStatus + zstdLevel int sequence uint8 } @@ -73,7 +73,7 @@ func newCompressedReadWriter(rw packetReadWriter, algorithm CompressAlgorithm, z return &compressedReadWriter{ packetReadWriter: rw, algorithm: algorithm, - zstdLevel: zstd.EncoderLevelFromZstd(zstdLevel), + zstdLevel: zstdLevel, logger: logger, rwStatus: rwNone, } @@ -261,12 +261,11 @@ func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { var compressWriter io.WriteCloser switch crw.algorithm { case CompressionZlib: - compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.HuffmanOnly) + if compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.DefaultCompression); err != nil { + return nil, errors.WithStack(err) + } case CompressionZstd: - compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(crw.zstdLevel)) - } - if err != nil { - return nil, errors.WithStack(err) + compressWriter = zstd.NewWriterLevel(&compressedPacket, crw.zstdLevel) } if _, err = compressWriter.Write(data); err != nil { return nil, errors.WithStack(err) @@ -282,17 +281,12 @@ func (crw *compressedReadWriter) uncompress(data []byte, uncompressedLength int) var compressedReader io.ReadCloser switch crw.algorithm { case CompressionZlib: - compressedReader, err = zlib.NewReader(bytes.NewReader(data)) - case CompressionZstd: - var zstdReader *zstd.Decoder - if zstdReader, err = zstd.NewReader(bytes.NewReader(data), zstd.WithDecoderConcurrency(1)); err == nil { - compressedReader = zstdReader.IOReadCloser() + if compressedReader, err = zlib.NewReader(bytes.NewReader(data)); err != nil { + return nil, errors.WithStack(err) } + case CompressionZstd: + compressedReader = zstd.NewReader(bytes.NewReader(data)) } - if err != nil { - return nil, errors.WithStack(err) - } - uncompressed := make([]byte, uncompressedLength) if _, err = io.ReadFull(compressedReader, uncompressed); err != nil { return nil, errors.WithStack(err) From 7a5c676968c66ec988c85e49c7cdd8dc0b5e0182 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 27 Sep 2023 21:06:46 +0800 Subject: [PATCH 04/15] add tests --- pkg/proxy/backend/authenticator.go | 28 +-- pkg/proxy/backend/authenticator_test.go | 127 +++++++++++ pkg/proxy/backend/backend_conn_mgr_test.go | 20 +- pkg/proxy/backend/common_test.go | 18 ++ pkg/proxy/backend/mock_backend_test.go | 13 +- pkg/proxy/backend/mock_client_test.go | 2 + pkg/proxy/backend/testsuite_test.go | 5 +- pkg/proxy/net/compress.go | 15 +- pkg/proxy/net/compress_test.go | 252 +++++++++++++++++++++ pkg/proxy/net/packetio.go | 16 +- pkg/proxy/net/packetio_test.go | 75 ++++++ pkg/proxy/net/proxy.go | 4 +- pkg/proxy/net/proxy_test.go | 65 ++++-- pkg/proxy/net/tls_test.go | 84 +++++++ 14 files changed, 660 insertions(+), 64 deletions(-) create mode 100644 pkg/proxy/net/compress_test.go create mode 100644 pkg/proxy/net/tls_test.go diff --git a/pkg/proxy/backend/authenticator.go b/pkg/proxy/backend/authenticator.go index e042ca45..2889bdce 100644 --- a/pkg/proxy/backend/authenticator.go +++ b/pkg/proxy/backend/authenticator.go @@ -226,10 +226,10 @@ loop: pktIdx++ switch serverPkt[0] { case pnet.OKHeader.Byte(): - if err := auth.setCompress(clientIO, auth.capability); err != nil { + if err := setCompress(clientIO, auth.capability, auth.zstdLevel); err != nil { return err } - if err := auth.setCompress(backendIO, auth.capability&backendCapability); err != nil { + if err := setCompress(backendIO, auth.capability&backendCapability, auth.zstdLevel); err != nil { return err } return nil @@ -285,7 +285,7 @@ func (auth *Authenticator) handshakeSecondTime(logger *zap.Logger, clientIO, bac } if err = auth.handleSecondAuthResult(backendIO); err == nil { - return auth.setCompress(backendIO, auth.capability&backendCapability) + return setCompress(backendIO, auth.capability&backendCapability, auth.zstdLevel) } return err } @@ -317,7 +317,7 @@ func (auth *Authenticator) writeAuthHandshake( Attrs: auth.attrs, Collation: auth.collation, AuthData: authData, - Capability: auth.capability | authCap, + Capability: auth.capability&backendCapability | authCap, AuthPlugin: authPlugin, ZstdLevel: auth.zstdLevel, } @@ -381,16 +381,6 @@ func (auth *Authenticator) handleSecondAuthResult(backendIO *pnet.PacketIO) erro } } -func (auth *Authenticator) setCompress(clientIO *pnet.PacketIO, capability pnet.Capability) error { - algorithm := pnet.CompressionNone - if capability&pnet.ClientCompress > 0 { - algorithm = pnet.CompressionZlib - } else if capability&pnet.ClientZstdCompressionAlgorithm > 0 { - algorithm = pnet.CompressionZstd - } - return clientIO.SetCompressionAlgorithm(algorithm, auth.zstdLevel) -} - // changeUser is called once the client sends COM_CHANGE_USER. func (auth *Authenticator) changeUser(req *pnet.ChangeUserReq) { auth.user = req.User @@ -403,3 +393,13 @@ func (auth *Authenticator) changeUser(req *pnet.ChangeUserReq) { func (auth *Authenticator) updateCurrentDB(db string) { auth.dbname = db } + +func setCompress(packetIO *pnet.PacketIO, capability pnet.Capability, zstdLevel int) error { + algorithm := pnet.CompressionNone + if capability&pnet.ClientCompress > 0 { + algorithm = pnet.CompressionZlib + } else if capability&pnet.ClientZstdCompressionAlgorithm > 0 { + algorithm = pnet.CompressionZstd + } + return packetIO.SetCompressionAlgorithm(algorithm, zstdLevel) +} diff --git a/pkg/proxy/backend/authenticator_test.go b/pkg/proxy/backend/authenticator_test.go index 9fcbc89c..8de40e0f 100644 --- a/pkg/proxy/backend/authenticator_test.go +++ b/pkg/proxy/backend/authenticator_test.go @@ -164,6 +164,30 @@ func TestCapability(t *testing.T) { cfg.clientConfig.capability |= pnet.ClientSecureConnection }, }, + { + func(cfg *testConfig) { + cfg.backendConfig.capability &= ^pnet.ClientCompress + cfg.backendConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.backendConfig.capability |= pnet.ClientCompress + cfg.backendConfig.capability |= pnet.ClientZstdCompressionAlgorithm + }, + }, + { + func(cfg *testConfig) { + cfg.clientConfig.capability &= ^pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability |= pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + }, } tc := newTCPConnSuite(t) @@ -387,3 +411,106 @@ func TestProxyProtocol(t *testing.T) { clean() } } + +func TestCompressProtocol(t *testing.T) { + cfgs := [][]cfgOverrider{ + { + func(cfg *testConfig) { + cfg.backendConfig.capability &= ^pnet.ClientCompress + cfg.backendConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.backendConfig.capability |= pnet.ClientCompress + cfg.backendConfig.capability |= pnet.ClientZstdCompressionAlgorithm + }, + }, + { + func(cfg *testConfig) { + cfg.clientConfig.capability &= ^pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability |= pnet.ClientZstdCompressionAlgorithm + cfg.clientConfig.zstdLevel = 3 + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability |= pnet.ClientZstdCompressionAlgorithm + cfg.clientConfig.zstdLevel = 9 + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + }, + } + + tc := newTCPConnSuite(t) + cfgOverriders := getCfgCombinations(cfgs) + checker := func(t *testing.T, ts *testSuite, referCfg *testConfig) { + // If the client enables compression, client <-> proxy enables compression. + if referCfg.clientConfig.capability&pnet.ClientCompress > 0 { + require.Greater(t, ts.mp.authenticator.capability&pnet.ClientCompress, pnet.Capability(0)) + require.Greater(t, ts.mc.capability&pnet.ClientCompress, pnet.Capability(0)) + } else { + require.Equal(t, pnet.Capability(0), ts.mp.authenticator.capability&pnet.ClientCompress) + require.Equal(t, pnet.Capability(0), ts.mc.capability&pnet.ClientCompress) + } + // If both the client and the backend enables compression, proxy <-> backend enables compression. + if referCfg.clientConfig.capability&referCfg.backendConfig.capability&pnet.ClientCompress > 0 { + require.Greater(t, ts.mb.capability&pnet.ClientCompress, pnet.Capability(0)) + } else { + require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientCompress) + } + // If the client enables zstd compression, client <-> proxy enables zstd compression. + zstdCap := pnet.ClientCompress | pnet.ClientZstdCompressionAlgorithm + if referCfg.clientConfig.capability&zstdCap == zstdCap { + require.Greater(t, ts.mp.authenticator.capability&pnet.ClientZstdCompressionAlgorithm, pnet.Capability(0)) + require.Greater(t, ts.mc.capability&pnet.ClientZstdCompressionAlgorithm, pnet.Capability(0)) + require.Equal(t, referCfg.clientConfig.zstdLevel, ts.mp.authenticator.zstdLevel) + } else { + require.Equal(t, pnet.Capability(0), ts.mp.authenticator.capability&pnet.ClientZstdCompressionAlgorithm) + require.Equal(t, pnet.Capability(0), ts.mc.capability&pnet.ClientZstdCompressionAlgorithm) + } + // If both the client and the backend enables zstd compression, proxy <-> backend enables zstd compression. + if referCfg.clientConfig.capability&referCfg.backendConfig.capability&zstdCap == zstdCap { + require.Greater(t, ts.mb.capability&pnet.ClientZstdCompressionAlgorithm, pnet.Capability(0)) + require.Equal(t, referCfg.clientConfig.zstdLevel, ts.mb.zstdLevel) + } else { + require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientZstdCompressionAlgorithm) + } + } + for _, cfgs := range cfgOverriders { + referCfg := newTestConfig(cfgs...) + ts, clean := newTestSuite(t, tc, cfgs...) + ts.authenticateFirstTime(t, func(t *testing.T, ts *testSuite) { + checker(t, ts, referCfg) + }) + ts.authenticateSecondTime(t, func(t *testing.T, ts *testSuite) { + checker(t, ts, referCfg) + }) + clean() + } +} + +// After upgrading the backend, the backend capability may change. +func TestUpgradeBackendCap(t *testing.T) { + tc := newTCPConnSuite(t) + // Before upgrade, the client supports compression but the backend doesn't support. + ts, clean := newTestSuite(t, tc, func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.backendConfig.capability &= ^pnet.ClientCompress + }) + ts.authenticateFirstTime(t, func(t *testing.T, ts *testSuite) { + require.Greater(t, ts.mp.authenticator.capability&pnet.ClientCompress, pnet.Capability(0)) + require.Greater(t, ts.mc.capability&pnet.ClientCompress, pnet.Capability(0)) + require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientCompress) + }) + // After upgrade, the backend also supports compression. + ts.mb.backendConfig.capability |= pnet.ClientCompress + ts.authenticateSecondTime(t, func(t *testing.T, ts *testSuite) { + require.Greater(t, ts.mb.capability&pnet.ClientCompress, pnet.Capability(0)) + }) + clean() +} diff --git a/pkg/proxy/backend/backend_conn_mgr_test.go b/pkg/proxy/backend/backend_conn_mgr_test.go index b7ea68e4..6a76347d 100644 --- a/pkg/proxy/backend/backend_conn_mgr_test.go +++ b/pkg/proxy/backend/backend_conn_mgr_test.go @@ -527,8 +527,8 @@ func TestSpecialCmds(t *testing.T) { require.NoError(t, ts.redirectSucceed4Backend(packetIO)) require.Equal(t, "another_user", ts.mb.username) require.Equal(t, "session_db", ts.mb.db) - expectCap := pnet.Capability(ts.mp.handshakeHandler.GetCapability() &^ (pnet.ClientMultiStatements | pnet.ClientPluginAuthLenencClientData)) - gotCap := pnet.Capability(ts.mb.capability &^ pnet.ClientPluginAuthLenencClientData) + expectCap := ts.mp.handshakeHandler.GetCapability() & defaultTestClientCapability &^ (pnet.ClientMultiStatements | pnet.ClientPluginAuthLenencClientData) + gotCap := ts.mb.capability &^ pnet.ClientPluginAuthLenencClientData require.Equal(t, expectCap, gotCap, "expected=%s,got=%s", expectCap, gotCap) return nil }, @@ -793,18 +793,16 @@ func TestHandlerReturnError(t *testing.T) { } func TestOnTraffic(t *testing.T) { - i := 0 - inbytes, outbytes := []int{ - 0x99, - }, []int{ - 0xce, - } + var inBytes, outBytes uint64 ts := newBackendMgrTester(t, func(config *testConfig) { config.proxyConfig.bcConfig.CheckBackendInterval = 10 * time.Millisecond config.proxyConfig.handler.onTraffic = func(cc ConnContext) { - require.Equal(t, uint64(inbytes[i]), cc.ClientInBytes()) - require.Equal(t, uint64(outbytes[i]), cc.ClientOutBytes()) - i++ + require.Greater(t, cc.ClientInBytes(), uint64(0)) + require.GreaterOrEqual(t, cc.ClientInBytes(), inBytes) + inBytes = cc.ClientInBytes() + require.Greater(t, cc.ClientOutBytes(), uint64(0)) + require.GreaterOrEqual(t, cc.ClientOutBytes(), outBytes) + outBytes = cc.ClientOutBytes() } }) runners := []runner{ diff --git a/pkg/proxy/backend/common_test.go b/pkg/proxy/backend/common_test.go index bf5cfdd3..ead11711 100644 --- a/pkg/proxy/backend/common_test.go +++ b/pkg/proxy/backend/common_test.go @@ -84,6 +84,24 @@ func (tc *tcpConnSuite) newConn(t *testing.T, enableRoute bool) func() { } } +func (tc *tcpConnSuite) reconnectBackend(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + var wg waitgroup.WaitGroup + wg.Run(func() { + _ = tc.backendIO.Close() + conn, err := tc.backendListener.Accept() + require.NoError(t, err) + tc.backendIO = pnet.NewPacketIO(conn, lg) + }) + wg.Run(func() { + _ = tc.proxyBIO.Close() + backendConn, err := net.Dial("tcp", tc.backendListener.Addr().String()) + require.NoError(t, err) + tc.proxyBIO = pnet.NewPacketIO(backendConn, lg) + }) + wg.Wait() +} + func (tc *tcpConnSuite) run(clientRunner, backendRunner func(*pnet.PacketIO) error, proxyRunner func(*pnet.PacketIO, *pnet.PacketIO) error) (cerr, berr, perr error) { var wg waitgroup.WaitGroup if clientRunner != nil { diff --git a/pkg/proxy/backend/mock_backend_test.go b/pkg/proxy/backend/mock_backend_test.go index cc7d453a..303a87f1 100644 --- a/pkg/proxy/backend/mock_backend_test.go +++ b/pkg/proxy/backend/mock_backend_test.go @@ -47,10 +47,11 @@ type mockBackend struct { // Inputs that assigned by the test and will be sent to the client. *backendConfig // Outputs that received from the client and will be checked by the test. - username string - db string - attrs map[string]string - authData []byte + username string + db string + attrs map[string]string + authData []byte + zstdLevel int } func newMockBackend(cfg *backendConfig) *mockBackend { @@ -98,6 +99,7 @@ func (mb *mockBackend) authenticate(packetIO *pnet.PacketIO) error { mb.authData = resp.AuthData mb.attrs = resp.Attrs mb.capability = resp.Capability + mb.zstdLevel = resp.ZstdLevel // verify password return mb.verifyPassword(packetIO, resp) } @@ -125,6 +127,9 @@ func (mb *mockBackend) verifyPassword(packetIO *pnet.PacketIO, resp *pnet.Handsh if err := packetIO.WriteOKPacket(mb.status, pnet.OKHeader); err != nil { return err } + if err := setCompress(packetIO, mb.capability, mb.zstdLevel); err != nil { + return err + } } else { if err := packetIO.WriteErrPacket(mysql.NewDefaultError(mysql.ER_ACCESS_DENIED_ERROR)); err != nil { return err diff --git a/pkg/proxy/backend/mock_client_test.go b/pkg/proxy/backend/mock_client_test.go index de28a2c8..188f9da4 100644 --- a/pkg/proxy/backend/mock_client_test.go +++ b/pkg/proxy/backend/mock_client_test.go @@ -26,6 +26,7 @@ type clientConfig struct { capability pnet.Capability collation uint8 cmd pnet.Command + zstdLevel int // for both auth and cmd abnormalExit bool } @@ -82,6 +83,7 @@ func (mc *mockClient) authenticate(packetIO *pnet.PacketIO) error { AuthData: mc.authData, Capability: mc.capability, Collation: mc.collation, + ZstdLevel: mc.zstdLevel, } pkt = pnet.MakeHandshakeResponse(resp) if mc.capability&pnet.ClientSSL > 0 { diff --git a/pkg/proxy/backend/testsuite_test.go b/pkg/proxy/backend/testsuite_test.go index 13984e3b..f323e525 100644 --- a/pkg/proxy/backend/testsuite_test.go +++ b/pkg/proxy/backend/testsuite_test.go @@ -22,13 +22,13 @@ import ( // sent from the server and vice versa. const ( - defaultTestBackendCapability = pnet.ClientLongPassword | pnet.ClientFoundRows | pnet.ClientLongFlag | + defaultTestClientCapability = pnet.ClientLongPassword | pnet.ClientFoundRows | pnet.ClientLongFlag | pnet.ClientConnectWithDB | pnet.ClientNoSchema | pnet.ClientODBC | pnet.ClientLocalFiles | pnet.ClientIgnoreSpace | pnet.ClientProtocol41 | pnet.ClientInteractive | pnet.ClientSSL | pnet.ClientIgnoreSigpipe | pnet.ClientTransactions | pnet.ClientReserved | pnet.ClientSecureConnection | pnet.ClientMultiStatements | pnet.ClientMultiResults | pnet.ClientPluginAuth | pnet.ClientConnectAttrs | pnet.ClientPluginAuthLenencClientData | pnet.ClientDeprecateEOF - defaultTestClientCapability = defaultTestBackendCapability + defaultTestBackendCapability = defaultTestClientCapability | pnet.ClientCompress | pnet.ClientZstdCompressionAlgorithm ) var ( @@ -197,6 +197,7 @@ func (ts *testSuite) authenticateFirstTime(t *testing.T, c checker) { // This must be called after authenticateFirstTime. func (ts *testSuite) authenticateSecondTime(t *testing.T, c checker) { ts.mb.backendConfig.authSucceed = true + ts.tc.reconnectBackend(t) ts.runAndCheck(t, c, nil, ts.mb.authenticate, ts.mp.authenticateSecondTime) if c == nil { require.Equal(t, ts.mc.username, ts.mb.username) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 3dc7d57e..5a22054e 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -141,7 +141,7 @@ func (crw *compressedReadWriter) readFromConn() error { } } else { // If the data is compressed, the compressed length is the length of data after the compressed header and - // the uncompressed length is the length of data after uncompression. + // the uncompressed length is the length of data after decompression. data := make([]byte, compressedLength) if _, err = io.ReadFull(crw.packetReadWriter, data); err != nil { return err @@ -179,6 +179,9 @@ func (crw *compressedReadWriter) Write(data []byte) (n int, err error) { func (crw *compressedReadWriter) Flush() error { var err error data := crw.writeBuffer.Bytes() + if len(data) == 0 { + return nil + } crw.writeBuffer.Reset() // If the data is uncompressed, the uncompressed length is 0 and compressed length is the data length @@ -187,7 +190,7 @@ func (crw *compressedReadWriter) Flush() error { compressedLength := len(data) if len(data) >= minCompressSize { // If the data is compressed, the compressed length is the length of data after the compressed header and - // the uncompressed length is the length of data after uncompression. + // the uncompressed length is the length of data after decompression. uncompressedLength = len(data) if data, err = crw.compress(data); err != nil { return err @@ -213,6 +216,14 @@ func (crw *compressedReadWriter) Flush() error { return crw.packetReadWriter.Flush() } +// DirectWrite won't be used. +func (crw *compressedReadWriter) DirectWrite(data []byte) (n int, err error) { + if n, err = crw.Write(data); err != nil { + return + } + return n, crw.Flush() +} + // Peek won't be used. // Notice: the peeked data may be discarded if an error is returned. func (crw *compressedReadWriter) Peek(n int) (data []byte, err error) { diff --git a/pkg/proxy/net/compress_test.go b/pkg/proxy/net/compress_test.go new file mode 100644 index 00000000..15745cfd --- /dev/null +++ b/pkg/proxy/net/compress_test.go @@ -0,0 +1,252 @@ +// Copyright 2023 PingCAP, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "fmt" + "io" + "math/rand" + "net" + "testing" + + "github.com/pingcap/tiproxy/lib/util/logger" + "github.com/pingcap/tiproxy/pkg/testkit" + "github.com/stretchr/testify/require" +) + +// Test read/write with zlib compression. +func TestCompressZlib(t *testing.T) { + sizes := []int{minCompressSize - 1, 1024, maxCompressedSize, maxCompressedSize + 1, maxCompressedSize * 2} + lg, _ := logger.CreateLoggerForTest(t) + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + written := crw.OutBytes() + for _, size := range sizes { + fillAndWrite(t, crw, 'a', size) + require.NoError(t, crw.Flush()) + // Check compressed bytes. + outBytes := crw.OutBytes() + checkByteNum(t, outBytes-written, size) + written = outBytes + } + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + for _, size := range sizes { + readAndCheck(t, crw, 'a', size) + } + }, 1) +} + +// Test read/write with zstd compression. +func TestCompressZstd(t *testing.T) { + sizes := []int{minCompressSize - 1, 1024, maxCompressedSize, maxCompressedSize + 1, maxCompressedSize * 2} + levels := []int{1, 3, 9, 20} + lg, _ := logger.CreateLoggerForTest(t) + for _, level := range levels { + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZstd, level, lg) + written := crw.OutBytes() + for _, size := range sizes { + fillAndWrite(t, crw, 'a', size) + require.NoError(t, crw.Flush()) + // Check compressed bytes. + outBytes := crw.OutBytes() + checkByteNum(t, outBytes-written, size) + written = outBytes + } + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZstd, level, lg) + for _, size := range sizes { + readAndCheck(t, crw, 'a', size) + } + }, 1) + } +} + +// Test that multiple packets are merged into one compressed packet. +func TestCompressMergePkt(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + sizes := make([]int, 50) + for i := range sizes { + sizes[i] = int(rand.Int31n(maxCompressedSize / 2)) + } + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + written := 0 + for i, size := range sizes { + fillAndWrite(t, crw, 'a'+byte(i), size) + // Check that data is buffered until reaching maxCompressedSize. + written += size + require.Equal(t, written%maxCompressedSize, crw.writeBuffer.Len()) + } + require.NoError(t, crw.Flush()) + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + for i, size := range sizes { + readAndCheck(t, crw, 'a'+byte(i), size) + } + }, 1) +} + +// Test that DirectWrite(), Peek(), and Discard() work well. +func TestCompressPeekDiscard(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + size := 1000 + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + data := fillData('a', size) + _, err := crw.DirectWrite(data) + require.NoError(t, err) + + data = fillData('b', size) + _, err = crw.DirectWrite(data) + require.NoError(t, err) + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + peek, err := crw.Peek(10) + require.NoError(t, err) + checkData(t, peek, 'a') + readAndCheck(t, crw, 'a', size) + + _, err = crw.Discard(100) + require.NoError(t, err) + readAndCheck(t, crw, 'b', size-100) + }, 1) +} + +// Test that the uncompressed sequence is correct. +func TestCompressSequence(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + fillAndWrite(t, crw, 'a', 100) + fillAndWrite(t, crw, 'a', 100) + require.NoError(t, crw.Flush()) + require.Equal(t, uint8(2), crw.Sequence()) + // uncompressed sequence = compressed sequence + readAndCheck(t, crw, 'a', 100) + require.Equal(t, uint8(2), crw.Sequence()) + readAndCheck(t, crw, 'a', 100) + require.Equal(t, uint8(3), crw.Sequence()) + // uncompressed sequence = compressed sequence + fillAndWrite(t, crw, 'a', maxCompressedSize+1) + require.NoError(t, crw.Flush()) + require.Equal(t, uint8(3), crw.Sequence()) + // uncompressed sequence = compressed sequence + readAndCheck(t, crw, 'a', maxCompressedSize+1) + require.Equal(t, uint8(5), crw.Sequence()) + // flush empty buffer won't increase sequence + require.NoError(t, crw.Flush()) + require.NoError(t, crw.Flush()) + fillAndWrite(t, crw, 'a', 100) + require.NoError(t, crw.Flush()) + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + readAndCheck(t, crw, 'a', 100) + readAndCheck(t, crw, 'a', 100) + require.Equal(t, uint8(2), crw.Sequence()) + // uncompressed sequence = compressed sequence + fillAndWrite(t, crw, 'a', 100) + require.Equal(t, uint8(2), crw.Sequence()) + fillAndWrite(t, crw, 'a', 100) + require.Equal(t, uint8(3), crw.Sequence()) + require.NoError(t, crw.Flush()) + // uncompressed sequence = compressed sequence + readAndCheck(t, crw, 'a', maxCompressedSize+1) + require.Equal(t, uint8(3), crw.Sequence()) + // uncompressed sequence = compressed sequence + fillAndWrite(t, crw, 'a', maxCompressedSize+1) + require.NoError(t, crw.Flush()) + require.Equal(t, uint8(5), crw.Sequence()) + // flush empty buffer won't increase sequence + readAndCheck(t, crw, 'a', 100) + }, 1) +} + +// Test that the compressed header is correctly filled. +func TestCompressHeader(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + sizes := []int{minCompressSize - 1, maxCompressedSize, maxCompressedSize + 1} + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + for i, size := range sizes { + fillAndWrite(t, crw, 'a'+byte(i), size) + require.NoError(t, crw.Flush()) + } + }, + func(t *testing.T, c net.Conn) { + brw := newBasicReadWriter(c) + crw := newCompressedReadWriter(brw, CompressionZlib, 0, lg) + for i, size := range sizes { + header, err := brw.Peek(7) + require.NoError(t, err) + compressedLength := int(uint32(header[0]) | uint32(header[1])<<8 | uint32(header[2])<<16) + uncompressedLength := int(uint32(header[4]) | uint32(header[5])<<8 | uint32(header[6])<<16) + if size < minCompressSize { + require.Equal(t, size, compressedLength) + require.Equal(t, 0, uncompressedLength) + } else if size <= maxCompressedSize { + require.Greater(t, compressedLength, 0) + require.Less(t, compressedLength, size) + require.Equal(t, size, uncompressedLength) + } else { + require.Greater(t, compressedLength, 0) + require.Less(t, compressedLength, maxCompressedSize) + require.Equal(t, maxCompressedSize, uncompressedLength) + } + readAndCheck(t, crw, 'a'+byte(i), size) + } + }, 1) +} + +func fillAndWrite(t *testing.T, crw *compressedReadWriter, b byte, length int) { + data := fillData(b, length) + _, err := crw.Write(data) + require.NoError(t, err) + crw.SetSequence(crw.Sequence() + 1) +} + +func fillData(b byte, length int) []byte { + data := make([]byte, length) + for i := range data { + data[i] = b + } + return data +} + +func readAndCheck(t *testing.T, crw *compressedReadWriter, b byte, length int) { + data := make([]byte, length) + _, err := io.ReadFull(crw, data) + require.NoError(t, err) + checkData(t, data, b) + crw.SetSequence(crw.Sequence() + 1) +} + +func checkData(t *testing.T, data []byte, b byte) { + for i := range data { + if data[i] != b { + require.Fail(t, fmt.Sprintf("expected %c, but got %c", b, data[i])) + } + } +} + +func checkByteNum(t *testing.T, diff uint64, size int) { + if size < minCompressSize { + require.Equal(t, uint64(size+7), diff) + } else { + require.Greater(t, diff, uint64(0)) + require.Less(t, diff, uint64(size+7)) + } +} diff --git a/pkg/proxy/net/packetio.go b/pkg/proxy/net/packetio.go index ef7cd210..2db79bf3 100644 --- a/pkg/proxy/net/packetio.go +++ b/pkg/proxy/net/packetio.go @@ -50,17 +50,13 @@ const ( defaultReaderSize = 16 * 1024 ) -// The functions in bufferedIO are implemented by bufio.ReadWriter. -type bufferedIO interface { - Peek(n int) ([]byte, error) - Discard(n int) (int, error) - Flush() error -} - // packetReadWriter acts like a net.Conn with read and write buffer. type packetReadWriter interface { net.Conn - bufferedIO + // Peek / Discard / Flush are implemented by bufio.ReadWriter. + Peek(n int) ([]byte, error) + Discard(n int) (int, error) + Flush() error DirectWrite(p []byte) (int, error) Proxy() *proxyprotocol.Proxy TLSConnectionState() tls.ConnectionState @@ -82,7 +78,7 @@ type basicReadWriter struct { sequence uint8 } -func newBasicBufConn(conn net.Conn) *basicReadWriter { +func newBasicReadWriter(conn net.Conn) *basicReadWriter { return &basicReadWriter{ Conn: conn, ReadWriter: bufio.NewReadWriter(bufio.NewReaderSize(conn, defaultReaderSize), bufio.NewWriterSize(conn, defaultWriterSize)), @@ -166,7 +162,7 @@ func NewPacketIO(conn net.Conn, lg *zap.Logger, opts ...PacketIOption) *PacketIO p := &PacketIO{ rawConn: conn, logger: lg, - readWriter: newBasicBufConn(conn), + readWriter: newBasicReadWriter(conn), } p.ApplyOpts(opts...) return p diff --git a/pkg/proxy/net/packetio_test.go b/pkg/proxy/net/packetio_test.go index 14e0bdfc..e700a681 100644 --- a/pkg/proxy/net/packetio_test.go +++ b/pkg/proxy/net/packetio_test.go @@ -287,3 +287,78 @@ func TestPredefinedPacket(t *testing.T) { 1, ) } + +// Test the combination of proxy, tls and compress. +func TestProxyTLSCompress(t *testing.T) { + stls, ctls, err := security.CreateTLSConfigForTest() + require.NoError(t, err) + addr, p := mockProxy(t) + ch := make(chan []byte) + write := func(p *PacketIO, data []byte) { + outBytes := p.OutBytes() + require.NoError(t, p.WritePacket(data, true)) + ch <- data + require.Greater(t, p.OutBytes(), outBytes) + require.True(t, p.IsPeerActive()) + require.NotEmpty(t, p.RemoteAddr().String()) + } + read := func(p *PacketIO) { + inBytes := p.InBytes() + data := <-ch + pkt, err := p.ReadPacket() + require.NoError(t, err) + require.Equal(t, data, pkt) + require.Greater(t, p.InBytes(), inBytes) + require.True(t, p.IsPeerActive()) + require.NotEmpty(t, p.RemoteAddr().String()) + } + for _, enableCompress := range []bool{true, false} { + for _, enableTLS := range []bool{true, false} { + for _, enableProxy := range []bool{true, false} { + testTCPConn(t, func(t *testing.T, cli *PacketIO) { + if enableProxy { + cli.EnableProxyClient(p) + } + write(cli, []byte("test1")) + if enableTLS { + require.NoError(t, cli.ClientTLSHandshake(ctls)) + require.True(t, cli.TLSConnectionState().HandshakeComplete) + } + read(cli) + if enableCompress { + cli.ResetSequence() + require.NoError(t, cli.SetCompressionAlgorithm(CompressionZlib, 0)) + } + write(cli, []byte("test3")) + read(cli) + // make sure the peer won't quit in advance + ch <- nil + }, func(t *testing.T, srv *PacketIO) { + if enableProxy { + srv.EnableProxyServer() + } + read(srv) + if enableProxy { + require.Equal(t, addr.String(), srv.RemoteAddr().String()) + require.Equal(t, addr.String(), srv.Proxy().SrcAddress.String()) + } + if enableTLS { + state, err := srv.ServerTLSHandshake(stls) + require.NoError(t, err) + require.True(t, state.HandshakeComplete) + require.True(t, srv.TLSConnectionState().HandshakeComplete) + } + write(srv, []byte("test2")) + if enableCompress { + srv.ResetSequence() + require.NoError(t, srv.SetCompressionAlgorithm(CompressionZlib, 0)) + } + read(srv) + write(srv, []byte("test4")) + // make sure the peer won't quit in advance + <-ch + }, 1) + } + } + } +} diff --git a/pkg/proxy/net/proxy.go b/pkg/proxy/net/proxy.go index b746e285..a3c0b293 100644 --- a/pkg/proxy/net/proxy.go +++ b/pkg/proxy/net/proxy.go @@ -95,7 +95,7 @@ func (prw *proxyReadWriter) Write(p []byte) (n int, err error) { } prw.proxyInited.Store(true) } - return prw.Write(p) + return prw.packetReadWriter.Write(p) } func (prw *proxyReadWriter) parseProxyV2() (*proxyprotocol.Proxy, error) { @@ -125,7 +125,7 @@ func (prw *proxyReadWriter) RemoteAddr() net.Addr { if prw.addr != nil { return prw.addr } - return prw.RemoteAddr() + return prw.packetReadWriter.RemoteAddr() } func (prw *proxyReadWriter) Proxy() *proxyprotocol.Proxy { diff --git a/pkg/proxy/net/proxy_test.go b/pkg/proxy/net/proxy_test.go index 7fc61ae5..da775025 100644 --- a/pkg/proxy/net/proxy_test.go +++ b/pkg/proxy/net/proxy_test.go @@ -10,31 +10,14 @@ import ( "testing" "github.com/pingcap/tiproxy/pkg/proxy/proxyprotocol" + "github.com/pingcap/tiproxy/pkg/testkit" "github.com/stretchr/testify/require" ) func TestProxyParse(t *testing.T) { - tcpaddr, err := net.ResolveTCPAddr("tcp", "192.168.1.1:34") - require.NoError(t, err) - + tcpaddr, p := mockProxy(t) testPipeConn(t, func(t *testing.T, cli *PacketIO) { - p := &proxyprotocol.Proxy{ - Version: proxyprotocol.ProxyVersion2, - Command: proxyprotocol.ProxyCommandLocal, - SrcAddress: tcpaddr, - DstAddress: tcpaddr, - TLV: []proxyprotocol.ProxyTlv{ - { - Typ: proxyprotocol.ProxyTlvALPN, - Content: nil, - }, - { - Typ: proxyprotocol.ProxyTlvUniqueID, - Content: []byte("test"), - }, - }, - } b, err := p.ToBytes() require.NoError(t, err) _, err = io.Copy(cli.readWriter, bytes.NewReader(b)) @@ -52,3 +35,47 @@ func TestProxyParse(t *testing.T) { 1, ) } + +func TestProxyReadWrite(t *testing.T) { + addr, p := mockProxy(t) + message := []byte("hello world") + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + prw := newProxyClient(newBasicReadWriter(c), p) + n, err := prw.Write(message) + require.NoError(t, err) + require.Equal(t, len(message), n) + require.NoError(t, prw.Flush()) + }, + func(t *testing.T, c net.Conn) { + prw := newProxyServer(newBasicReadWriter(c)) + data := make([]byte, len(message)) + n, err := prw.Read(data) + require.NoError(t, err) + require.Equal(t, len(message), n) + require.Equal(t, p.SrcAddress, prw.Proxy().SrcAddress) + require.Equal(t, addr.String(), prw.RemoteAddr().String()) + }, 1) +} + +func mockProxy(t *testing.T) (*net.TCPAddr, *proxyprotocol.Proxy) { + tcpaddr, err := net.ResolveTCPAddr("tcp", "192.168.1.1:34") + require.NoError(t, err) + p := &proxyprotocol.Proxy{ + Version: proxyprotocol.ProxyVersion2, + Command: proxyprotocol.ProxyCommandLocal, + SrcAddress: tcpaddr, + DstAddress: tcpaddr, + TLV: []proxyprotocol.ProxyTlv{ + { + Typ: proxyprotocol.ProxyTlvALPN, + Content: nil, + }, + { + Typ: proxyprotocol.ProxyTlvUniqueID, + Content: []byte("test"), + }, + }, + } + return tcpaddr, p +} diff --git a/pkg/proxy/net/tls_test.go b/pkg/proxy/net/tls_test.go new file mode 100644 index 00000000..91f4d192 --- /dev/null +++ b/pkg/proxy/net/tls_test.go @@ -0,0 +1,84 @@ +// Copyright 2023 PingCAP, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package net + +import ( + "crypto/tls" + "io" + "net" + "testing" + + "github.com/pingcap/tiproxy/lib/util/security" + "github.com/pingcap/tiproxy/pkg/testkit" + "github.com/stretchr/testify/require" +) + +func TestTLSReadWrite(t *testing.T) { + stls, ctls, err := security.CreateTLSConfigForTest() + require.NoError(t, err) + message := []byte("hello world") + ch := make(chan []byte) + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + brw := newBasicReadWriter(c) + conn := &tlsInternalConn{brw} + tlsConn := tls.Client(conn, ctls) + require.NoError(t, tlsConn.Handshake()) + trw := newTLSReadWriter(brw, tlsConn) + // check tls connection state + require.True(t, trw.TLSConnectionState().HandshakeComplete) + // check out bytes + outBytes := trw.OutBytes() + // Wait before writing, otherwise the message is buffered during TLS in the other goroutine. + ch <- message + n, err := trw.Write(message) + require.NoError(t, err) + require.NoError(t, trw.Flush()) + require.Equal(t, len(message), n) + require.Greater(t, trw.OutBytes(), outBytes+uint64(len(message))) + // check direct write + for i := 0; i < 2; i++ { + n, err = trw.DirectWrite(message) + require.NoError(t, err) + require.Equal(t, len(message), n) + } + }, + func(t *testing.T, c net.Conn) { + brw := newBasicReadWriter(c) + conn := &tlsInternalConn{brw} + tlsConn := tls.Server(conn, stls) + require.NoError(t, tlsConn.Handshake()) + trw := newTLSReadWriter(brw, tlsConn) + // check tls connection state + require.True(t, trw.TLSConnectionState().HandshakeComplete) + // check in bytes + inBytes := trw.InBytes() + message := <-ch + data := make([]byte, len(message)) + n, err := io.ReadFull(trw, data) + require.NoError(t, err) + require.Equal(t, len(message), n) + require.Equal(t, message, data) + require.Greater(t, trw.InBytes(), inBytes+uint64(len(message))) + // check peek + peek, err := trw.Peek(1) + require.NoError(t, err) + require.Len(t, peek, 1) + require.Equal(t, message[0], peek[0]) + data = make([]byte, len(message)) + n, err = io.ReadFull(trw, data) + require.NoError(t, err) + require.Equal(t, len(data), n) + require.Equal(t, message, data) + // check discard + n, err = trw.Discard(1) + require.NoError(t, err) + require.Equal(t, 1, n) + data = make([]byte, len(message)-1) + n, err = io.ReadFull(trw, data) + require.NoError(t, err) + require.Equal(t, len(data), n) + require.Equal(t, message[1:], data) + }, 1) +} From 8658e8c8a0b82a2dc14cd7c071ca69802c59f5cf Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 09:26:10 +0800 Subject: [PATCH 05/15] update TestUpgradeBackendCap --- pkg/proxy/backend/authenticator_test.go | 68 ++++++++++++++++++------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/pkg/proxy/backend/authenticator_test.go b/pkg/proxy/backend/authenticator_test.go index 8de40e0f..47d3e99d 100644 --- a/pkg/proxy/backend/authenticator_test.go +++ b/pkg/proxy/backend/authenticator_test.go @@ -446,8 +446,6 @@ func TestCompressProtocol(t *testing.T) { }, } - tc := newTCPConnSuite(t) - cfgOverriders := getCfgCombinations(cfgs) checker := func(t *testing.T, ts *testSuite, referCfg *testConfig) { // If the client enables compression, client <-> proxy enables compression. if referCfg.clientConfig.capability&pnet.ClientCompress > 0 { @@ -481,6 +479,9 @@ func TestCompressProtocol(t *testing.T) { require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientZstdCompressionAlgorithm) } } + + tc := newTCPConnSuite(t) + cfgOverriders := getCfgCombinations(cfgs) for _, cfgs := range cfgOverriders { referCfg := newTestConfig(cfgs...) ts, clean := newTestSuite(t, tc, cfgs...) @@ -496,21 +497,52 @@ func TestCompressProtocol(t *testing.T) { // After upgrading the backend, the backend capability may change. func TestUpgradeBackendCap(t *testing.T) { + cfgs := [][]cfgOverrider{ + { + func(cfg *testConfig) { + cfg.clientConfig.capability &= ^pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability |= pnet.ClientZstdCompressionAlgorithm + cfg.clientConfig.zstdLevel = 3 + }, + func(cfg *testConfig) { + cfg.clientConfig.capability |= pnet.ClientCompress + cfg.clientConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + }, + { + func(cfg *testConfig) { + cfg.backendConfig.capability &= ^pnet.ClientCompress + cfg.backendConfig.capability &= ^pnet.ClientZstdCompressionAlgorithm + }, + }, + } + tc := newTCPConnSuite(t) - // Before upgrade, the client supports compression but the backend doesn't support. - ts, clean := newTestSuite(t, tc, func(cfg *testConfig) { - cfg.clientConfig.capability |= pnet.ClientCompress - cfg.backendConfig.capability &= ^pnet.ClientCompress - }) - ts.authenticateFirstTime(t, func(t *testing.T, ts *testSuite) { - require.Greater(t, ts.mp.authenticator.capability&pnet.ClientCompress, pnet.Capability(0)) - require.Greater(t, ts.mc.capability&pnet.ClientCompress, pnet.Capability(0)) - require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientCompress) - }) - // After upgrade, the backend also supports compression. - ts.mb.backendConfig.capability |= pnet.ClientCompress - ts.authenticateSecondTime(t, func(t *testing.T, ts *testSuite) { - require.Greater(t, ts.mb.capability&pnet.ClientCompress, pnet.Capability(0)) - }) - clean() + cfgOverriders := getCfgCombinations(cfgs) + for _, cfgs := range cfgOverriders { + referCfg := newTestConfig(cfgs...) + ts, clean := newTestSuite(t, tc, cfgs...) + // Before upgrade, the backend doesn't support compression. + ts.authenticateFirstTime(t, func(t *testing.T, ts *testSuite) { + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientCompress, ts.mp.authenticator.capability&pnet.ClientCompress) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientCompress, ts.mc.capability&pnet.ClientCompress) + require.Equal(t, pnet.Capability(0), ts.mb.capability&pnet.ClientCompress) + }) + // After upgrade, the backend also supports compression. + ts.mb.backendConfig.capability |= pnet.ClientCompress + ts.mb.backendConfig.capability |= pnet.ClientZstdCompressionAlgorithm + ts.authenticateSecondTime(t, func(t *testing.T, ts *testSuite) { + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientCompress, ts.mc.capability&pnet.ClientCompress) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientCompress, ts.mp.authenticator.capability&pnet.ClientCompress) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientCompress, ts.mb.capability&pnet.ClientCompress) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientZstdCompressionAlgorithm, ts.mc.capability&pnet.ClientZstdCompressionAlgorithm) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientZstdCompressionAlgorithm, ts.mp.authenticator.capability&pnet.ClientZstdCompressionAlgorithm) + require.Equal(t, referCfg.clientConfig.capability&pnet.ClientZstdCompressionAlgorithm, ts.mb.capability&pnet.ClientZstdCompressionAlgorithm) + }) + clean() + } } From 7bfdb669c06387e5f2181c5dc15fd428fbe9d3a0 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 10:45:01 +0800 Subject: [PATCH 06/15] enable cgo --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 45966f17..905ffd2d 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ cmd: $(EXECUTABLE_TARGETS) cmd_%: OUTPUT=$(patsubst cmd_%,./bin/%,$@) cmd_%: SOURCE=$(patsubst cmd_%,./cmd/%,$@) cmd_%: - go build $(BUILDFLAGS) -o $(OUTPUT) $(SOURCE) + CGO_ENABLED=1 go build $(BUILDFLAGS) -o $(OUTPUT) $(SOURCE) golangci-lint: GOBIN=$(GOBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.1 @@ -68,7 +68,7 @@ tidy: build: cd lib && go build ./... - go build ./... + CGO_ENABLED=1 go build ./... metrics: go install github.com/google/go-jsonnet/cmd/jsonnet@latest From 81ebd690ca06c25eaba24df4426d496ed842fa94 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 12:21:27 +0800 Subject: [PATCH 07/15] try update go version to fix cgo problem --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a26a5410..0a36236f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pingcap/tiproxy -go 1.19 +go 1.21 require ( github.com/BurntSushi/toml v1.2.1 From 3a11ec0aa364d8aa63cf5b8b3e2a9ccbedc883c7 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 12:26:49 +0800 Subject: [PATCH 08/15] fix go.sum --- go.sum | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/go.sum b/go.sum index 877a3507..a730911a 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -21,12 +22,14 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -37,12 +40,17 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.21.0 h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14= +cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 h1:KQgdWmEOmaJKxaUUZwHAYh12t+b+ZJf8q3friycK1kA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0/go.mod h1:ZPW/Z0kLCTdDZaDbYTetxc9Cxl/2lNqxYHYNOF2bti0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 h1:VBvHGLJbaY0+c66NZHdS9cgjHVYSH6DDa0XJMyrblsI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0/go.mod h1:GJzjM4SR9T0KyX5gKCVyz1ytD8FeWeUPCwtFCt1AyfE= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 h1:BUYIbDf/mMZ8945v3QkG3OuqGVyS4Iek0AOLwdRAYoc= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 h1:62Ew5xXg5UCGIXDOM7+y4IL5/6mQJq1nenhBCJAeGX8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0/go.mod h1:eHWhQKXc1Gv1DvWH//UzgWjWFEo0Pp4pH2vBzjBw8Fc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -52,13 +60,17 @@ github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mo github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -66,16 +78,19 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 h1:Q/yk4z/cHUVZfgTqtD09qeYBxHwshQAjVRX73qs8UH0= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= +github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.44.48 h1:jLDC9RsNoYMLFlKpB8LdqUnoDdC2yvkS4QbuyPQJ8+M= +github.com/aws/aws-sdk-go v1.44.48/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= @@ -89,7 +104,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo= +github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o= github.com/carlmjohnson/flagext v0.21.0 h1:/c4uK3ie786Z7caXLcIMvePNSSiH3bQVGDvmGLMme60= +github.com/carlmjohnson/flagext v0.21.0/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -102,6 +119,7 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= +github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -123,15 +141,18 @@ github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 h1:Igd6YmtOZ77EgLAIaE9+mHl7+sAKaZ5m4iMI0Dz/J2A= +github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= +github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U= github.com/coocood/freecache v1.2.1/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= +github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -162,18 +183,22 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -186,6 +211,7 @@ github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHj github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -323,6 +349,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= +github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -330,6 +357,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -373,6 +401,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -384,9 +413,12 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jedib0t/go-pretty/v6 v6.2.2 h1:o3McN0rQ4X+IU+HduppSp9TwRdGLRW2rhJXy9CJaCRw= +github.com/jedib0t/go-pretty/v6 v6.2.2/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df h1:Zrb0IbuLOGHL7nrO2WrcuNWgDTlzFv3zY69QMx4ggQE= +github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= @@ -417,8 +449,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -435,11 +469,17 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= +github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= +github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= +github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= github.com/lestrrat-go/jwx/v2 v2.0.6 h1:RlyYNLV892Ed7+FTfj1ROoF6x7WxL965PGTHso/60G0= +github.com/lestrrat-go/jwx/v2 v2.0.6/go.mod h1:aVrGuwEr3cp2Prw6TtQvr8sQxe+84gruID5C9TxT64Q= github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= @@ -448,6 +488,7 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -456,6 +497,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -487,6 +529,7 @@ github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= +github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= @@ -509,6 +552,7 @@ github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZO github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pingcap/badger v1.5.1-0.20221229114011-ddffaa0fff7a h1:QB16qn8wx5X4SRn3/5axrjPMNS3WRt87+5Bfrnmt6IA= +github.com/pingcap/badger v1.5.1-0.20221229114011-ddffaa0fff7a/go.mod h1:p8QnkZnmyV8L/M/jzYb8rT7kv3bz9m7bn1Ju94wDifs= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= @@ -523,7 +567,9 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 h1:kJolJWbyadVeL8RKBlqmXQR7FRKPsIeU85TUYyhbhiQ= github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZLmhahmvHm7n9DUxGRQT00208= +github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= +github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20221213093948-9ccc6beaf0aa h1:v0Z0nC0knwWHn3e9br8EMNfLBB14QDULn142UGjiTMQ= github.com/pingcap/kvproto v0.0.0-20221213093948-9ccc6beaf0aa/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= @@ -535,6 +581,7 @@ github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYh github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= +github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc h1:DBiqr5/UMuX8G254xMoA3vXYBeIjt+IOYEiIdrF5/Kk= github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc/go.mod h1:oJkN5ywDbwqxha7HuK+kmHQqFJwJs33f6gT9Gl289Vc= github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc h1:9DwkflhVYQUhGYBGjsqzR5zS5cxQmkNqfAjiSRRZ6fw= @@ -542,6 +589,7 @@ github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc/go.mod h1:wjvp github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 h1:DbmCfCbcavo0JG+gSp0ySvv1ub/c/j3hsnYzyYPzONo= github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -585,6 +633,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= +github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -604,8 +653,10 @@ github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4 github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= +github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= @@ -653,6 +704,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= +github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221221095600-1a473d1f9b4b h1:4RNtqw1/tW67qP9fFgfQpTVd7DrfkaAWu4vsC18QmBo= github.com/tiancaiamao/gp v0.0.0-20221221095600-1a473d1f9b4b/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tidwall/btree v1.5.2 h1:5eA83Gfki799V3d3bJo9sWk+yL2LRoTEah3O/SA6/8w= @@ -686,7 +738,9 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vbauerster/mpb/v7 v7.5.3 h1:BkGfmb6nMrrBQDFECR/Q7RkKCw7ylMetCb4079CGs4w= +github.com/vbauerster/mpb/v7 v7.5.3/go.mod h1:i+h4QY6lmLvBNK2ah1fSreiw3ajskRlBp9AhY/PnuOE= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy/gId+IEMrFHLuQ5R/WV0KNxWLler8X2OY= +github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -694,6 +748,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs= +github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= @@ -724,12 +779,14 @@ go.etcd.io/etcd/raft/v3 v3.5.6/go.mod h1:wL8kkRGx1Hp8FmZUuHfL3K2/OaGIDaXGr1N7i2G go.etcd.io/etcd/server/v3 v3.5.6 h1:RXuwaB8AMiV62TqcqIt4O4bG8NWjsxOkDJVT3MZI5Ds= go.etcd.io/etcd/server/v3 v3.5.6/go.mod h1:6/Gfe8XTGXQJgLYQ65oGKMfPivb2EASLUSMSWN9Sroo= go.etcd.io/etcd/tests/v3 v3.5.2 h1:uk7/uMGVebpBDl+roivowHt6gJ5Fnqwik3syDkoSKdo= +go.etcd.io/etcd/tests/v3 v3.5.2/go.mod h1:Jdzbei4uFi9C3xDBfCwckRXjlX0UPooiP4g/zXgBMgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= @@ -767,6 +824,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -887,6 +945,7 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -968,6 +1027,7 @@ golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1039,11 +1099,13 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1061,6 +1123,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1068,6 +1131,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1189,5 +1253,8 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= +sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= From 8486d96c17ee8801aee3ec7233960dbe6291b391 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 19:01:00 +0800 Subject: [PATCH 09/15] replace DataDog with klauspost for zstd --- go.mod | 4 +-- go.sum | 73 ++------------------------------------- pkg/proxy/net/compress.go | 19 ++++++---- 3 files changed, 18 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 0a36236f..ab8f3965 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/pingcap/tiproxy -go 1.21 +go 1.19 require ( github.com/BurntSushi/toml v1.2.1 - github.com/DataDog/zstd v1.5.5 github.com/bahlo/generic-list-go v0.2.0 github.com/cenkalti/backoff/v4 v4.2.1 github.com/fsnotify/fsnotify v1.6.0 @@ -12,6 +11,7 @@ require ( github.com/gin-gonic/gin v1.8.1 github.com/go-mysql-org/go-mysql v1.6.0 github.com/go-sql-driver/mysql v1.7.0 + github.com/klauspost/compress v1.16.6 github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc github.com/pingcap/tiproxy/lib v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index a730911a..845c5ffa 100644 --- a/go.sum +++ b/go.sum @@ -14,7 +14,6 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= -cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -22,14 +21,12 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= -cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -40,37 +37,27 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.21.0 h1:HwnT2u2D309SFDHQII6m18HlrCi3jAXhUMTLOWXYH14= -cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 h1:KQgdWmEOmaJKxaUUZwHAYh12t+b+ZJf8q3friycK1kA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0/go.mod h1:ZPW/Z0kLCTdDZaDbYTetxc9Cxl/2lNqxYHYNOF2bti0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 h1:VBvHGLJbaY0+c66NZHdS9cgjHVYSH6DDa0XJMyrblsI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0/go.mod h1:GJzjM4SR9T0KyX5gKCVyz1ytD8FeWeUPCwtFCt1AyfE= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 h1:BUYIbDf/mMZ8945v3QkG3OuqGVyS4Iek0AOLwdRAYoc= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 h1:62Ew5xXg5UCGIXDOM7+y4IL5/6mQJq1nenhBCJAeGX8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0/go.mod h1:eHWhQKXc1Gv1DvWH//UzgWjWFEo0Pp4pH2vBzjBw8Fc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= -github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= -github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= -github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -78,19 +65,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 h1:Q/yk4z/cHUVZfgTqtD09qeYBxHwshQAjVRX73qs8UH0= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= -github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.44.48 h1:jLDC9RsNoYMLFlKpB8LdqUnoDdC2yvkS4QbuyPQJ8+M= -github.com/aws/aws-sdk-go v1.44.48/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= @@ -104,9 +88,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blacktear23/go-proxyprotocol v1.0.2 h1:zR7PZeoU0wAkElcIXenFiy3R56WB6A+UEVi4c6RH8wo= -github.com/blacktear23/go-proxyprotocol v1.0.2/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o= github.com/carlmjohnson/flagext v0.21.0 h1:/c4uK3ie786Z7caXLcIMvePNSSiH3bQVGDvmGLMme60= -github.com/carlmjohnson/flagext v0.21.0/go.mod h1:Eenv0epIUAr4NuedNmkzI8WmBmjIxZC239XcKxYS2ac= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -119,7 +101,6 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA= -github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -141,18 +122,15 @@ github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 h1:Igd6YmtOZ77EgLAIaE9+mHl7+sAKaZ5m4iMI0Dz/J2A= -github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= -github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U= github.com/coocood/freecache v1.2.1/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= -github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -183,22 +161,18 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -211,7 +185,6 @@ github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHj github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -349,7 +322,6 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= -github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -357,7 +329,6 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -401,7 +372,6 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -413,12 +383,9 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jedib0t/go-pretty/v6 v6.2.2 h1:o3McN0rQ4X+IU+HduppSp9TwRdGLRW2rhJXy9CJaCRw= -github.com/jedib0t/go-pretty/v6 v6.2.2/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df h1:Zrb0IbuLOGHL7nrO2WrcuNWgDTlzFv3zY69QMx4ggQE= -github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= @@ -448,11 +415,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= -github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= -github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -469,17 +435,11 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= -github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= -github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= -github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= -github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= github.com/lestrrat-go/jwx/v2 v2.0.6 h1:RlyYNLV892Ed7+FTfj1ROoF6x7WxL965PGTHso/60G0= -github.com/lestrrat-go/jwx/v2 v2.0.6/go.mod h1:aVrGuwEr3cp2Prw6TtQvr8sQxe+84gruID5C9TxT64Q= github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= @@ -488,7 +448,6 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -497,7 +456,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -529,7 +487,6 @@ github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= -github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= @@ -552,7 +509,6 @@ github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZO github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pingcap/badger v1.5.1-0.20221229114011-ddffaa0fff7a h1:QB16qn8wx5X4SRn3/5axrjPMNS3WRt87+5Bfrnmt6IA= -github.com/pingcap/badger v1.5.1-0.20221229114011-ddffaa0fff7a/go.mod h1:p8QnkZnmyV8L/M/jzYb8rT7kv3bz9m7bn1Ju94wDifs= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= @@ -567,9 +523,7 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 h1:kJolJWbyadVeL8RKBlqmXQR7FRKPsIeU85TUYyhbhiQ= github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZLmhahmvHm7n9DUxGRQT00208= -github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= -github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/kvproto v0.0.0-20221213093948-9ccc6beaf0aa h1:v0Z0nC0knwWHn3e9br8EMNfLBB14QDULn142UGjiTMQ= github.com/pingcap/kvproto v0.0.0-20221213093948-9ccc6beaf0aa/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= @@ -581,7 +535,6 @@ github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c h1:crhkw6DD+07Bg1wYh github.com/pingcap/log v1.1.1-0.20221116035753-734d527bc87c/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 h1:HYbcxtnkN3s5tqrZ/z3eJS4j3Db8wMphEm1q10lY/TM= -github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4/go.mod h1:sDCsM39cGiv2vwunZkaFA917vVkqDTGSPbbV7z4Oops= github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc h1:DBiqr5/UMuX8G254xMoA3vXYBeIjt+IOYEiIdrF5/Kk= github.com/pingcap/tidb v1.1.0-beta.0.20230103132820-3ccff46aa3bc/go.mod h1:oJkN5ywDbwqxha7HuK+kmHQqFJwJs33f6gT9Gl289Vc= github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc h1:9DwkflhVYQUhGYBGjsqzR5zS5cxQmkNqfAjiSRRZ6fw= @@ -589,7 +542,6 @@ github.com/pingcap/tidb/parser v0.0.0-20230103132820-3ccff46aa3bc/go.mod h1:wjvp github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813 h1:DbmCfCbcavo0JG+gSp0ySvv1ub/c/j3hsnYzyYPzONo= github.com/pingcap/tipb v0.0.0-20221123081521-2fb828910813/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= -github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -633,7 +585,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= -github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -653,10 +604,8 @@ github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4 github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= @@ -704,7 +653,6 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJfDRtkanvQPiooDH8HvJ2FBh+iKT/OmiQQ= -github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221221095600-1a473d1f9b4b h1:4RNtqw1/tW67qP9fFgfQpTVd7DrfkaAWu4vsC18QmBo= github.com/tiancaiamao/gp v0.0.0-20221221095600-1a473d1f9b4b/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tidwall/btree v1.5.2 h1:5eA83Gfki799V3d3bJo9sWk+yL2LRoTEah3O/SA6/8w= @@ -738,9 +686,7 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vbauerster/mpb/v7 v7.5.3 h1:BkGfmb6nMrrBQDFECR/Q7RkKCw7ylMetCb4079CGs4w= -github.com/vbauerster/mpb/v7 v7.5.3/go.mod h1:i+h4QY6lmLvBNK2ah1fSreiw3ajskRlBp9AhY/PnuOE= github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy/gId+IEMrFHLuQ5R/WV0KNxWLler8X2OY= -github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -748,7 +694,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457 h1:tBbuFCtyJNKT+BFAv6qjvTFpVdy97IYNaBwGUXifIUs= -github.com/xitongsys/parquet-go v1.5.5-0.20201110004701-b09c49d6d457/go.mod h1:pheqtXeHQFzxJk45lRQ0UIGIivKnLXvialZSFWs81A8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= @@ -779,14 +724,12 @@ go.etcd.io/etcd/raft/v3 v3.5.6/go.mod h1:wL8kkRGx1Hp8FmZUuHfL3K2/OaGIDaXGr1N7i2G go.etcd.io/etcd/server/v3 v3.5.6 h1:RXuwaB8AMiV62TqcqIt4O4bG8NWjsxOkDJVT3MZI5Ds= go.etcd.io/etcd/server/v3 v3.5.6/go.mod h1:6/Gfe8XTGXQJgLYQ65oGKMfPivb2EASLUSMSWN9Sroo= go.etcd.io/etcd/tests/v3 v3.5.2 h1:uk7/uMGVebpBDl+roivowHt6gJ5Fnqwik3syDkoSKdo= -go.etcd.io/etcd/tests/v3 v3.5.2/go.mod h1:Jdzbei4uFi9C3xDBfCwckRXjlX0UPooiP4g/zXgBMgQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= @@ -824,7 +767,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -945,7 +887,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1027,7 +968,6 @@ golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1099,13 +1039,11 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1123,7 +1061,6 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1131,7 +1068,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1253,8 +1189,5 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= -sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= -stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 5a22054e..c8e4fd07 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -8,7 +8,7 @@ import ( "compress/zlib" "io" - "github.com/DataDog/zstd" + "github.com/klauspost/compress/zstd" "github.com/pingcap/tiproxy/lib/util/errors" "go.uber.org/zap" ) @@ -266,17 +266,20 @@ func (crw *compressedReadWriter) Discard(n int) (d int, err error) { return } +// DataDog/zstd is much faster but it's not good at cross-platform. +// https://github.com/go-mysql-org/go-mysql/issues/799 func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { var err error var compressedPacket bytes.Buffer var compressWriter io.WriteCloser switch crw.algorithm { case CompressionZlib: - if compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.DefaultCompression); err != nil { - return nil, errors.WithStack(err) - } + compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.DefaultCompression) case CompressionZstd: - compressWriter = zstd.NewWriterLevel(&compressedPacket, crw.zstdLevel) + compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(zstd.EncoderLevel(crw.zstdLevel))) + } + if err != nil { + return nil, errors.WithStack(err) } if _, err = compressWriter.Write(data); err != nil { return nil, errors.WithStack(err) @@ -296,7 +299,11 @@ func (crw *compressedReadWriter) uncompress(data []byte, uncompressedLength int) return nil, errors.WithStack(err) } case CompressionZstd: - compressedReader = zstd.NewReader(bytes.NewReader(data)) + var decoder *zstd.Decoder + if decoder, err = zstd.NewReader(bytes.NewReader(data)); err != nil { + return nil, errors.WithStack(err) + } + compressedReader = decoder.IOReadCloser() } uncompressed := make([]byte, uncompressedLength) if _, err = io.ReadFull(compressedReader, uncompressed); err != nil { From ad5db9d24d71902bcf166161fe7bab08a4f1c710 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 19:09:01 +0800 Subject: [PATCH 10/15] revert CGO --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 905ffd2d..45966f17 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ cmd: $(EXECUTABLE_TARGETS) cmd_%: OUTPUT=$(patsubst cmd_%,./bin/%,$@) cmd_%: SOURCE=$(patsubst cmd_%,./cmd/%,$@) cmd_%: - CGO_ENABLED=1 go build $(BUILDFLAGS) -o $(OUTPUT) $(SOURCE) + go build $(BUILDFLAGS) -o $(OUTPUT) $(SOURCE) golangci-lint: GOBIN=$(GOBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.1 @@ -68,7 +68,7 @@ tidy: build: cd lib && go build ./... - CGO_ENABLED=1 go build ./... + go build ./... metrics: go install github.com/google/go-jsonnet/cmd/jsonnet@latest From 6fcba8e3912a27a9c301de5d93f23e4f49f9f991 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Thu, 28 Sep 2023 19:46:20 +0800 Subject: [PATCH 11/15] convert zstd level --- pkg/proxy/net/compress.go | 6 +++--- pkg/proxy/net/compress_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index c8e4fd07..8354ea3b 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -65,7 +65,7 @@ type compressedReadWriter struct { algorithm CompressAlgorithm logger *zap.Logger rwStatus rwStatus - zstdLevel int + zstdLevel zstd.EncoderLevel sequence uint8 } @@ -73,7 +73,7 @@ func newCompressedReadWriter(rw packetReadWriter, algorithm CompressAlgorithm, z return &compressedReadWriter{ packetReadWriter: rw, algorithm: algorithm, - zstdLevel: zstdLevel, + zstdLevel: zstd.EncoderLevelFromZstd(zstdLevel), logger: logger, rwStatus: rwNone, } @@ -276,7 +276,7 @@ func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { case CompressionZlib: compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.DefaultCompression) case CompressionZstd: - compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(zstd.EncoderLevel(crw.zstdLevel))) + compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(crw.zstdLevel)) } if err != nil { return nil, errors.WithStack(err) diff --git a/pkg/proxy/net/compress_test.go b/pkg/proxy/net/compress_test.go index 15745cfd..e780aec3 100644 --- a/pkg/proxy/net/compress_test.go +++ b/pkg/proxy/net/compress_test.go @@ -43,7 +43,7 @@ func TestCompressZlib(t *testing.T) { // Test read/write with zstd compression. func TestCompressZstd(t *testing.T) { sizes := []int{minCompressSize - 1, 1024, maxCompressedSize, maxCompressedSize + 1, maxCompressedSize * 2} - levels := []int{1, 3, 9, 20} + levels := []int{1, 3, 9, 22} lg, _ := logger.CreateLoggerForTest(t) for _, level := range levels { testkit.TestTCPConn(t, From f12c5c7bef3590c5e5bb15e9b28531ea3729b523 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 2 Oct 2023 13:14:56 +0800 Subject: [PATCH 12/15] free temp buffer if too large --- pkg/proxy/net/compress.go | 83 +++++++++++++++------------------- pkg/proxy/net/compress_test.go | 23 ++++++++++ 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 8354ea3b..742d2cf0 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -37,7 +37,9 @@ const ( // maxCompressedSize is the max uncompressed data size for a compressed packet. // Packets bigger than maxCompressedSize will be split into multiple compressed packets. // MySQL is 16K for the first packet and the rest for the second, MySQL Connector/J is 16M. - // The length itself must fit in the 3 byte field in the header. + // Two restrictions for the length: + // - it should be smaller than 16M so that the length can fit in the 3 byte field in the header. + // - it should be larger than 4M so that the compressed sequence can fit in the 3 byte field when max_allowed_packet is 1G. maxCompressedSize = 1<<24 - 1 // minCompressSize is the min uncompressed data size for compressed data. // Packets smaller than minCompressSize won't be compressed. @@ -60,7 +62,7 @@ var _ packetReadWriter = (*compressedReadWriter)(nil) type compressedReadWriter struct { packetReadWriter - readBuffer []byte + readBuffer bytes.Buffer writeBuffer bytes.Buffer algorithm CompressAlgorithm logger *zap.Logger @@ -100,17 +102,15 @@ func (crw *compressedReadWriter) beginRW(status rwStatus) { func (crw *compressedReadWriter) Read(p []byte) (n int, err error) { crw.beginRW(rwRead) // Read from the connection to fill the buffer if the buffer is empty. - if len(crw.readBuffer) == 0 { + if crw.readBuffer.Len() == 0 { if err = crw.readFromConn(); err != nil { return } } - n = copy(p, crw.readBuffer) - if n == len(crw.readBuffer) { - // Free the buffer. Too many idle connections may hold too much unnecessary memory. - crw.readBuffer = nil - } else { - crw.readBuffer = crw.readBuffer[n:] + n, err = crw.readBuffer.Read(p) + // Trade off between memory and efficiency. + if n == len(p) && crw.readBuffer.Len() == 0 && crw.readBuffer.Cap() > defaultReaderSize { + crw.readBuffer = bytes.Buffer{} } return } @@ -135,9 +135,9 @@ func (crw *compressedReadWriter) readFromConn() error { if uncompressedLength == 0 { // If the data is uncompressed, the uncompressed length is 0 and compressed length is the data length // after the compressed header. - crw.readBuffer = make([]byte, compressedLength) - if _, err = io.ReadFull(crw.packetReadWriter, crw.readBuffer); err != nil { - return err + crw.readBuffer.Grow(compressedLength) + if _, err = io.CopyN(&crw.readBuffer, crw.packetReadWriter, int64(compressedLength)); err != nil { + return errors.WithStack(err) } } else { // If the data is compressed, the compressed length is the length of data after the compressed header and @@ -146,7 +146,7 @@ func (crw *compressedReadWriter) readFromConn() error { if _, err = io.ReadFull(crw.packetReadWriter, data); err != nil { return err } - if crw.readBuffer, err = crw.uncompress(data, uncompressedLength); err != nil { + if err = crw.uncompress(data, uncompressedLength); err != nil { return err } } @@ -182,7 +182,12 @@ func (crw *compressedReadWriter) Flush() error { if len(data) == 0 { return nil } - crw.writeBuffer.Reset() + // Trade off between memory and efficiency. + if crw.writeBuffer.Cap() > defaultWriterSize { + crw.writeBuffer = bytes.Buffer{} + } else { + crw.writeBuffer.Reset() + } // If the data is uncompressed, the uncompressed length is 0 and compressed length is the data length // after the compressed header. @@ -228,42 +233,26 @@ func (crw *compressedReadWriter) DirectWrite(data []byte) (n int, err error) { // Notice: the peeked data may be discarded if an error is returned. func (crw *compressedReadWriter) Peek(n int) (data []byte, err error) { crw.beginRW(rwRead) - var readBuffer []byte - for len(readBuffer) < n { - if len(crw.readBuffer) == 0 { - if err = crw.readFromConn(); err != nil { - return - } + for crw.readBuffer.Len() < n { + if err = crw.readFromConn(); err != nil { + return } - readBuffer = append(readBuffer, crw.readBuffer...) - crw.readBuffer = nil } data = make([]byte, 0, n) - copy(data, readBuffer) - crw.readBuffer = readBuffer + copy(data, crw.readBuffer.Bytes()) return } // Discard won't be used. func (crw *compressedReadWriter) Discard(n int) (d int, err error) { crw.beginRW(rwRead) - for left := n; left > 0; { - if len(crw.readBuffer) == 0 { - if err = crw.readFromConn(); err != nil { - return - } - } - if left >= len(crw.readBuffer) { - left -= len(crw.readBuffer) - crw.readBuffer = nil - d += len(crw.readBuffer) - } else { - left = 0 - crw.readBuffer = crw.readBuffer[left:] - d += left + for crw.readBuffer.Len() < n { + if err = crw.readFromConn(); err != nil { + return } } - return + crw.readBuffer.Next(n) + return n, err } // DataDog/zstd is much faster but it's not good at cross-platform. @@ -290,27 +279,27 @@ func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { return compressedPacket.Bytes(), nil } -func (crw *compressedReadWriter) uncompress(data []byte, uncompressedLength int) ([]byte, error) { +func (crw *compressedReadWriter) uncompress(data []byte, uncompressedLength int) error { var err error var compressedReader io.ReadCloser switch crw.algorithm { case CompressionZlib: if compressedReader, err = zlib.NewReader(bytes.NewReader(data)); err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } case CompressionZstd: var decoder *zstd.Decoder if decoder, err = zstd.NewReader(bytes.NewReader(data)); err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } compressedReader = decoder.IOReadCloser() } - uncompressed := make([]byte, uncompressedLength) - if _, err = io.ReadFull(compressedReader, uncompressed); err != nil { - return nil, errors.WithStack(err) + crw.readBuffer.Grow(uncompressedLength) + if _, err = io.CopyN(&crw.readBuffer, compressedReader, int64(uncompressedLength)); err != nil { + return errors.WithStack(err) } if err = compressedReader.Close(); err != nil { - return nil, errors.WithStack(err) + return errors.WithStack(err) } - return uncompressed, nil + return nil } diff --git a/pkg/proxy/net/compress_test.go b/pkg/proxy/net/compress_test.go index e780aec3..e00e2bdc 100644 --- a/pkg/proxy/net/compress_test.go +++ b/pkg/proxy/net/compress_test.go @@ -211,6 +211,29 @@ func TestCompressHeader(t *testing.T) { }, 1) } +// Test that Read and Write returns correct errors. +func TestReadWriteError(t *testing.T) { + lg, _ := logger.CreateLoggerForTest(t) + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + }, + func(t *testing.T, c net.Conn) { + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + _, err := crw.Read(make([]byte, 1)) + require.True(t, IsDisconnectError(err)) + }, 1) + testkit.TestTCPConn(t, + func(t *testing.T, c net.Conn) { + }, + func(t *testing.T, c net.Conn) { + require.NoError(t, c.Close()) + crw := newCompressedReadWriter(newBasicReadWriter(c), CompressionZlib, 0, lg) + _, err := crw.Write(make([]byte, 1)) + require.NoError(t, err) + require.ErrorIs(t, crw.Flush(), net.ErrClosed) + }, 1) +} + func fillAndWrite(t *testing.T, crw *compressedReadWriter, b byte, length int) { data := fillData(b, length) _, err := crw.Write(data) From b22a705af93197a6ec1e3da5a2e49d4d043e197d Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Wed, 4 Oct 2023 08:50:37 +0800 Subject: [PATCH 13/15] set zlib level to 6 --- pkg/proxy/net/compress.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 742d2cf0..54f92573 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -45,6 +45,8 @@ const ( // Packets smaller than minCompressSize won't be compressed. // MySQL and MySQL Connector/J are both 50. minCompressSize = 50 + // defaultZlibLevel is the compression level for zlib. MySQL is 6. + zlibCompressionLevel = 6 ) func (p *PacketIO) SetCompressionAlgorithm(algorithm CompressAlgorithm, zstdLevel int) error { @@ -263,7 +265,7 @@ func (crw *compressedReadWriter) compress(data []byte) ([]byte, error) { var compressWriter io.WriteCloser switch crw.algorithm { case CompressionZlib: - compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlib.DefaultCompression) + compressWriter, err = zlib.NewWriterLevel(&compressedPacket, zlibCompressionLevel) case CompressionZstd: compressWriter, err = zstd.NewWriter(&compressedPacket, zstd.WithEncoderLevel(crw.zstdLevel)) } From 6653fa3de98e13c1ec579c325e3fa369839ce766 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 9 Oct 2023 14:51:42 +0800 Subject: [PATCH 14/15] update comments --- pkg/proxy/net/compress.go | 2 +- pkg/proxy/net/compress_test.go | 6 +++--- pkg/proxy/net/tls.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/proxy/net/compress.go b/pkg/proxy/net/compress.go index 54f92573..5e6b8d63 100644 --- a/pkg/proxy/net/compress.go +++ b/pkg/proxy/net/compress.go @@ -36,7 +36,7 @@ const ( const ( // maxCompressedSize is the max uncompressed data size for a compressed packet. // Packets bigger than maxCompressedSize will be split into multiple compressed packets. - // MySQL is 16K for the first packet and the rest for the second, MySQL Connector/J is 16M. + // MySQL has 16K for the first packet. The rest packets and MySQL Connector/J are 16M. // Two restrictions for the length: // - it should be smaller than 16M so that the length can fit in the 3 byte field in the header. // - it should be larger than 4M so that the compressed sequence can fit in the 3 byte field when max_allowed_packet is 1G. diff --git a/pkg/proxy/net/compress_test.go b/pkg/proxy/net/compress_test.go index e00e2bdc..48a54141 100644 --- a/pkg/proxy/net/compress_test.go +++ b/pkg/proxy/net/compress_test.go @@ -28,7 +28,7 @@ func TestCompressZlib(t *testing.T) { require.NoError(t, crw.Flush()) // Check compressed bytes. outBytes := crw.OutBytes() - checkByteNum(t, outBytes-written, size) + checkWrittenByteSize(t, outBytes-written, size) written = outBytes } }, @@ -55,7 +55,7 @@ func TestCompressZstd(t *testing.T) { require.NoError(t, crw.Flush()) // Check compressed bytes. outBytes := crw.OutBytes() - checkByteNum(t, outBytes-written, size) + checkWrittenByteSize(t, outBytes-written, size) written = outBytes } }, @@ -265,7 +265,7 @@ func checkData(t *testing.T, data []byte, b byte) { } } -func checkByteNum(t *testing.T, diff uint64, size int) { +func checkWrittenByteSize(t *testing.T, diff uint64, size int) { if size < minCompressSize { require.Equal(t, uint64(size+7), diff) } else { diff --git a/pkg/proxy/net/tls.go b/pkg/proxy/net/tls.go index 4170af7b..2d5bdf91 100644 --- a/pkg/proxy/net/tls.go +++ b/pkg/proxy/net/tls.go @@ -12,7 +12,8 @@ import ( // tlsHandshakeConn is only used as the underlying connection in tls.Conn. // TLS handshake must read from the buffered reader because the handshake data may be already buffered in the reader. -// TLS handshake can not use the buffered writer directly because it assumes the data is always flushed immediately. +// TLS handshake can not use the buffered writer directly because it assumes the data will be flushed automatically, +// however buffered writer may not flush without calling `Flush`. type tlsInternalConn struct { packetReadWriter } From 3e1c136b74dbf6cb9d69a3f6ecfcb8cd8f121309 Mon Sep 17 00:00:00 2001 From: djshow832 <873581766@qq.com> Date: Mon, 9 Oct 2023 15:00:35 +0800 Subject: [PATCH 15/15] fix proxy peek --- pkg/proxy/net/proxy.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/proxy/net/proxy.go b/pkg/proxy/net/proxy.go index a3c0b293..f4da3146 100644 --- a/pkg/proxy/net/proxy.go +++ b/pkg/proxy/net/proxy.go @@ -63,9 +63,6 @@ func (prw *proxyReadWriter) Read(b []byte) (int, error) { return 0, errors.Wrap(ErrReadConn, err) } if bytes.Equal(header[:], proxyprotocol.MagicV2[:4]) { - if _, err = prw.Discard(4); err != nil { - return 0, err - } proxyHeader, err := prw.parseProxyV2() if err != nil { return 0, errors.Wrap(ErrReadConn, err) @@ -99,16 +96,16 @@ func (prw *proxyReadWriter) Write(p []byte) (n int, err error) { } func (prw *proxyReadWriter) parseProxyV2() (*proxyprotocol.Proxy, error) { - rem, err := prw.packetReadWriter.Peek(8) + rem, err := prw.packetReadWriter.Peek(len(proxyprotocol.MagicV2)) if err != nil { return nil, errors.WithStack(errors.Wrap(ErrReadConn, err)) } - if !bytes.Equal(rem, proxyprotocol.MagicV2[4:]) { + if !bytes.Equal(rem, proxyprotocol.MagicV2) { return nil, nil } // yes, it is proxyV2 - _, err = prw.packetReadWriter.Discard(8) + _, err = prw.packetReadWriter.Discard(len(proxyprotocol.MagicV2)) if err != nil { return nil, errors.WithStack(errors.Wrap(ErrReadConn, err)) }