From 192888579236676e53842a32438520b6354a0846 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 4 Dec 2024 15:45:16 +0200 Subject: [PATCH 1/8] emulated etcd version --- cmd/root.go | 3 +++ pkg/kine/endpoint/endpoint.go | 6 +++--- pkg/kine/server/maintenance.go | 5 +++-- pkg/kine/server/server.go | 6 ++++-- pkg/server/server.go | 3 +++ 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 0c220bef..76d5cd26 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -32,6 +32,7 @@ var ( metricsAddress string otel bool otelAddress string + emulatedEtcdVersion string connectionPoolConfig generic.ConnectionPoolConfig @@ -101,6 +102,7 @@ var ( rootCmdOpts.diskMode, rootCmdOpts.clientSessionCacheSize, rootCmdOpts.minTLSVersion, + rootCmdOpts.emulatedEtcdVersion, rootCmdOpts.watchAvailableStorageInterval, rootCmdOpts.watchAvailableStorageMinBytes, rootCmdOpts.lowAvailableStorageAction, @@ -173,6 +175,7 @@ func init() { rootCmd.Flags().BoolVar(&rootCmdOpts.otel, "otel", false, "enable traces endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.otelAddress, "otel-listen", "127.0.0.1:4317", "listen address for OpenTelemetry endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.metricsAddress, "metrics-listen", "127.0.0.1:9042", "listen address for metrics endpoint") + rootCmd.Flags().StringVar(&rootCmdOpts.emulatedEtcdVersion, "emulated-etcd-version", "3.5.13", "The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.13, in order to indicate support for watch progress notifications.") rootCmd.Flags().IntVar(&rootCmdOpts.connectionPoolConfig.MaxIdle, "datastore-max-idle-connections", 5, "Maximum number of idle connections retained by datastore. If value = 0, the system default will be used. If value < 0, idle connections will not be reused.") rootCmd.Flags().IntVar(&rootCmdOpts.connectionPoolConfig.MaxOpen, "datastore-max-open-connections", 5, "Maximum number of open connections used by datastore. If value <= 0, then there is no limit") rootCmd.Flags().DurationVar(&rootCmdOpts.connectionPoolConfig.MaxLifetime, "datastore-connection-max-lifetime", 60*time.Second, "Maximum amount of time a connection may be reused. If value <= 0, then there is no limit.") diff --git a/pkg/kine/endpoint/endpoint.go b/pkg/kine/endpoint/endpoint.go index 46880ca9..92f0e6dd 100644 --- a/pkg/kine/endpoint/endpoint.go +++ b/pkg/kine/endpoint/endpoint.go @@ -30,8 +30,8 @@ type Config struct { GRPCServer *grpc.Server Listener string Endpoint string + EmulatedEtcdVersion string ConnectionPoolConfig generic.ConnectionPoolConfig - tls.Config } @@ -65,7 +65,7 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) { listen = KineSocket } - b := server.New(backend) + b := server.New(backend, config.EmulatedEtcdVersion) grpcServer := grpcServer(config) b.Register(grpcServer) @@ -130,7 +130,7 @@ func ListenAndReturnBackend(ctx context.Context, config Config) (ETCDConfig, ser listen = KineSocket } - b := server.New(backend) + b := server.New(backend, config.EmulatedEtcdVersion) grpcServer := grpcServer(config) b.Register(grpcServer) diff --git a/pkg/kine/server/maintenance.go b/pkg/kine/server/maintenance.go index bc05aa83..932686ef 100644 --- a/pkg/kine/server/maintenance.go +++ b/pkg/kine/server/maintenance.go @@ -19,8 +19,9 @@ func (s *KVServerBridge) Status(ctx context.Context, r *etcdserverpb.StatusReque return nil, err } return &etcdserverpb.StatusResponse{ - Header: &etcdserverpb.ResponseHeader{}, - DbSize: size, + Header: &etcdserverpb.ResponseHeader{}, + DbSize: size, + Version: s.emulatedETCDVersion, }, nil } diff --git a/pkg/kine/server/server.go b/pkg/kine/server/server.go index e204e952..5768b6d4 100644 --- a/pkg/kine/server/server.go +++ b/pkg/kine/server/server.go @@ -18,11 +18,13 @@ var ( ) type KVServerBridge struct { - limited *LimitedServer + limited *LimitedServer + emulatedETCDVersion string } -func New(backend Backend) *KVServerBridge { +func New(backend Backend, emulatedEtcdVersion string) *KVServerBridge { return &KVServerBridge{ + emulatedETCDVersion: emulatedEtcdVersion, limited: &LimitedServer{ backend: backend, }, diff --git a/pkg/server/server.go b/pkg/server/server.go index 21c2f89f..f83b8da8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -64,6 +64,7 @@ func New( diskMode bool, clientSessionCacheSize uint, minTLSVersion string, + emulatedEtcdVersion string, watchAvailableStorageInterval time.Duration, watchAvailableStorageMinBytes uint64, lowAvailableStorageAction string, @@ -222,6 +223,8 @@ func New( } // set datastore connection pool options kineConfig.ConnectionPoolConfig = connectionPoolConfig + // set emulated etcd version + kineConfig.EmulatedEtcdVersion = emulatedEtcdVersion // handle tuning parameters if exists, err := fileExists(dir, "tuning.yaml"); err != nil { return nil, fmt.Errorf("failed to check for tuning.yaml: %w", err) From 1b60942cb55cdf1603af7eaedf28cbf62a596b5e Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Wed, 4 Dec 2024 15:26:37 +0100 Subject: [PATCH 2/8] nit --- docs/configuration.md | 1 + pkg/kine/server/maintenance.go | 2 +- pkg/kine/server/server.go | 4 ++-- pkg/server/server.go | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ee80413c..c96b7457 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ The following configuration options are available listed in a table format: | `--otel` | Enable traces endpoint | `false` | | `--otel-listen` | The address to listen for OpenTelemetry endpoint | `127.0.0.1:4317` | | `--metrics-listen` | The address to listen for metrics endpoint | `127.0.0.1:9042` | +| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.13, in order to indicate support for watch progress notifications. | `3.5.13` | | `--datastore-max-idle-connections` | Maximum number of idle connections retained by datastore | `5` | | `--datastore-max-open-connections` | Maximum number of open connections used by datastore | `5` | | `--datastore-connection-max-lifetime` | Maximum amount of time a connection may be reused | `60s` | diff --git a/pkg/kine/server/maintenance.go b/pkg/kine/server/maintenance.go index 932686ef..5e79cd9e 100644 --- a/pkg/kine/server/maintenance.go +++ b/pkg/kine/server/maintenance.go @@ -21,7 +21,7 @@ func (s *KVServerBridge) Status(ctx context.Context, r *etcdserverpb.StatusReque return &etcdserverpb.StatusResponse{ Header: &etcdserverpb.ResponseHeader{}, DbSize: size, - Version: s.emulatedETCDVersion, + Version: s.emulatedEtcdVersion, }, nil } diff --git a/pkg/kine/server/server.go b/pkg/kine/server/server.go index 5768b6d4..9fc01ab5 100644 --- a/pkg/kine/server/server.go +++ b/pkg/kine/server/server.go @@ -19,12 +19,12 @@ var ( type KVServerBridge struct { limited *LimitedServer - emulatedETCDVersion string + emulatedEtcdVersion string } func New(backend Backend, emulatedEtcdVersion string) *KVServerBridge { return &KVServerBridge{ - emulatedETCDVersion: emulatedEtcdVersion, + emulatedEtcdVersion: emulatedEtcdVersion, limited: &LimitedServer{ backend: backend, }, diff --git a/pkg/server/server.go b/pkg/server/server.go index f83b8da8..16cb5a77 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -343,7 +343,7 @@ func (s *Server) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to start kine: %w", err) } - logrus.WithFields(logrus.Fields{"address": s.kineConfig.Listener, "database": s.kineConfig.Endpoint}).Print("Started kine") + logrus.WithFields(logrus.Fields{"address": s.kineConfig.Listener, "database": s.kineConfig.Endpoint, "emulatedEtcdVersion": s.kineConfig.EmulatedEtcdVersion}).Print("Started kine") s.backend = backend From 0fd87d4ddf9965afe695dd7ca9c32b260a1a8a44 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 5 Dec 2024 09:25:49 +0100 Subject: [PATCH 3/8] test no WatchList supported etcd version with 1.32rc --- .github/workflows/k8s-snap-integration.yaml | 14 +++++++++----- cmd/root.go | 2 +- docs/configuration.md | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/k8s-snap-integration.yaml b/.github/workflows/k8s-snap-integration.yaml index c23b0cbf..75b6ff5f 100644 --- a/.github/workflows/k8s-snap-integration.yaml +++ b/.github/workflows/k8s-snap-integration.yaml @@ -2,7 +2,7 @@ name: Integration Test K8s-snap on: push: - branches: ["master"] + branches: ["master", "KU-2276/emulated-etcd-version"] #TODO remove KU-2276/emulated-etcd-version pull_request: permissions: @@ -14,7 +14,11 @@ concurrency: jobs: build: - name: K8s-snap Integration Test + name: K8s-snap Integration Test ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] runs-on: ubuntu-20.04 steps: @@ -30,9 +34,9 @@ jobs: uses: actions/setup-go@v5 with: go-version: "1.21" - - name: Download latest k8s-snap + - name: Download latest k8s-snap #TODO remove v1.32.0-rc0 run: | - sudo snap download k8s --channel=latest/edge --basename k8s + sudo snap download k8s --channel=latest/edge/v1.32.0-rc0 --basename k8s - name: Install lxd run: | sudo snap refresh lxd --channel 5.21/stable @@ -56,7 +60,7 @@ jobs: env: TEST_SNAP: ${{ github.workspace }}/k8s-updated.snap TEST_SUBSTRATE: lxd - TEST_LXD_IMAGE: ubuntu:22.04 + TEST_LXD_IMAGE: ${{ matrix.os }} TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports run: | git clone https://github.com/canonical/k8s-snap.git diff --git a/cmd/root.go b/cmd/root.go index 76d5cd26..ad59189c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -175,7 +175,7 @@ func init() { rootCmd.Flags().BoolVar(&rootCmdOpts.otel, "otel", false, "enable traces endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.otelAddress, "otel-listen", "127.0.0.1:4317", "listen address for OpenTelemetry endpoint") rootCmd.Flags().StringVar(&rootCmdOpts.metricsAddress, "metrics-listen", "127.0.0.1:9042", "listen address for metrics endpoint") - rootCmd.Flags().StringVar(&rootCmdOpts.emulatedEtcdVersion, "emulated-etcd-version", "3.5.13", "The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.13, in order to indicate support for watch progress notifications.") + rootCmd.Flags().StringVar(&rootCmdOpts.emulatedEtcdVersion, "emulated-etcd-version", "3.5.12", "The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.12, in order to indicate no support for watch progress notifications yet.") rootCmd.Flags().IntVar(&rootCmdOpts.connectionPoolConfig.MaxIdle, "datastore-max-idle-connections", 5, "Maximum number of idle connections retained by datastore. If value = 0, the system default will be used. If value < 0, idle connections will not be reused.") rootCmd.Flags().IntVar(&rootCmdOpts.connectionPoolConfig.MaxOpen, "datastore-max-open-connections", 5, "Maximum number of open connections used by datastore. If value <= 0, then there is no limit") rootCmd.Flags().DurationVar(&rootCmdOpts.connectionPoolConfig.MaxLifetime, "datastore-connection-max-lifetime", 60*time.Second, "Maximum amount of time a connection may be reused. If value <= 0, then there is no limit.") diff --git a/docs/configuration.md b/docs/configuration.md index c96b7457..12f72a68 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,7 +21,7 @@ The following configuration options are available listed in a table format: | `--otel` | Enable traces endpoint | `false` | | `--otel-listen` | The address to listen for OpenTelemetry endpoint | `127.0.0.1:4317` | | `--metrics-listen` | The address to listen for metrics endpoint | `127.0.0.1:9042` | -| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.13, in order to indicate support for watch progress notifications. | `3.5.13` | +| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.12, in order to indicate no support for watch progress notifications yet. | `3.5.12` | | `--datastore-max-idle-connections` | Maximum number of idle connections retained by datastore | `5` | | `--datastore-max-open-connections` | Maximum number of open connections used by datastore | `5` | | `--datastore-connection-max-lifetime` | Maximum amount of time a connection may be reused | `60s` | From ad5ede2fe84fc64176382a2d98ede7637557ffb3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 5 Dec 2024 10:10:57 +0100 Subject: [PATCH 4/8] clean up --- .github/workflows/k8s-snap-integration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s-snap-integration.yaml b/.github/workflows/k8s-snap-integration.yaml index 75b6ff5f..5153271d 100644 --- a/.github/workflows/k8s-snap-integration.yaml +++ b/.github/workflows/k8s-snap-integration.yaml @@ -2,7 +2,7 @@ name: Integration Test K8s-snap on: push: - branches: ["master", "KU-2276/emulated-etcd-version"] #TODO remove KU-2276/emulated-etcd-version + branches: ["master"] pull_request: permissions: @@ -34,9 +34,9 @@ jobs: uses: actions/setup-go@v5 with: go-version: "1.21" - - name: Download latest k8s-snap #TODO remove v1.32.0-rc0 + - name: Download latest k8s-snap run: | - sudo snap download k8s --channel=latest/edge/v1.32.0-rc0 --basename k8s + sudo snap download k8s --channel=latest/edge --basename k8s - name: Install lxd run: | sudo snap refresh lxd --channel 5.21/stable From 48b6848322d7ab33c91c4b5e6e9942d9252644ee Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Thu, 5 Dec 2024 14:49:56 +0100 Subject: [PATCH 5/8] clean-up --- .github/workflows/k8s-snap-integration.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/k8s-snap-integration.yaml b/.github/workflows/k8s-snap-integration.yaml index 5153271d..c23b0cbf 100644 --- a/.github/workflows/k8s-snap-integration.yaml +++ b/.github/workflows/k8s-snap-integration.yaml @@ -14,11 +14,7 @@ concurrency: jobs: build: - name: K8s-snap Integration Test ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] + name: K8s-snap Integration Test runs-on: ubuntu-20.04 steps: @@ -60,7 +56,7 @@ jobs: env: TEST_SNAP: ${{ github.workspace }}/k8s-updated.snap TEST_SUBSTRATE: lxd - TEST_LXD_IMAGE: ${{ matrix.os }} + TEST_LXD_IMAGE: ubuntu:22.04 TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports run: | git clone https://github.com/canonical/k8s-snap.git From e93b230539339381f1b76f25ccb6c0b705803deb Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Fri, 6 Dec 2024 14:18:01 +0100 Subject: [PATCH 6/8] switch to etcd 3.5.7 --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 12f72a68..4672a3a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,7 +21,7 @@ The following configuration options are available listed in a table format: | `--otel` | Enable traces endpoint | `false` | | `--otel-listen` | The address to listen for OpenTelemetry endpoint | `127.0.0.1:4317` | | `--metrics-listen` | The address to listen for metrics endpoint | `127.0.0.1:9042` | -| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.12, in order to indicate no support for watch progress notifications yet. | `3.5.12` | +| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.7, in order to indicate no support for watch progress notifications yet. | `3.5.7` | | `--datastore-max-idle-connections` | Maximum number of idle connections retained by datastore | `5` | | `--datastore-max-open-connections` | Maximum number of open connections used by datastore | `5` | | `--datastore-connection-max-lifetime` | Maximum amount of time a connection may be reused | `60s` | From 58783342557f27345ac2a391f5178d75d2fafea3 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Mon, 9 Dec 2024 14:03:14 +0100 Subject: [PATCH 7/8] correct get req range end and limit check --- pkg/kine/server/get.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/kine/server/get.go b/pkg/kine/server/get.go index 03daeb4a..91a26ec5 100644 --- a/pkg/kine/server/get.go +++ b/pkg/kine/server/get.go @@ -23,11 +23,9 @@ func (l *LimitedServer) get(ctx context.Context, r *etcdserverpb.RangeRequest) ( attribute.Int64("revision", r.Revision), ) - if len(r.RangeEnd) != 0 { - return nil, fmt.Errorf("unexpected rangeEnd: want empty, got %s", r.RangeEnd) - } - if r.Limit != 0 { - return nil, fmt.Errorf("unexpected limit: want 0, got %d", r.Limit) + if r.Limit != 0 && len(r.RangeEnd) != 0 { + err := fmt.Errorf("invalid combination of rangeEnd and limit, limit should be 0 got %d", r.Limit) + return nil, err } rev, kv, err := l.backend.List(ctx, string(r.Key), "", 1, r.Revision) From 6440c52bdd7b3cc7bf4410d99067ce7dcbb5cd62 Mon Sep 17 00:00:00 2001 From: louiseschmidtgen Date: Tue, 10 Dec 2024 15:34:22 +0100 Subject: [PATCH 8/8] consistency in emulated etcd version --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4672a3a1..12f72a68 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,7 +21,7 @@ The following configuration options are available listed in a table format: | `--otel` | Enable traces endpoint | `false` | | `--otel-listen` | The address to listen for OpenTelemetry endpoint | `127.0.0.1:4317` | | `--metrics-listen` | The address to listen for metrics endpoint | `127.0.0.1:9042` | -| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.7, in order to indicate no support for watch progress notifications yet. | `3.5.7` | +| `--emulated-etcd-version` | The emulated etcd version to return on a call to the status endpoint. Defaults to 3.5.12, in order to indicate no support for watch progress notifications yet. | `3.5.12` | | `--datastore-max-idle-connections` | Maximum number of idle connections retained by datastore | `5` | | `--datastore-max-open-connections` | Maximum number of open connections used by datastore | `5` | | `--datastore-connection-max-lifetime` | Maximum amount of time a connection may be reused | `60s` |