Merge pull request #424 from DataDog/jamie/restructure

jamie/restructure

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04 as base
+FROM --platform=linux/amd64 ubuntu:22.04 as base
 
 # Install pre-reqs
 ARG DEBIAN_FRONTEND=noninteractive
@@ -8,10 +8,18 @@ RUN apt install -y apt-utils jq build-essential unzip curl git pkg-config softwa
 WORKDIR /root
 
 # Install Go
-RUN curl -sOL https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+RUN curl -sOL https://go.dev/dl/go1.20.3.linux-amd64.tar.gz
+RUN tar -C /usr/local -xzf go1.20.3.linux-amd64.tar.gz
 ENV PATH=$PATH:/usr/local/go/bin:/go/bin
 ENV GOPATH=/go
+RUN rm -rf go*
+
+# Install protoc
+RUN curl -sOL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.1/protoc-3.19.1-linux-x86_64.zip
+RUN unzip protoc-3.19.1-linux-x86_64.zip -d protoc
+RUN mv protoc/bin/* /usr/local/bin/
+RUN mv protoc/include/* /usr/local/include/
+RUN rm -rf protoc*
 
 # Install librdkafka
 RUN curl -sL https://packages.confluent.io/deb/6.1/archive.key | apt-key add - 2>/dev/null
@@ -20,45 +28,37 @@ RUN apt-get update && apt-get install -y librdkafka1 librdkafka-dev >/dev/null
 
 # Init repo.
 WORKDIR /go/src/github.com/DataDog/kafka-kit
+COPY cluster cluster
+COPY cmd cmd
+COPY internal internal
+COPY kafkaadmin kafkaadmin
+COPY kafkametrics kafkametrics
+COPY kafkazk kafkazk
+COPY mapper mapper
+COPY proto proto
 COPY tools.go tools.go
 COPY go.mod go.mod
 COPY go.sum go.sum
 RUN go mod download
 
-# Install protoc
-RUN curl -sOL https://github.com/protocolbuffers/protobuf/releases/download/v3.19.1/protoc-3.19.1-linux-x86_64.zip
-RUN unzip protoc-3.19.1-linux-x86_64.zip -d protoc
-RUN mv protoc/bin/* /usr/local/bin/
-RUN mv protoc/include/* /usr/local/include/
-RUN rm -rf protoc*
-
 # Install protoc / gRPC deps; these versions are managed in go.mod
 RUN go get -d github.com/googleapis/googleapis
 RUN go install \
   google.golang.org/protobuf/cmd/protoc-gen-go \
   google.golang.org/grpc/cmd/protoc-gen-go-grpc \
   github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
 
-# Copy source.
-COPY cmd cmd
-COPY cluster cluster
-COPY kafkaadmin kafkaadmin
-COPY kafkametrics kafkametrics
-COPY kafkazk kafkazk
-COPY mapper mapper
-COPY registry registry
-
 # Codegen
-RUN protoc -I ./registry -I $GOPATH/pkg/mod/$(awk '/googleapis/ {printf "%s@%s", $1, $2}' go.mod) \
-    --go_out ./registry \
+RUN protoc -I ./proto/registrypb -I $GOPATH/pkg/mod/$(awk '/googleapis/ {printf "%s@%s", $1, $2}' go.mod) \
+    --go_out ./proto/registrypb \
     --go_opt paths=source_relative \
-    --go-grpc_out ./registry \
+    --go-grpc_out ./proto/registrypb \
     --go-grpc_opt paths=source_relative \
-    --grpc-gateway_out ./registry \
+    --grpc-gateway_out ./proto/registrypb \
     --grpc-gateway_opt logtostderr=true \
     --grpc-gateway_opt paths=source_relative \
     --grpc-gateway_opt generate_unbound_methods=true \
-    registry/registry/registry.proto
+    proto/registrypb/registry.proto
 
 # Build
 RUN go install ./cmd/...
@@ -73,5 +73,7 @@ ENTRYPOINT ["/entrypoint.sh"]
 FROM registry.ddbuild.io/images/base/gbi-ubuntu_2204 as dd-image
 
 COPY --from=base /entrypoint.sh /
+COPY --from=base /go/src /go/src
 COPY --from=base /go/bin /usr/bin
+
 ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.ssl_setup

@@ -1,4 +1,4 @@
-from ubuntu:22.04
+from --platform=linux/amd64 ubuntu:22.04
 
 RUN apt-get update >/dev/null
 RUN apt-get install -y openjdk-8-jdk openssl >/dev/null

Makefile

@@ -4,33 +4,34 @@ K := $(foreach exec,$(EXECUTABLES),\
 
 # Run a Docker compose environment.
 run-compose: compose-build
-	docker-compose up -d --scale kafka=3
+	docker compose up -d --wait
 
 # Tear down the Docker compose environment.
 stop-compose:
-	docker-compose down
+	docker compose down
 
 kill-compose:
-		docker-compose kill
+	docker compose kill
 
 # Ensure any local images used by compose are up to date.
 compose-build:
-	docker-compose build
+	docker compose build
 
 # Build the Kafka-Kit image.
 build-image:
-	docker buildx build --load --platform linux/amd64 -t kafka-kit -f Dockerfile .
+	docker buildx build --load --platform linux/amd64 -t kafka-kit --target base -f Dockerfile .
 
 # Run unit tests.
 test:
 	go test -v ./...
 
 # Run all tests.
-integration-test: kill-compose stop-compose compose-build run-compose
-	docker-compose run --rm --name integration-test registry go test -timeout 30s --tags integration ./...
+integration-test: stop-compose build-image run-compose
+	docker run --platform linux/amd64 --rm --network kafka-kit_default --name integration-test kafka-kit go test -timeout 30s --tags integration ./...
 
 # Generate proto code outputs.
 generate-code: build-image
-	docker create --name kafka-kit kafka-kit >/dev/null; \
-	docker cp kafka-kit:/go/src/github.com/DataDog/kafka-kit/registry/registry/. ${CURDIR}/registry/registry; \
+	docker create --platform linux/amd64 --name kafka-kit kafka-kit >/dev/null; \
+	docker cp kafka-kit:/go/src/github.com/DataDog/kafka-kit/proto/registrypb/. ${CURDIR}/proto/registrypb; \
 	docker rm kafka-kit >/dev/null
+

cluster/zookeeper/locking_test.go

@@ -13,20 +13,20 @@ import (
 func TestLock(t *testing.T) {
 	lock := newMockZooKeeperLock()
 	ctx, cf := context.WithTimeout(context.Background(), 1*time.Second)
-	_ = cf
+	_ = cf // Escape the linter.
 
 	// This lock should succeed normally.
 	err := lock.Lock(ctx)
 	assert.Nil(t, err)
 
 	// This lock should time out.
 	err2 := lock.Lock(ctx)
-	assert.Equal(t, err2, ErrLockingTimedOut, "Expected ErrLockingTimedOut")
+	assert.Equal(t, ErrLockingTimedOut, err2, "Expected ErrLockingTimedOut")
 }
 
 func TestLockSameOwner(t *testing.T) {
 	lock := newMockZooKeeperLock()
-	ctx, cf := context.WithTimeout(context.Background(), 1*time.Second)
+	ctx, cf := context.WithTimeout(context.Background(), 3*time.Second)
 	ctx = context.WithValue(ctx, "owner", "owner")
 	_ = cf
 
@@ -37,12 +37,12 @@ func TestLockSameOwner(t *testing.T) {
 	// This should also succeed (with a soft error) because we have the same
 	// instance, same owner key/value.
 	err2 := lock.Lock(ctx)
-	assert.Equal(t, err2, ErrAlreadyOwnLock)
+	assert.Equal(t, ErrAlreadyOwnLock, err2)
 }
 
 func TestUnlock(t *testing.T) {
 	lock := newMockZooKeeperLock()
-	ctx, cf := context.WithTimeout(context.Background(), 1*time.Second)
+	ctx, cf := context.WithTimeout(context.Background(), 3*time.Second)
 	_ = cf
 
 	// This lock should succeed normally.
@@ -76,8 +76,8 @@ func TestExpireLockAhead(t *testing.T) {
 	id, _ := idFromZnode(node)
 
 	// Check that the lock state has been populated.
-	assert.Equal(t, lock.owner, "test_owner")
-	assert.Equal(t, lock.lockZnode, "/locks/_c_979cb11f40bb3dbc6908edeaac8f2de1-lock-000000001")
+	assert.Equal(t, "test_owner", lock.owner)
+	assert.Equal(t, "/locks/_c_979cb11f40bb3dbc6908edeaac8f2de1-lock-000000001", lock.lockZnode)
 
 	// Get the current lock entries.
 	le, _ := lock.locks()
@@ -97,10 +97,10 @@ func TestExpireLockAhead(t *testing.T) {
 	// This should now fail; the lock was expired and the only entry is ID 2
 	// for the pending claim we entered above.
 	expired, err = lock.expireLockAhead(le, id)
-	assert.Equal(t, err, ErrExpireLockFailed{message: "unable to determine which lock to enqueue behind"})
+	assert.Equal(t, ErrExpireLockFailed{message: "unable to determine which lock to enqueue behind"}, err)
 	assert.False(t, expired)
 
 	// Check that the lock state has been cleared.
 	assert.Nil(t, lock.owner)
-	assert.Equal(t, lock.lockZnode, "")
+	assert.Equal(t, "", lock.lockZnode)
 }