From 5c0d7a3278c7b7421d23d2fa4bcc8300d54ed4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Fri, 14 Feb 2025 08:09:46 +0100 Subject: [PATCH] feat(minio): add service name and version to MinIO requests (#728) Because - We want to identify the service that is performing a MinIO action. This commit - Adds client information (app name and version) to the MinIO requests according to instill-ai/x/pull/37. --- .github/workflows/images.yml | 2 ++ .github/workflows/integration-test.yml | 1 + Dockerfile | 41 ++++++++++++++++++++++---- cmd/main/main.go | 16 ++++++++-- cmd/worker/main.go | 18 +++++++++-- go.mod | 2 +- go.sum | 4 +-- 7 files changed, 71 insertions(+), 13 deletions(-) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 43b030b7..635bc256 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -53,6 +53,7 @@ jobs: provenance: false build-args: | SERVICE_NAME=model-backend + SERVICE_VERSION=${{ github.sha }} tags: instill/model-backend:latest cache-from: type=registry,ref=instill/model-backend:buildcache cache-to: type=registry,ref=instill/model-backend:buildcache,mode=max @@ -79,6 +80,7 @@ jobs: provenance: false build-args: | SERVICE_NAME=model-backend + SERVICE_VERSION=${{steps.set_version.outputs.no_v_tag}} tags: instill/model-backend:${{steps.set_version.outputs.no_v_tag}} cache-from: type=registry,ref=instill/model-backend:buildcache cache-to: type=registry,ref=instill/model-backend:buildcache,mode=max diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 40d28ef2..050ed603 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -80,6 +80,7 @@ jobs: load: true build-args: | SERVICE_NAME=model-backend + SERVICE_VERSION=${{ github.sha }} tags: instill/model-backend:latest - name: Checkout repo (instill-core) diff --git a/Dockerfile b/Dockerfile index a355329d..a931f990 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM --platform=$BUILDPLATFORM golang:1.22.5 AS build -ARG SERVICE_NAME +ARG SERVICE_NAME SERVICE_VERSION WORKDIR /src @@ -9,11 +9,40 @@ RUN go mod download COPY . . ARG TARGETOS TARGETARCH -RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /${SERVICE_NAME} ./cmd/main -RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /${SERVICE_NAME}-migrate ./cmd/migration -RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /${SERVICE_NAME}-init ./cmd/init -RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /${SERVICE_NAME}-worker ./cmd/worker -RUN --mount=target=. --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /${SERVICE_NAME}-init-model ./cmd/model +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH \ + go build -ldflags "-X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}" \ + -o /${SERVICE_NAME} ./cmd/main + +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH \ + go build -ldflags "-X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-migrate" \ + -o /${SERVICE_NAME}-migrate ./cmd/migration + +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH \ + go build -ldflags "-X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-init" \ + -o /${SERVICE_NAME}-init ./cmd/init + +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH \ + go build -ldflags "-X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-worker" \ + -o /${SERVICE_NAME}-worker ./cmd/worker + +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + GOOS=$TARGETOS CGO_ENABLED=0 GOARCH=$TARGETARCH \ + go build -ldflags "-X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-init-model" \ + -o /${SERVICE_NAME}-init-model ./cmd/model # Mounting points RUN mkdir /model-config diff --git a/cmd/main/main.go b/cmd/main/main.go index f3c0874e..58427182 100644 --- a/cmd/main/main.go +++ b/cmd/main/main.go @@ -57,6 +57,10 @@ import ( var propagator = b3.New(b3.WithInjectEncoding(b3.B3MultipleHeader)) +// These variables might be overridden at buildtime. +var version = "dev" +var serviceName = "model-backend" + func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler) http.Handler { return h2c.NewHandler( @@ -231,9 +235,17 @@ func main() { timeseries := repository.MustNewInfluxDB(ctx, config.Config.Server.Debug) defer timeseries.Close() - // Initialize Minio client + // Initialize MinIO client retentionHandler := service.NewRetentionHandler() - minioClient, err := miniox.NewMinioClientAndInitBucket(ctx, &config.Config.Minio, logger, retentionHandler.ListExpiryRules()...) + minioClient, err := miniox.NewMinioClientAndInitBucket(ctx, miniox.ClientParams{ + Config: config.Config.Minio, + Logger: logger, + ExpiryRules: retentionHandler.ListExpiryRules(), + AppInfo: miniox.AppInfo{ + Name: serviceName, + Version: version, + }, + }) if err != nil { logger.Fatal("failed to create minio client", zap.Error(err)) } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 5b6fa900..7ec57a32 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -24,6 +24,7 @@ import ( "github.com/instill-ai/model-backend/pkg/datamodel" "github.com/instill-ai/model-backend/pkg/ray" "github.com/instill-ai/model-backend/pkg/repository" + "github.com/instill-ai/model-backend/pkg/service" "github.com/instill-ai/x/temporal" "github.com/instill-ai/x/zapadapter" @@ -34,6 +35,10 @@ import ( miniox "github.com/instill-ai/x/minio" ) +// These variables might be overridden at buildtime. +var version = "dev" +var serviceName = "model-backend-worker" + func initTemporalNamespace(ctx context.Context, client temporalclient.Client) { logger, _ := customlogger.GetZapLogger(ctx) @@ -158,8 +163,17 @@ func main() { initTemporalNamespace(ctx, tempClient) } - // Initialize Minio client - minioClient, err := miniox.NewMinioClientAndInitBucket(ctx, &config.Config.Minio, logger) + // Initialize MinIO client + retentionHandler := service.NewRetentionHandler() + minioClient, err := miniox.NewMinioClientAndInitBucket(ctx, miniox.ClientParams{ + Config: config.Config.Minio, + Logger: logger, + ExpiryRules: retentionHandler.ListExpiryRules(), + AppInfo: miniox.AppInfo{ + Name: serviceName, + Version: version, + }, + }) if err != nil { logger.Fatal("failed to create minio client", zap.Error(err)) } diff --git a/go.mod b/go.mod index 24c3db26..d33efcc3 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241211175103-4f1558f81c9c github.com/instill-ai/usage-client v0.3.0-alpha - github.com/instill-ai/x v0.6.0-alpha.0.20250212192855-6af31ff7cc27 + github.com/instill-ai/x v0.6.0-alpha.0.20250213104218-8000506aa455 github.com/jackc/pgx/v5 v5.6.0 github.com/knadh/koanf v1.5.0 github.com/lestrrat-go/jspointer v0.0.0-20181205001929-82fadba7561c diff --git a/go.sum b/go.sum index 3fef27d3..61c67691 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241211175103-4f1558f81c9c h1: github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241211175103-4f1558f81c9c/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY= github.com/instill-ai/usage-client v0.3.0-alpha h1:yY5eNn5zINqy8wpOogiNmrVYzJKnd1KMnMxlYBpr7Tk= github.com/instill-ai/usage-client v0.3.0-alpha/go.mod h1:8lvtZulkhQ7t8alttb2KkLKYoCp5u4oatzDbfFlEld0= -github.com/instill-ai/x v0.6.0-alpha.0.20250212192855-6af31ff7cc27 h1:5MwjakOj/G1iP7NwUBS7WCKvnAeI72mhgqq/abfuV+0= -github.com/instill-ai/x v0.6.0-alpha.0.20250212192855-6af31ff7cc27/go.mod h1:4oSOcDRtho+uLswiPvty5sF5OxiiprUh8KCOiFdKyPw= +github.com/instill-ai/x v0.6.0-alpha.0.20250213104218-8000506aa455 h1:hMkl7Csrasx0c8tMG/2cRF6CtA2W32DM9NGtmo051L4= +github.com/instill-ai/x v0.6.0-alpha.0.20250213104218-8000506aa455/go.mod h1:4oSOcDRtho+uLswiPvty5sF5OxiiprUh8KCOiFdKyPw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=