chore(wip): refactor internal storage types to use proto types

.github/workflows/test.yml

@@ -19,6 +19,7 @@ jobs:
     name: "Tests (Go)"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         database: ["mysql", "postgres", "cockroachdb", "sqlite", "libsql"]
     steps:

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/gobwas/glob v0.2.3
 	github.com/gofrs/uuid v4.4.0+incompatible
+	github.com/gogo/protobuf v1.3.2
 	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-github/v32 v32.1.0
@@ -127,7 +128,6 @@ require (
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect

go.work.sum

@@ -60,7 +60,6 @@ cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOV
 cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
 cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
 cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
-cloud.google.com/go/compute v1.23.2/go.mod h1:JJ0atRC0J/oWYiiVBmsSsrRnh92DhZPG4hFDcR04Rns=
 cloud.google.com/go/contactcenterinsights v1.11.0/go.mod h1:hutBdImE4XNZ1NV4vbPJKSFOnQruhC5Lj9bZqWMTKiU=
 cloud.google.com/go/contactcenterinsights v1.11.2/go.mod h1:A9PIR5ov5cRcd28KlDbmmXE8Aay+Gccer2h4wzkYFso=
 cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4=
@@ -648,7 +647,6 @@ github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMd
 github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
 github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -769,6 +767,7 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=

internal/cache/cache.go

@@ -18,5 +18,5 @@ type Cacher interface {
 }
 
 func Key(k string) string {
-	return fmt.Sprintf("flipt:%x", md5.Sum([]byte(k)))
+	return fmt.Sprintf("flipt:v1:%x", md5.Sum([]byte(k)))
 }

internal/cache/cache_test.go

@@ -0,0 +1,26 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestKey(t *testing.T) {
+	type args struct {
+		k string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"empty", args{k: ""}, "flipt:v1:d41d8cd98f00b204e9800998ecf8427e"},
+		{"non-empty", args{k: "foo"}, "flipt:v1:acbd18db4cc2f85cedef654fccc4a4d8"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, Key(tt.args.k))
+		})
+	}
+}

internal/cmd/grpc.go

@@ -26,6 +26,7 @@ import (
 	"go.flipt.io/flipt/internal/server/audit/webhook"
 	"go.flipt.io/flipt/internal/server/auth"
 	"go.flipt.io/flipt/internal/server/evaluation"
+	evaluationdata "go.flipt.io/flipt/internal/server/evaluation/data"
 	"go.flipt.io/flipt/internal/server/metadata"
 	middlewaregrpc "go.flipt.io/flipt/internal/server/middleware/grpc"
 	"go.flipt.io/flipt/internal/storage"
@@ -282,10 +283,11 @@ func NewGRPCServer(
 	}
 
 	var (
-		fliptsrv  = fliptserver.New(logger, store)
-		metasrv   = metadata.New(cfg, info)
-		evalsrv   = evaluation.New(logger, store)
-		healthsrv = health.NewServer()
+		fliptsrv    = fliptserver.New(logger, store)
+		metasrv     = metadata.New(cfg, info)
+		evalsrv     = evaluation.New(logger, store)
+		evalDataSrv = evaluationdata.New(logger, store)
+		healthsrv   = health.NewServer()
 	)
 
 	var (
@@ -340,6 +342,7 @@ func NewGRPCServer(
 	register.Add(fliptsrv)
 	register.Add(metasrv)
 	register.Add(evalsrv)
+	register.Add(evalDataSrv)
 
 	// forward internal gRPC logging to zap
 	grpcLogLevel, err := zapcore.ParseLevel(cfg.Log.GRPCLevel)

internal/cmd/http.go

@@ -56,10 +56,11 @@ func NewHTTPServer(
 		}
 		isConsole = cfg.Log.Encoding == config.LogEncodingConsole
 
-		r           = chi.NewRouter()
-		api         = gateway.NewGatewayServeMux(logger)
-		evaluateAPI = gateway.NewGatewayServeMux(logger)
-		httpPort    = cfg.Server.HTTPPort
+		r               = chi.NewRouter()
+		api             = gateway.NewGatewayServeMux(logger)
+		evaluateAPI     = gateway.NewGatewayServeMux(logger)
+		evaluateDataAPI = gateway.NewGatewayServeMux(logger)
+		httpPort        = cfg.Server.HTTPPort
 	)
 
 	if cfg.Server.Protocol == config.HTTPS {
@@ -74,6 +75,10 @@ func NewHTTPServer(
 		return nil, fmt.Errorf("registering grpc gateway: %w", err)
 	}
 
+	if err := evaluation.RegisterDataServiceHandler(ctx, evaluateDataAPI, conn); err != nil {
+		return nil, fmt.Errorf("registering grpc gateway: %w", err)
+	}
+
 	if cfg.Cors.Enabled {
 		cors := cors.New(cors.Options{
 			AllowedOrigins:   cfg.Cors.AllowedOrigins,
@@ -140,6 +145,7 @@ func NewHTTPServer(
 
 		r.Mount("/api/v1", api)
 		r.Mount("/evaluate/v1", evaluateAPI)
+		r.Mount("/internal/v1", evaluateDataAPI)
 
 		// mount all authentication related HTTP components
 		// to the chi router.

internal/cmd/protoc-gen-go-flipt-sdk/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"slices"
 	"strings"
 
 	"google.golang.org/protobuf/compiler/protogen"
@@ -13,6 +14,8 @@ import (
 const (
 	importPath  = "go.flipt.io/flipt/sdk/go"
 	emptyImport = "google.golang.org/protobuf/types/known/emptypb"
+
+	ignoreDecl = "flipt:sdk:ignore"
 )
 
 func main() {
@@ -34,6 +37,8 @@ func main() {
 				continue
 			}
 
+			f.Services = slices.DeleteFunc(f.Services, shouldIgnoreService)
+
 			generateSubSDK(gen, f)
 		}
 
@@ -261,3 +266,11 @@ func New(t Transport, opts ...Option) SDK {
 
     return sdk
 }`
+
+func shouldIgnoreService(srv *protogen.Service) bool {
+	if srv.Comments.Leading != "" {
+		leading := strings.TrimPrefix(string(srv.Comments.Leading), "//")
+		return strings.TrimSpace(leading) == ignoreDecl
+	}
+	return false
+}

internal/ext/exporter.go

@@ -75,7 +75,7 @@ func (e *Exporter) Export(ctx context.Context, w io.Writer) error {
 
 		intermediateNamespaces := make([]string, 0)
 
-		for batch := int32(0); remaining; batch++ {
+		for remaining {
 			resp, err := e.store.ListNamespaces(ctx, &flipt.ListNamespaceRequest{
 				PageToken: nextPage,
 				Limit:     batchSize,
@@ -246,7 +246,7 @@ func (e *Exporter) Export(ctx context.Context, w io.Writer) error {
 		nextPage = ""
 
 		// export segments/constraints in batches
-		for batch := int32(0); remaining; batch++ {
+		for remaining {
 			resp, err := e.store.ListSegments(
 				ctx,
 				&flipt.ListSegmentRequest{