diff --git a/.dockerignore b/.dockerignore index 9c385f9e..a9d8d646 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,6 @@ schema.graphql **/*_gen.go **/prisma-* **/query-engine-* + +Dockerfile +*.dockerfile diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4372f1d0..af6cf62a 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -43,6 +43,47 @@ jobs: restore-keys: ${{ runner.os }}-go- key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - - name: integration + - name: test if: steps.changes.outputs.go == 'true' - run: docker build . -f test/integration/integration.dockerfile -t integration && docker run integration + run: docker build . --build-arg IMAGE="golang:1" -f test/integration/integration.dockerfile -t integration && docker run integration + + integration-alpine: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + go: + - '.github/workflows/**/*.yml' + - '**/*.go' + - '**/*.gotpl' + - '**/*.mod' + - '**/*.sum' + - '**/*.work' + + # fix for GitHub actions MacOS + - name: Setup docker + if: runner.os == 'macos' && steps.changes.outputs.go == 'true' + run: | + brew install docker + colima start + + # For testcontainers to find the Colima socket + # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + + - uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache + restore-keys: ${{ runner.os }}-go- + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: test on alpine + if: steps.changes.outputs.go == 'true' + run: docker build . --build-arg IMAGE="golang:1-alpine" -f test/integration/integration.dockerfile -t integration && docker run integration diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c73707c..0dcc487d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,4 +41,4 @@ jobs: - name: test if: steps.changes.outputs.go == 'true' - run: go test ./... -v + run: go test ./... -v -failfast diff --git a/binaries/binaries.go b/binaries/binaries.go index edba1b2d..b68584d8 100644 --- a/binaries/binaries.go +++ b/binaries/binaries.go @@ -8,18 +8,17 @@ import ( "os" "path" "path/filepath" - "time" "github.com/steebchen/prisma-client-go/binaries/platform" "github.com/steebchen/prisma-client-go/logger" ) // PrismaVersion is a hardcoded version of the Prisma CLI. -const PrismaVersion = "4.15.0" +const PrismaVersion = "4.16.0" // EngineVersion is a hardcoded version of the Prisma Engine. // The versions can be found under https://github.com/prisma/prisma-engines/commits/main -const EngineVersion = "70a9adfd11f836696eca398396544fe07a8d8edb" +const EngineVersion = "b20ead4d3ab9e78ac112966e242ded703f4a052c" // PrismaURL points to an S3 bucket URL where the CLI binaries are stored. var PrismaURL = "https://packaged-cli.prisma.sh/%s-%s-%s-%s.gz" @@ -86,26 +85,22 @@ func GlobalCacheDir() string { return path.Join(cache, baseDirName, "cli", PrismaVersion) } -func FetchEngine(toDir string, engineName string, binaryPlatformName string) error { - logger.Debug.Printf("checking %s...", engineName) +func FetchEngine(dir string, engineName string, binaryName string) error { + logger.Debug.Printf("checking %s %s...", engineName, binaryName) - to := platform.CheckForExtension(binaryPlatformName, path.Join(toDir, EngineVersion, fmt.Sprintf("prisma-%s-%s", engineName, binaryPlatformName))) - - binaryPlatformRemoteName := binaryPlatformName - if binaryPlatformRemoteName == "linux" { - binaryPlatformRemoteName = "linux-static-x64" - } - url := platform.CheckForExtension(binaryPlatformName, fmt.Sprintf(EngineURL, EngineVersion, binaryPlatformRemoteName, engineName)) - - logger.Debug.Printf("download url %s", url) + to := GetEnginePath(dir, engineName, binaryName) if _, err := os.Stat(to); !os.IsNotExist(err) { - logger.Debug.Printf("%s is cached", to) + logger.Debug.Printf("%s is cached at %s", engineName, to) return nil } + url := platform.CheckForExtension(binaryName, fmt.Sprintf(EngineURL, EngineVersion, binaryName, engineName)) + logger.Debug.Printf("%s is missing, downloading...", engineName) + logger.Debug.Printf("downloading %s from %s to %s", engineName, url, to) + if err := download(url, to); err != nil { return fmt.Errorf("could not download %s to %s: %w", url, to, err) } @@ -130,7 +125,7 @@ func FetchNative(toDir string) error { } for _, e := range Engines { - if _, err := DownloadEngine(e.Name, toDir); err != nil { + if err := FetchEngine(toDir, e.Name, platform.BinaryPlatformNameStatic()); err != nil { return fmt.Errorf("could not download engines: %w", err) } } @@ -160,38 +155,8 @@ func DownloadCLI(toDir string) error { return nil } -func GetEnginePath(dir, engine, binaryName string) string { - return platform.CheckForExtension(binaryName, path.Join(dir, EngineVersion, fmt.Sprintf("prisma-%s-%s", engine, binaryName))) -} - -func DownloadEngine(name string, toDir string) (file string, err error) { - binaryName := platform.BinaryPlatformName() - - logger.Debug.Printf("checking %s...", name) - - to := platform.CheckForExtension(binaryName, path.Join(toDir, EngineVersion, fmt.Sprintf("prisma-%s-%s", name, binaryName))) - - url := platform.CheckForExtension(binaryName, fmt.Sprintf(EngineURL, EngineVersion, binaryName, name)) - - logger.Debug.Printf("download url %s", url) - - if _, err := os.Stat(to); !os.IsNotExist(err) { - logger.Debug.Printf("%s is cached", to) - return to, nil - } - - logger.Debug.Printf("%s is missing, downloading...", name) - - startDownload := time.Now() - if err := download(url, to); err != nil { - return "", fmt.Errorf("could not download %s to %s: %w", url, to, err) - } - - logger.Debug.Printf("%s engine download took %s", name, time.Since(startDownload)) - - logger.Debug.Printf("%s done", name) - - return to, nil +func GetEnginePath(dir, engineName, binaryName string) string { + return platform.CheckForExtension(binaryName, path.Join(dir, EngineVersion, fmt.Sprintf("prisma-%s-%s", engineName, binaryName))) } func download(url string, to string) error { diff --git a/binaries/bindata/bindata.go b/binaries/bindata/bindata.go index 2bd90b52..082fac0c 100644 --- a/binaries/bindata/bindata.go +++ b/binaries/bindata/bindata.go @@ -32,27 +32,11 @@ func WriteFile(name, pkg, platform, from, to string) error { } func writeHeader(w io.Writer, pkg, name, platform string) error { - var constraints string - if platform == "linux" { - if name == "linux" { - // TODO dynamically construct these with allTargets in run.go - // TODO only include these for engines, not for the CLI - constraints = `// +build !debian_openssl_1_0_x -// +build !debian_openssl_1_1_x -// +build !rhel_openssl_1_0_x -// +build !rhel_openssl_1_1_x` - constraints += "\n" - } else { - constraints = "// +build linux\n" - } - } - _, err := fmt.Fprintf(w, `// Code generated by Prisma Client Go. DO NOT EDIT. +//go:build !codeanalysis && !prisma_ignore && %s +// +build !codeanalysis,!prisma_ignore,%s + //nolint -// +build !codeanalysis -// +build %s -// +build !prisma_ignore -%s package %s import ( @@ -62,7 +46,7 @@ import ( func init() { unpack.Unpack(data, "%s", "%s") } -`, name, constraints, pkg, name, binaries.EngineVersion) +`, platform, platform, pkg, name, binaries.EngineVersion) return err } diff --git a/binaries/platform/platform.go b/binaries/platform/platform.go index 2600ea47..4249d32e 100644 --- a/binaries/platform/platform.go +++ b/binaries/platform/platform.go @@ -12,9 +12,9 @@ import ( var binaryNameWithSSLCache string -// BinaryPlatformName returns the name of the prisma binary which should be used, -// for example "darwin" or "linux-openssl-1.1.x" -func BinaryPlatformName() string { +// BinaryPlatformNameDynamic returns the name of the prisma binary which should be used, +// for example "darwin" or "linux-openssl-1.1.x". This can include dynamically linked binaries. +func BinaryPlatformNameDynamic() string { if binaryNameWithSSLCache != "" { return binaryNameWithSSLCache } @@ -34,10 +34,6 @@ func BinaryPlatformName() string { distro := getLinuxDistro() - if distro == "alpine" { - return fmt.Sprintf("linux-static-%s", arch) - } - ssl := getOpenSSL() name := fmt.Sprintf("%s-openssl-%s", distro, ssl) @@ -47,6 +43,25 @@ func BinaryPlatformName() string { return name } +// BinaryPlatformNameStatic returns the name of the prisma binary which should be used, +// for example "darwin" or "linux-static-x64". This only includes statically linked binaries. +func BinaryPlatformNameStatic() string { + platform := Name() + arch := Arch() + + // other supported platforms are darwin and windows + if platform != "linux" { + // special case for darwin arm64 + if platform == "darwin" && arch == "arm64" { + return "darwin-arm64" + } + // otherwise, return `darwin` or `windows` + return platform + } + + return fmt.Sprintf("linux-static-%s", arch) +} + // Name returns the platform name func Name() string { return runtime.GOOS diff --git a/binaries/unpack/unpack.go b/binaries/unpack/unpack.go index 15a2d0cc..9b7e512a 100644 --- a/binaries/unpack/unpack.go +++ b/binaries/unpack/unpack.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path" + "strings" "time" "github.com/steebchen/prisma-client-go/binaries" @@ -13,10 +14,14 @@ import ( // TODO check checksum after expanding file +const FileEnv = "PRISMA_INTERNAL_QUERY_ENGINE_PATH" + // noinspection GoUnusedExportedFunction func Unpack(data []byte, name string, version string) { start := time.Now() + name = strings.ReplaceAll(name, "_", "-") + filename := fmt.Sprintf("prisma-query-engine-%s", name) // TODO check if dev env/dev binary in ~/.prisma @@ -31,7 +36,7 @@ func Unpack(data []byte, name string, version string) { } if _, err := os.Stat(file); err == nil { - logger.Debug.Printf("query engine exists, not unpacking. %s", time.Since(start)) + logger.Debug.Printf("query engine exists, not unpacking. %s. at %s", time.Since(start), file) return } @@ -53,4 +58,8 @@ func Unpack(data []byte, name string, version string) { } logger.Debug.Printf("unpacked at %s in %s", file, time.Since(start)) + + if err := os.Setenv(FileEnv, file); err != nil { + panic(err) + } } diff --git a/cli/cli.go b/cli/cli.go index 4ea17f47..e40d82e5 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -28,7 +28,7 @@ func Run(arguments []string, output bool) error { logger.Debug.Printf("running %s %+v", path.Join(dir, prisma), arguments) cmd := exec.Command(path.Join(dir, prisma), arguments...) //nolint:gosec - binaryName := platform.CheckForExtension(platform.Name(), platform.BinaryPlatformName()) + binaryName := platform.CheckForExtension(platform.Name(), platform.BinaryPlatformNameStatic()) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, "PRISMA_HIDE_UPDATE_MESSAGE=true") diff --git a/engine/lifecycle.go b/engine/lifecycle.go index b6bcf9b2..d9191463 100644 --- a/engine/lifecycle.go +++ b/engine/lifecycle.go @@ -14,6 +14,7 @@ import ( "github.com/steebchen/prisma-client-go/binaries" "github.com/steebchen/prisma-client-go/binaries/platform" + "github.com/steebchen/prisma-client-go/binaries/unpack" "github.com/steebchen/prisma-client-go/logger" ) @@ -69,10 +70,12 @@ func (e *QueryEngine) Disconnect() error { func (e *QueryEngine) ensure() (string, error) { ensureEngine := time.Now() - binariesPath := binaries.GlobalUnpackDir(binaries.EngineVersion) + unpackPath := binaries.GlobalUnpackDir(binaries.EngineVersion) + cachePath := binaries.GlobalCacheDir() + // check for darwin/windows/linux first - binaryName := platform.CheckForExtension(platform.Name(), platform.Name()) - exactBinaryName := platform.CheckForExtension(platform.Name(), platform.BinaryPlatformName()) + binaryName := platform.CheckForExtension(platform.Name(), platform.BinaryPlatformNameStatic()) + exactBinaryName := platform.CheckForExtension(platform.Name(), platform.BinaryPlatformNameDynamic()) var file string // forceVersion saves whether a version check should be done, which should be disabled @@ -80,13 +83,16 @@ func (e *QueryEngine) ensure() (string, error) { forceVersion := true name := "prisma-query-engine-" - localPath := path.Join("./", name+binaryName) - localExactPath := path.Join("./", name+exactBinaryName) - globalPath := path.Join(binariesPath, name+binaryName) - globalExactPath := path.Join(binariesPath, name+exactBinaryName) + localStatic := path.Join("./", name+binaryName) + localExact := path.Join("./", name+exactBinaryName) + globalUnpackStatic := path.Join(unpackPath, name+binaryName) + globalUnpackExact := path.Join(unpackPath, name+exactBinaryName) + cacheStatic := path.Join(cachePath, binaries.EngineVersion, name+binaryName) + cacheExact := path.Join(cachePath, binaries.EngineVersion, name+exactBinaryName) - logger.Debug.Printf("expecting local query engine `%s` or `%s`", localPath, localExactPath) - logger.Debug.Printf("expecting global query engine `%s` or `%s`", globalPath, globalExactPath) + logger.Debug.Printf("checking for local query engine `%s` or `%s`", localStatic, localExact) + logger.Debug.Printf("checking for global query engine `%s` or `%s`", globalUnpackStatic, globalUnpackExact) + logger.Debug.Printf("checking for cached query engine `%s` or `%s`", cacheStatic, cacheExact) // TODO write tests for all cases @@ -102,19 +108,34 @@ func (e *QueryEngine) ensure() (string, error) { file = prismaQueryEngineBinary forceVersion = false } else { - if info, err := os.Stat(localExactPath); err == nil { - file = localExactPath - logger.Debug.Printf("exact query engine found in working directory: %s %+v", file, info) - } else if info, err = os.Stat(localPath); err == nil { - file = localPath - logger.Debug.Printf("query engine found in working directory: %s %+v", file, info) + if qe := os.Getenv(unpack.FileEnv); qe != "" { + logger.Debug.Printf("using unpacked file env %s %s", unpack.FileEnv, qe) + + if info, err := os.Stat(qe); err == nil { + file = qe + logger.Debug.Printf("exact query engine found in working directory: %s %+v", file, info) + } else { + return "", fmt.Errorf("prisma query engine was expected at %s via FileEnv but was not found", qe) + } } - if info, err := os.Stat(globalExactPath); err == nil { - file = globalExactPath + if info, err := os.Stat(localExact); err == nil { + file = localExact + logger.Debug.Printf("exact query engine found in working directory: %s %+v", file, info) + } else if info, err = os.Stat(localStatic); err == nil { + file = localStatic + logger.Debug.Printf("query engine found in working directory: %s %+v", file, info) + } else if info, err = os.Stat(cacheExact); err == nil { + file = cacheExact + logger.Debug.Printf("query engine found in cache path: %s %+v", file, info) + } else if info, err = os.Stat(cacheStatic); err == nil { + file = cacheStatic + logger.Debug.Printf("exact query engine found in cache path: %s %+v", file, info) + } else if info, err = os.Stat(globalUnpackExact); err == nil { + file = globalUnpackExact logger.Debug.Printf("query engine found in global path: %s %+v", file, info) - } else if info, err = os.Stat(globalPath); err == nil { - file = globalPath + } else if info, err = os.Stat(globalUnpackStatic); err == nil { + file = globalUnpackStatic logger.Debug.Printf("exact query engine found in global path: %s %+v", file, info) } } diff --git a/generator/generator.go b/generator/generator.go index 4c900e7c..0b97bce4 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -94,6 +94,4 @@ type BinaryPaths struct { MigrationEngine map[string]string `json:"migrationEngine"` // key target, value path // QueryEngine (optional) QueryEngine map[string]string `json:"queryEngine"` - // IntrospectionEngine (optional) - IntrospectionEngine map[string]string `json:"introspectionEngine"` } diff --git a/generator/run.go b/generator/run.go index 2c36c111..19870e17 100644 --- a/generator/run.go +++ b/generator/run.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "text/template" @@ -24,6 +25,16 @@ func addDefaults(input *Root) { if input.Generator.Config.Package == "" { input.Generator.Config.Package = DefaultPackageName } + + if binaryTargets := os.Getenv("PRISMA_CLI_BINARY_TARGETS"); binaryTargets != "" { + s := strings.Split(binaryTargets, ",") + var targets []BinaryTarget + for _, t := range s { + targets = append(targets, BinaryTarget{Value: t}) + } + input.Generator.BinaryTargets = targets + logger.Debug.Printf("overriding binary targets: %+v", targets) + } } // Run invokes the generator, which builds the templates and writes to the specified output file. @@ -143,20 +154,34 @@ func generateBinaries(input *Root) error { } var targets []string + var isNonLinux bool + + logger.Debug.Printf("defined binary targets: %v", input.Generator.BinaryTargets) for _, target := range input.Generator.BinaryTargets { targets = append(targets, target.Value) + if target.Value == "darwin" || target.Value == "windows" { + isNonLinux = true + } } - targets = add(targets, "native") - targets = add(targets, "linux") + // add native by default if native binary is darwin or linux + // this prevents conflicts when building on linux + if isNonLinux || len(targets) == 0 { + targets = add(targets, "native") + } + + logger.Debug.Printf("final binary targets: %v", targets) // TODO refactor for _, name := range targets { if name == "native" { - name = platform.BinaryPlatformName() + name = platform.BinaryPlatformNameStatic() + logger.Debug.Printf("swapping 'native' binary target with '%s'", name) } + name = TransformBinaryTarget(name) + // first, ensure they are actually downloaded if err := binaries.FetchEngine(binaries.GlobalCacheDir(), "query-engine", name); err != nil { return fmt.Errorf("failed fetching binaries: %w", err) @@ -172,22 +197,24 @@ func generateBinaries(input *Root) error { func generateQueryEngineFiles(binaryTargets []string, pkg, outputDir string) error { for _, name := range binaryTargets { + pt := runtime.GOOS + if strings.Contains(name, "debian") || strings.Contains(name, "rhel") || strings.Contains(name, "musl") { + pt = "linux" + } + if name == "native" { - name = platform.BinaryPlatformName() + name = platform.BinaryPlatformNameStatic() } - enginePath := binaries.GetEnginePath(binaries.GlobalCacheDir(), "query-engine", name) + name = TransformBinaryTarget(name) - pt := name - if strings.Contains(name, "debian") || strings.Contains(name, "rhel") { - pt = "linux" - } + enginePath := binaries.GetEnginePath(binaries.GlobalCacheDir(), "query-engine", name) filename := fmt.Sprintf("query-engine-%s_gen.go", name) to := path.Join(outputDir, filename) // TODO check if already exists, but make sure version matches - if err := bindata.WriteFile(strings.ReplaceAll(name, "-", "_"), pkg, pt, enginePath, to); err != nil { + if err := bindata.WriteFile(name, pkg, pt, enginePath, to); err != nil { return fmt.Errorf("generate write go file: %w", err) } @@ -205,3 +232,12 @@ func add(list []string, item string) []string { } return list } + +func TransformBinaryTarget(name string) string { + // TODO this is a temp fix as the exact alpine libraries are not working + if name == "linux" || strings.Contains(name, "musl") { + name = "linux-static-" + platform.Arch() + logger.Debug.Printf("overriding binary name with '%s' due to linux or musl", name) + } + return name +} diff --git a/test/integration/integration.dockerfile b/test/integration/integration.dockerfile index 2d6b4c27..299e3d38 100644 --- a/test/integration/integration.dockerfile +++ b/test/integration/integration.dockerfile @@ -1,30 +1,34 @@ -FROM golang:1.20.5 as build +ARG IMAGE +FROM $IMAGE as build WORKDIR /app +RUN go version + ENV PRISMA_CLIENT_GO_LOG=info ENV DEBUG=* +COPY go.mod go.sum ./ +RUN go mod download -x + COPY . ./ WORKDIR /app/test/integration -RUN go mod download -x - RUN go run github.com/steebchen/prisma-client-go db push --schema schemax.prisma # build the integration binary with all dependencies RUN go build -o /app/main . # start a new stage to test if the runtime fetching works -FROM golang:1.20.5 +FROM $IMAGE WORKDIR /app COPY --from=build /app/main /app/main COPY --from=build /app/test/integration/dev.db /app/dev.db -ENV PRISMA_CLIENT_GO_LOG=info +ENV PRISMA_CLIENT_GO_LOG=debug ENV DEBUG=* CMD ["/app/main"] diff --git a/test/projects/binaries/binaries_test.go b/test/projects/binaries/binaries_test.go index de6a4e4c..64ecea56 100644 --- a/test/projects/binaries/binaries_test.go +++ b/test/projects/binaries/binaries_test.go @@ -5,17 +5,12 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/steebchen/prisma-client-go/binaries/platform" ) func TestBinaries(t *testing.T) { t.Parallel() - // this test only verifies that specifying `binaryTargets` downloaded the separate file into the directory - _, err := os.Stat("./query-engine-" + platform.BinaryPlatformName() + "_gen.go") - assert.Equal(t, err, nil) - - _, err = os.Stat("./query-engine-debian-openssl-1.1.x_gen.go") + // check if specified engine in schema.prisma exists + _, err := os.Stat("./query-engine-debian-openssl-1.1.x_gen.go") assert.Equal(t, err, nil) } diff --git a/test/projects/binaries/schema.prisma b/test/projects/binaries/schema.prisma index b82b6593..5e1fd442 100644 --- a/test/projects/binaries/schema.prisma +++ b/test/projects/binaries/schema.prisma @@ -8,7 +8,7 @@ generator db { output = "." disableGitignore = true package = "binaries" - binaryTargets = ["native", "debian-openssl-1.1.x"] + binaryTargets = ["debian-openssl-1.1.x"] } model User {