diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3732c454c..817315f809 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,6 +81,7 @@ jobs: just errtrace just build ftl # Ensure all the prerequisites are built before we use goreleaser just build-language-plugins + just build-sqlc-gen-ftl goreleaser release --skip=validate env: GITHUB_TOKEN: ${{ github.token }} diff --git a/Justfile b/Justfile index 0291e89f87..26eff66a61 100644 --- a/Justfile +++ b/Justfile @@ -19,7 +19,7 @@ ZIP_DIRS := "go-runtime/compile/build-template " + \ CONSOLE_ROOT := "frontend/console" FRONTEND_OUT := CONSOLE_ROOT + "/dist/index.html" EXTENSION_OUT := "frontend/vscode/dist/extension.js" -SQLC_GEN_FTL_OUT := "sqlc-gen-ftl/target/wasm32-wasip1/release/sqlc-gen-ftl.wasm" +SQLC_GEN_FTL_OUT := "internal/sqlc/resources/sqlc-gen-ftl.wasm" PROTOS_IN := "common/protos backend/protos" PROTOS_OUT := "backend/protos/xyz/block/ftl/console/v1/console.pb.go " + \ "backend/protos/xyz/block/ftl//v1/ftl.pb.go " + \ @@ -194,14 +194,18 @@ build-extension: pnpm-install @mk {{EXTENSION_OUT}} : frontend/vscode/src frontend/vscode/package.json -- "cd frontend/vscode && rm -f ftl-*.vsix && pnpm run compile" # Build the sqlc-ftl-gen plugin, used to generate FTL schema from SQL -build-sqlc-gen-ftl: build-rust-protos +build-sqlc-gen-ftl: build-rust-protos download-sqlc @mk {{SQLC_GEN_FTL_OUT}} : sqlc-gen-ftl/src -- \ - "cd sqlc-gen-ftl && \ - cargo build --target wasm32-wasip1 --release" + "cargo build --manifest-path sqlc-gen-ftl/Cargo.toml --target wasm32-wasip1 --release && \ + cp sqlc-gen-ftl/target/wasm32-wasip1/release/sqlc-gen-ftl.wasm internal/sqlc/resources" test-sqlc-gen-ftl: @cargo test --manifest-path sqlc-gen-ftl/Cargo.toml --features ci --test sqlc_gen_ftl_test -- --nocapture +# Download SQLC binaries, embedded in the FTL binary as resources +download-sqlc: + @bash scripts/provide-sqlc-resources + # Generate Rust protos build-rust-protos: @mk sqlc-gen-ftl/src/protos : backend/protos -- \ diff --git a/internal/sqlc/Dockerfile b/internal/sqlc/Dockerfile new file mode 100644 index 0000000000..6e08414d33 --- /dev/null +++ b/internal/sqlc/Dockerfile @@ -0,0 +1,13 @@ +FROM alpine:latest + +RUN apk add --no-cache bash curl tar + +WORKDIR /app + +COPY scripts/download-sqlc /app/download-sqlc + +RUN chmod +x /app/download-sqlc + +ENV SQLC_VERSION="" + +CMD ["/app/download-sqlc"] diff --git a/internal/sqlc/embed.go b/internal/sqlc/embed.go new file mode 100644 index 0000000000..45eb094d72 --- /dev/null +++ b/internal/sqlc/embed.go @@ -0,0 +1,36 @@ +package sqlc + +import ( + "embed" + "fmt" + "io" + "os" + "path/filepath" +) + +//go:embed resources/* +var embeddedResources embed.FS + +func extractEmbeddedFile(resourceName, destPath string) error { + if err := os.MkdirAll(filepath.Dir(destPath), 0750); err != nil { + return fmt.Errorf("failed to create destination directory: %w", err) + } + + srcFile, err := embeddedResources.Open(filepath.Join("resources", resourceName)) + if err != nil { + return fmt.Errorf("failed to open embedded resource %s: %w", resourceName, err) + } + defer srcFile.Close() + + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("failed to create destination file: %w", err) + } + defer destFile.Close() + + if _, err := io.Copy(destFile, srcFile); err != nil { + return fmt.Errorf("failed to copy file contents: %w", err) + } + + return nil +} diff --git a/internal/sqlc/resources/darwin_amd64/sqlc b/internal/sqlc/resources/darwin_amd64/sqlc new file mode 100755 index 0000000000..a5e3f9e97c Binary files /dev/null and b/internal/sqlc/resources/darwin_amd64/sqlc differ diff --git a/internal/sqlc/resources/darwin_arm64/sqlc b/internal/sqlc/resources/darwin_arm64/sqlc new file mode 100755 index 0000000000..a7c316e9e7 Binary files /dev/null and b/internal/sqlc/resources/darwin_arm64/sqlc differ diff --git a/internal/sqlc/resources/linux_amd64/sqlc b/internal/sqlc/resources/linux_amd64/sqlc new file mode 100755 index 0000000000..bb6f5cdee2 Binary files /dev/null and b/internal/sqlc/resources/linux_amd64/sqlc differ diff --git a/internal/sqlc/resources/linux_arm64/sqlc b/internal/sqlc/resources/linux_arm64/sqlc new file mode 100755 index 0000000000..0334b59910 Binary files /dev/null and b/internal/sqlc/resources/linux_arm64/sqlc differ diff --git a/internal/sqlc/resources/sqlc-gen-ftl.wasm b/internal/sqlc/resources/sqlc-gen-ftl.wasm new file mode 100755 index 0000000000..8832cc49ce Binary files /dev/null and b/internal/sqlc/resources/sqlc-gen-ftl.wasm differ diff --git a/internal/sqlc/sqlc.go b/internal/sqlc/sqlc.go new file mode 100644 index 0000000000..6a994a6d5c --- /dev/null +++ b/internal/sqlc/sqlc.go @@ -0,0 +1,264 @@ +package sqlc + +import ( + "archive/zip" + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "runtime" + "sync" + "syscall" + + "github.com/block/ftl/internal" + "github.com/block/ftl/internal/exec" + "github.com/block/ftl/internal/log" + "github.com/block/ftl/internal/moduleconfig" +) + +var ( + binaryCache struct { + path string + tmpDir string + mu sync.Mutex + } + pluginCache struct { + path string + sha256 string + tmpDir string + mu sync.Mutex + } + initOnce sync.Once + sqlcBinaryName = "sqlc" +) + +// maintain a cache of the SQLC binary/WASM plugin per session of the FTL CLI +func init() { + initOnce.Do(func() { + cleanupFn := func() { + binaryCache.mu.Lock() + if binaryCache.tmpDir != "" { + if err := os.RemoveAll(binaryCache.tmpDir); err != nil { + fmt.Fprintf(os.Stderr, "failed to cleanup SQLC binary cache: %v\n", err) + } + } + binaryCache.mu.Unlock() + + pluginCache.mu.Lock() + if pluginCache.tmpDir != "" { + if err := os.RemoveAll(pluginCache.tmpDir); err != nil { + fmt.Fprintf(os.Stderr, "failed to cleanup SQLC plugin cache: %v\n", err) + } + } + pluginCache.mu.Unlock() + } + defer cleanupFn() + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + cleanupFn() + os.Exit(1) + }() + }) +} + +type ConfigContext struct { + Dir string + Module string + Engine string + SchemaDir string + QueriesDir string + OutDir string + Plugin WASMPlugin +} + +func (c ConfigContext) scaffoldFile() error { + err := internal.ScaffoldZip(sqlcTemplateFiles(), c.OutDir, c) + if err != nil { + return fmt.Errorf("failed to scaffold SQLC config file: %w", err) + } + return nil +} + +func (c ConfigContext) getPath() (string, error) { + relPath, err := filepath.Rel(c.Dir, c.OutDir) + if err != nil { + return "", fmt.Errorf("failed to get relative path: %w", err) + } + return filepath.Join(relPath, "sqlc.yml"), nil +} + +type WASMPlugin struct { + URL string + SHA256 string +} + +func Generate(ctx context.Context, mc moduleconfig.ModuleConfig) error { + cfg, err := newConfigContext(ctx, mc) + if err != nil { + return fmt.Errorf("failed to create SQLC config: %w", err) + } + + if err := cfg.scaffoldFile(); err != nil { + return err + } + + binaryPath, err := getSQLCBinary(ctx) + if err != nil { + return err + } + + cfgPath, err := cfg.getPath() + if err != nil { + return err + } + + cmd := exec.Command(ctx, log.Info, cfg.Dir, binaryPath, "generate", "--file", cfgPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("error executing sqlc: %w", err) + } + + return nil +} + +func newConfigContext(ctx context.Context, mc moduleconfig.ModuleConfig) (ConfigContext, error) { + deployDir := filepath.Clean(filepath.Join(mc.Dir, mc.DeployDir)) + schemaDir := filepath.Clean(filepath.Join(mc.Dir, mc.SQLMigrationDirectory)) + relSchemaDir, err := filepath.Rel(deployDir, schemaDir) + if err != nil { + return ConfigContext{}, fmt.Errorf("failed to get relative schema directory: %w", err) + } + plugin, err := getWASMPlugin(ctx) + if err != nil { + return ConfigContext{}, err + } + return ConfigContext{ + Dir: deployDir, + Module: mc.Module, + Engine: "mysql", + SchemaDir: relSchemaDir, + QueriesDir: filepath.Join(relSchemaDir, "queries"), + OutDir: deployDir, + Plugin: plugin, + }, nil +} + +func getSQLCBinary(ctx context.Context) (string, error) { + logger := log.FromContext(ctx) + binaryCache.mu.Lock() + defer binaryCache.mu.Unlock() + + if binaryCache.path != "" { + if _, err := os.Stat(binaryCache.path); err == nil { + return binaryCache.path, nil + } + logger.Warnf("cached SQLC binary no longer exists, recreating") + binaryCache.path = "" + if binaryCache.tmpDir != "" { + _ = os.RemoveAll(binaryCache.tmpDir) + } + } + + tmpDir, err := os.MkdirTemp("", "ftl-sqlc-*") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + logger.Debugf("created new SQLC binary cache in %s", tmpDir) + + binaryDirName := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) + binaryPath := filepath.Join(binaryDirName, sqlcBinaryName) + extractPath := filepath.Join(tmpDir, sqlcBinaryName) + if err := extractEmbeddedFile(binaryPath, extractPath); err != nil { + os.RemoveAll(tmpDir) + return "", fmt.Errorf("failed to extract SQLC binary: %w", err) + } + + if err := os.Chmod(extractPath, 0750); err != nil { //nolint:gosec + os.RemoveAll(tmpDir) + return "", fmt.Errorf("failed to make binary executable: %w", err) + } + + // verify binary is executable + verifyCmd := exec.Command(ctx, log.Debug, filepath.Dir(extractPath), extractPath, "version") + if err := verifyCmd.Run(); err != nil { + os.RemoveAll(tmpDir) + return "", fmt.Errorf("extracted SQLC binary verification failed: %w", err) + } + + binaryCache.path = extractPath + binaryCache.tmpDir = tmpDir + logger.Debugf("successfully cached new SQLC binary") + return extractPath, nil +} + +func getWASMPlugin(ctx context.Context) (WASMPlugin, error) { + logger := log.FromContext(ctx) + pluginCache.mu.Lock() + defer pluginCache.mu.Unlock() + + if pluginCache.path != "" { + if _, err := os.Stat(pluginCache.path); err == nil { + return toWASMPlugin(pluginCache.path, pluginCache.sha256), nil + } + pluginCache.path = "" + pluginCache.sha256 = "" + if pluginCache.tmpDir != "" { + _ = os.RemoveAll(pluginCache.tmpDir) + } + } + + // create new plugin cache + tmpDir, err := os.MkdirTemp("", "ftl-sqlc-plugin-*") + if err != nil { + return WASMPlugin{}, fmt.Errorf("failed to create temp directory: %w", err) + } + logger.Debugf("created new SQLC WASM plugin cache in %s", tmpDir) + + pluginPath := filepath.Join(tmpDir, "sqlc-gen-ftl.wasm") + if err := extractEmbeddedFile("sqlc-gen-ftl.wasm", pluginPath); err != nil { + os.RemoveAll(tmpDir) + return WASMPlugin{}, err + } + + pluginSHA, err := computeSHA256(pluginPath) + if err != nil { + os.RemoveAll(tmpDir) + return WASMPlugin{}, err + } + + pluginCache.path = pluginPath + pluginCache.sha256 = pluginSHA + pluginCache.tmpDir = tmpDir + + return toWASMPlugin(pluginPath, pluginSHA), nil +} + +func toWASMPlugin(path, sha string) WASMPlugin { + return WASMPlugin{ + URL: fmt.Sprintf("file://%s", path), + SHA256: sha, + } +} + +func computeSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", fmt.Errorf("failed to compute hash: %w", err) + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func sqlcTemplateFiles() *zip.Reader { + return internal.ZipRelativeToCaller("template") +} diff --git a/internal/sqlc/sqlc_test.go b/internal/sqlc/sqlc_test.go new file mode 100644 index 0000000000..c3d110cbca --- /dev/null +++ b/internal/sqlc/sqlc_test.go @@ -0,0 +1,72 @@ +package sqlc + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/block/ftl/common/schema" + "github.com/block/ftl/internal/log" + "github.com/block/ftl/internal/moduleconfig" + "github.com/block/scaffolder" +) + +func TestGenerate(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + + tmpDir, err := os.MkdirTemp("", "sqlc-test-*") + assert.NoError(t, err) + defer os.RemoveAll(tmpDir) + + schemaDir := filepath.Join(tmpDir, "schema") + err = scaffolder.Scaffold("testdata", schemaDir, nil) + assert.NoError(t, err) + + mc := moduleconfig.ModuleConfig{ + Dir: tmpDir, + Module: "test", + SQLMigrationDirectory: "schema", + DeployDir: ".ftl", + } + + err = Generate(ctx, mc) + assert.NoError(t, err) + + actual, err := schema.ModuleFromProtoFile(filepath.Join(tmpDir, ".ftl", "queries.pb")) + assert.NoError(t, err, "failed to parse generated schema") + + expected := &schema.Module{ + Name: "test", + Decls: []schema.Decl{ + &schema.Data{ + Name: "CreateRequestQuery", + Fields: []*schema.Field{ + {Name: "data", Type: &schema.String{}}, + }, + }, + &schema.Data{ + Name: "GetRequestDataResult", + Fields: []*schema.Field{ + {Name: "data", Type: &schema.String{}}, + }, + }, + &schema.Verb{ + Name: "CreateRequest", + Request: &schema.Ref{Module: "test", Name: "CreateRequestQuery"}, + Response: &schema.Unit{}, + }, + &schema.Verb{ + Name: "GetRequestData", + Request: &schema.Unit{}, + Response: &schema.Array{Element: &schema.Ref{ + Module: "test", + Name: "GetRequestDataResult", + }}, + }, + }, + } + + assert.Equal(t, expected, actual) +} diff --git a/internal/sqlc/template/sqlc.yml.tmpl b/internal/sqlc/template/sqlc.yml.tmpl new file mode 100644 index 0000000000..bcf1491677 --- /dev/null +++ b/internal/sqlc/template/sqlc.yml.tmpl @@ -0,0 +1,15 @@ +version: '2' +plugins: +- name: ftl + wasm: + url: {{ .Plugin.URL }} + sha256: {{ .Plugin.SHA256 }} +sql: +- schema: {{ .SchemaDir }} + queries: {{ .QueriesDir }} + engine: {{ .Engine }} + codegen: + - out: . + plugin: ftl + options: + module: {{ .Module }} diff --git a/internal/sqlc/testdata/queries/queries.sql b/internal/sqlc/testdata/queries/queries.sql new file mode 100644 index 0000000000..705fc8fdc0 --- /dev/null +++ b/internal/sqlc/testdata/queries/queries.sql @@ -0,0 +1,5 @@ +-- name: GetRequestData :many +SELECT data FROM requests; + +-- name: CreateRequest :exec +INSERT INTO requests (data) VALUES (?); diff --git a/internal/sqlc/testdata/schema.sql b/internal/sqlc/testdata/schema.sql new file mode 100644 index 0000000000..3582db502b --- /dev/null +++ b/internal/sqlc/testdata/schema.sql @@ -0,0 +1,8 @@ +-- migrate:up +CREATE TABLE requests +( + data TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +-- migrate:down +DROP TABLE requests; diff --git a/scripts/download-sqlc b/scripts/download-sqlc new file mode 100755 index 0000000000..53bbf20377 --- /dev/null +++ b/scripts/download-sqlc @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Ensure SQLC_VERSION is set +if [ -z "${SQLC_VERSION:-}" ]; then + echo "SQLC_VERSION is not set." + exit 1 +fi + +PLATFORMS=("linux" "darwin") +ARCHITECTURES=("amd64" "arm64") + +# Clean and recreate the directory +rm -rf /app/build/* + +# Download binaries for each platform and architecture +for OS in "${PLATFORMS[@]}"; do + for ARCH in "${ARCHITECTURES[@]}"; do + SQLC_URL="https://github.com/sqlc-dev/sqlc/releases/download/v${SQLC_VERSION}/sqlc_${SQLC_VERSION}_${OS}_${ARCH}.tar.gz" + DEST_DIR="/app/build/${OS}_${ARCH}" + + if curl --output /dev/null --silent --head --fail "$SQLC_URL"; then + mkdir -p "$DEST_DIR" + echo "Downloading sqlc version ${SQLC_VERSION} for ${OS}_${ARCH}..." + + curl -L --fail --silent -o "${DEST_DIR}/sqlc.tar.gz" "$SQLC_URL" + cd "$DEST_DIR" + + tar xzf sqlc.tar.gz + rm sqlc.tar.gz + chmod +x sqlc + else + echo "No binary available for ${OS}_${ARCH}, skipping..." + fi + done +done diff --git a/scripts/provide-sqlc-resources b/scripts/provide-sqlc-resources new file mode 100644 index 0000000000..938791079a --- /dev/null +++ b/scripts/provide-sqlc-resources @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get the absolute path of the project root +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RESOURCES_DIR="${PROJECT_ROOT}/internal/sqlc/resources" + +# Determine the SQLC version using Hermit +if [ ! "${HERMIT_ENV:-}" ]; then + # shellcheck disable=SC1091 + . "${PROJECT_ROOT}/bin/activate-hermit" +fi + +SQLC_VERSION=$(hermit info sqlc | grep "Version:" | cut -d' ' -f2) + +if [ -z "$SQLC_VERSION" ]; then + echo "Failed to get SQLC version from Hermit." + exit 1 +fi + +# Create resources directory if it doesn't exist +mkdir -p "${RESOURCES_DIR}" + +# Run the Docker container with the determined variables +docker build -t sqlc-downloader -f internal/sqlc/Dockerfile . +docker run --rm \ + -v "${RESOURCES_DIR}:/app/build" \ + -e SQLC_VERSION="${SQLC_VERSION}" \ + sqlc-downloader + +# If on macOS, sign the binaries locally +if [[ "$(uname)" == "Darwin" ]] && [[ -z "${CI-}" ]]; then + echo "Detected macOS environment, signing binaries..." + for arch in amd64 arm64; do + echo "Signing darwin_${arch} binary..." + codesign --remove-signature "internal/sqlc/resources/darwin_${arch}/sqlc" 2>/dev/null || true + codesign -s - --force "internal/sqlc/resources/darwin_${arch}/sqlc" + done +fi