feat(sdk): introduce flag for disabling hashing artifacts + move hash…

… to rust (#645)

#### Motivation and context

The mdk codegen and typegraph mutually depends on each other (typegraph
needs a concrete mdk.wasm for hashing, and for the mdk.wasm to be built,
it needs type generation based on the typegraph)
Add a flag to enable processing a partial typegraph when using `meta
gen` (partial == no artifact resolution).

#### Migration notes

`get_file_hash` has been moved to core sdk (under the name `hash_file`)

### Checklist

- [x] The change come with new or modified tests
- [ ] Hard-to-understand functions have explanatory comments
- [ ] End-user documentation is updated to reflect the change

---------

Co-authored-by: Yohe-Am <[email protected]>

.github/workflows/tests.yml

@@ -175,58 +175,55 @@ jobs:
           cargo run --locked --package meta-cli -- --help
           cargo test --locked --package meta-cli
 
-  # test-docker:
-  #   needs: changes
-  #   if: ${{ needs.changes.outputs.typegate == 'true' }}
-  #   runs-on: ${{ matrix.runner }}
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       include:
-  #         - platform: linux/amd64
-  #           runner: ubuntu-latest
-  #         # - platform: linux/amd64
-  #         #   runner: custom-ubuntu-large
-  #         # FIXME: try macos-14 runner once all actions support it
-  #         # docker buildx action broken as of 2024-02-09
-  #         # TODO
-  #         # - platform: linux/arm64
-  #         #   runner: custom-macos
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: docker/setup-buildx-action@v3
-  #     - uses: scherermichael-oss/action-has-permission@master
-  #       id: check
-  #       with:
-  #         required-permission: write
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  #     - uses: docker/login-action@v3
-  #       if: steps.check.outputs.has-permission
-  #       with:
-  #         registry: docker.io
-  #         username: zifeo
-  #         password: ${{ secrets.DOCKERHUB_TOKEN }}
-  #     - name: Build with cache (internal)
-  #       uses: docker/build-push-action@v5
-  #       if: steps.check.outputs.has-permission
-  #       with:
-  #         file: dev/Dockerfile
-  #         platforms: ${{ matrix.platform }}
-  #         push: false
-  #         cache-from: type=registry,ref=docker.io/zifeo/metatype-cache:ci
-  #         cache-to: type=registry,ref=docker.io/zifeo/metatype-cache:ci,mode=max
-  #         # check target runs extra validation steps in the Dockerfile
-  #         target: check
-  #     - name: Build without cache (external)
-  #       uses: docker/build-push-action@v5
-  #       if: "! steps.check.outputs.has-permission"
-  #       with:
-  #         file: dev/Dockerfile
-  #         platforms: linux/amd64
-  #         push: false
-  #         cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/typegate:latest
-  #         target: check
+  test-docker:
+    needs: changes
+    if: ${{ needs.changes.outputs.typegate == 'true' }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          # FIXME: try macos-14 runner once all actions support it
+          # docker buildx action broken as of 2024-02-09
+          # - platform: linux/arm64
+          #   runner: custom-macos
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: scherermichael-oss/action-has-permission@master
+        id: check
+        with:
+          required-permission: write
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/login-action@v3
+        if: steps.check.outputs.has-permission
+        with:
+          registry: docker.io
+          username: zifeo
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build with cache (internal)
+        uses: docker/build-push-action@v5
+        if: steps.check.outputs.has-permission
+        with:
+          file: dev/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: false
+          cache-from: type=registry,ref=docker.io/zifeo/metatype-cache:ci
+          cache-to: type=registry,ref=docker.io/zifeo/metatype-cache:ci,mode=max
+          # check target runs extra validation steps in the Dockerfile
+          target: check
+      - name: Build without cache (external)
+        uses: docker/build-push-action@v5
+        if: "! steps.check.outputs.has-permission"
+        with:
+          file: dev/Dockerfile
+          platforms: linux/amd64
+          push: false
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/typegate:latest
+          target: check
 
   test-full:
     needs: changes

dev/update.ts

@@ -60,16 +60,20 @@ const tsFiles = [
     {
       root: projectDir,
       globstar: true,
-      exclude: ["typegate/tests/e2e/nextjs"],
+      exclude: [
+        "typegate/tests/e2e/nextjs",
+        "typegate/tests/runtimes/temporal/worker",
+      ],
     },
   ),
 ].map((f: WalkEntry) => f.path);
 
 await runOrExit([
-  "deno",
+  Deno.execPath(),
   "cache",
   `--config=${denoConfigPath}`,
-  "--unstable",
+  "--unstable-worker-options",
+  "--unstable-net",
   ...flags["cache-only"] ? [] : ["--reload", "--lock-write"],
   ...tsFiles,
 ], projectDir);

dev/utils.ts

@@ -13,16 +13,17 @@ export async function runOrExit(
   cwd?: string,
   env: Record<string, string> = {},
 ) {
+  // console.error("shell: ", cmd.join(" "), { env, cwd });
   const p = new Deno.Command(cmd[0], {
     args: cmd.slice(1),
     cwd,
-    stdout: "piped",
-    stderr: "piped",
+    // stdout: "piped",
+    // stderr: "piped",
     env: { ...Deno.env.toObject(), ...env },
   }).spawn();
 
-  await p.stdout.pipeTo(Deno.stdout.writable, { preventClose: true });
-  await p.stderr.pipeTo(Deno.stderr.writable, { preventClose: true });
+  // await p.stdout.pipeTo(Deno.stdout.writable, { preventClose: true });
+  // await p.stderr.pipeTo(Deno.stderr.writable, { preventClose: true });
 
   const { code, success } = await p.status;
   if (!success) {

examples/deploy/deploy.py

@@ -82,7 +82,10 @@ def deploy_example_python(g: Graph):
     runtime_actions=None,  # usually set from the cli
 )
 artifacts_config = ArtifactResolutionConfig(
-    prefix=None, dir=None, prisma_migration=config_params
+    prefix=None,
+    dir=None,
+    prisma_migration=config_params,
+    disable_artifact_resolution=None,
 )
 
 

meta-cli/src/com/server.rs

@@ -59,6 +59,7 @@ async fn config(req: HttpRequest) -> impl Responder {
     let endpoint = ServerStore::get_endpoint();
     let secrets = ServerStore::get_secrets();
     let migration_action_glob = ServerStore::get_migration_action_glob();
+    let disable_artifact_resolution = !ServerStore::get_artifact_resolution_flag();
 
     let mut migration_action_per_rt = vec![];
     if let Some(per_rt_actions) =
@@ -96,6 +97,7 @@ async fn config(req: HttpRequest) -> impl Responder {
                         "globalAction": migration_action_glob,
                         "runtimeAction": migration_action_per_rt
                     },
+                    "disableArtifactResolution": disable_artifact_resolution
                 },
             });
 

meta-cli/src/com/store.rs

@@ -68,6 +68,7 @@ pub struct ServerStore {
     endpoint: Endpoint,
     prefix: Option<String>,
     sdk_responses: HashMap<PathBuf, Arc<HashMap<String, SDKResponse>>>,
+    artifact_resolution: Option<bool>,
 }
 
 #[allow(dead_code)]
@@ -176,4 +177,13 @@ impl ServerStore {
     pub fn get_prefix() -> Option<String> {
         with_store(|s| s.prefix.to_owned())
     }
+
+    pub fn set_artifact_resolution_flag(value: bool) {
+        with_store_mut(|s| s.artifact_resolution = Some(value))
+    }
+
+    /// true by default
+    pub fn get_artifact_resolution_flag() -> bool {
+        with_store(|s| s.artifact_resolution.unwrap_or(true))
+    }
 }