Skip to content

Commit

Permalink
feat: Artifact removal (#668)
Browse files Browse the repository at this point in the history
- Add GC: remove artifacts when unreferenced by any deployed typegraph
- Improve resource management: use `AsyncDisposable` and
`AsyncDisposableStack`
- Improve testability (for parallel testing): always read the tmpDir
config from the `Typegate` object

#### Motivation and context

[MET-433](https://linear.app/metatypedev/issue/MET-433/file-removal)

#### Migration notes

_N/A_

### Checklist

- [x] The change come with new or modified tests
- [x] Hard-to-understand functions have explanatory comments
- [x] End-user documentation is updated to reflect the change


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
  - Enhanced search functionality with the addition of a new search bar.
  - Introduced new test configurations to improve script execution.
- Updated artifact storage documentation to clarify management
processes.
  - Added new extensions to support improved code commenting.

- **Bug Fixes**
- Removed outdated Deno import mapping settings to streamline
development environment setup.

- **Documentation**
- Expanded documentation on artifact tracking and management, including
reference counting and garbage collection mechanisms.

- **Refactor**
- Implemented interface changes in `QueryEngine` for better async
disposal management.
- Code restructuring in artifact management for enhanced performance and
maintainability.

- **Chores**
- Adjusted settings and configurations in the development environment to
align with current best practices.

- **Tests**
- Introduced new test cases for artifact upload and management
functionalities.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Natoandro <[email protected]>
Co-authored-by: destifo <[email protected]>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
  • Loading branch information
3 people authored May 8, 2024
1 parent 0cdd241 commit 50b064b
Show file tree
Hide file tree
Showing 70 changed files with 1,896 additions and 846 deletions.
196 changes: 196 additions & 0 deletions .ghjk/deno.lock

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions .ghjk/lock.json
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,15 @@
"env": {},
"allowedPortDeps": []
}
},
"test": {
"name": "test",
"dependsOn": [],
"env": {
"installs": [],
"env": {},
"allowedPortDeps": []
}
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion .vscode/extensions.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"chrischinchilla.vale-vscode",
"hediet.vscode-drawio",
"davidlday.languagetool-linter",
"dtsvet.vscode-wasm"
"dtsvet.vscode-wasm",
"bierner.comment-tagged-templates"
]
}
1 change: 0 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
"website/.docusaurus": true,
"website/node_modules": true
},
"deno.importMap": "typegate/import_map.json",
"deno.enablePaths": [
"typegate",
"dev",
Expand Down
12 changes: 6 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

156 changes: 78 additions & 78 deletions docs/workflows/artifact_upload_protocol.drawio.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
42 changes: 28 additions & 14 deletions ghjk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ const allowedPortDeps = [
defaultInst: thinInstallConfig(fat),
})),
];
export const secureConfig = ghjk.secureConfig({ allowedPortDeps });

const inCi = () => !!Deno.env.get("CI");
const inOci = () => !!Deno.env.get("OCI");
Expand All @@ -63,7 +62,7 @@ ghjk.install(
crateName: "wasm-tools",
version: WASM_TOOLS_VERSION,
locked: true,
}),
})
);

if (!inOci()) {
Expand All @@ -81,7 +80,7 @@ if (!inOci()) {
packageName: "@bytecodealliance/jco",
version: JCO_VERSION,
})[0],
ports.npmi({ packageName: "node-gyp", version: "10.0.1" })[0],
ports.npmi({ packageName: "node-gyp", version: "10.0.1" })[0]
);
}

Expand All @@ -90,7 +89,7 @@ if (Deno.build.os == "linux" && !Deno.env.has("NO_MOLD")) {
ports.mold({
version: MOLD_VERSION,
replaceLd: Deno.env.has("CI") || Deno.env.has("OCI"),
}),
})
);
}

Expand All @@ -100,20 +99,18 @@ if (!Deno.env.has("NO_PYTHON")) {
ports.pipi({
packageName: "poetry",
version: POETRY_VERSION,
})[0],
})[0]
);
if (!inOci()) {
ghjk.install(
ports.pipi({ packageName: "pre-commit" })[0],
);
ghjk.install(ports.pipi({ packageName: "pre-commit" })[0]);
}
}

if (inDev()) {
ghjk.install(
ports.act({}),
ports.cargobi({ crateName: "whiz", locked: true }),
installs.comp_py[0],
installs.comp_py[0]
);
}

Expand All @@ -122,13 +119,12 @@ ghjk.task("clean-deno-lock", {
// jq
],
async fn({ $ }) {
const jqOp1 =
`del(.packages.specifiers["npm:@typegraph/sdk@${METATYPE_VERSION}"])`;
const jqOp1 = `del(.packages.specifiers["npm:@typegraph/sdk@${METATYPE_VERSION}"])`;
const jqOp2 = `del(.packages.npm["@typegraph/sdk@${METATYPE_VERSION}"])`;
const jqOp = `${jqOp1} | ${jqOp2}`;
await Deno.writeTextFile(
"typegate/deno.lock",
await $`jq ${jqOp} typegate/deno.lock`.text(),
await $`jq ${jqOp} typegate/deno.lock`.text()
);
},
});
Expand All @@ -138,8 +134,9 @@ ghjk.task("gen-pyrt-bind", {
allowedPortDeps,
async fn({ $ }) {
await $.removeIfExists("./libs/pyrt_wit_wire/wit_wire");
await $`componentize-py -d ../../wit/wit-wire.wit bindings .`
.cwd("./libs/pyrt_wit_wire");
await $`componentize-py -d ../../wit/wit-wire.wit bindings .`.cwd(
"./libs/pyrt_wit_wire"
);
},
});

Expand Down Expand Up @@ -172,3 +169,20 @@ ghjk.task("dev", {
await $`deno run --allow-all ${script} ${args}`;
},
});

ghjk.task("test", {
async fn({ $, argv }) {
const script = $.path(projectDir).join("dev/test.ts");
await $`deno run --allow-all ${script} ${argv}`;
},
});

export const secureConfig = ghjk.secureConfig({
allowedPortDeps: [
...ghjk.stdDeps(),
...[installs.python_latest, installs.node].map((fat) => ({
manifest: fat.port,
defaultInst: thinInstallConfig(fat),
})),
],
});
3 changes: 3 additions & 0 deletions meta-cli/src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ pub struct NodeArgs {
#[clap(short, long, value_parser = UrlValueParser::new().http())]
pub gate: Option<Url>,

#[clap(short, long)]
pub prefix: Option<String>,

/// Username to use to connect to the typegate (basic auth).
#[clap(long)]
pub username: Option<String>,
Expand Down
3 changes: 3 additions & 0 deletions meta-cli/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ impl NodeConfig {
if let Some(gate) = &args.gate {
res.url = gate.clone();
}
if let Some(prefix) = &args.prefix {
res.prefix = Some(prefix.clone());
}
res.username = args.username.clone().or(res.username);
res.password = args.password.clone().or(res.password);
res
Expand Down
Loading

0 comments on commit 50b064b

Please sign in to comment.