From 3ee74750083a0d620da3fa32a95c0070f0367b4e Mon Sep 17 00:00:00 2001 From: Yohe-Am <56622350+Yohe-Am@users.noreply.github.com> Date: Sat, 25 May 2024 04:04:46 +0000 Subject: [PATCH] refactor(ghjk.ts): replace secureConfig with hack.ts --- README.md | 4 +- check.ts | 1 + deno.jsonc | 1 + examples/envs/ghjk.ts | 31 ++++ examples/kitchen/ghjk.ts | 100 +++++++++++ examples/protoc/ghjk.ts | 7 - examples/tasks/ghjk.ts | 6 +- files/deno/worker.ts | 2 +- files/mod.ts | 156 +++++++++-------- ghjk.ts | 22 +-- hack.ts | 21 +++ mod.ts | 350 ++++++++++++++++++++++++--------------- modules/tasks/deno.ts | 2 +- tests/envHooks.ts | 4 +- tests/envs.ts | 14 +- tests/ports.ts | 41 ++--- tests/reloadHooks.ts | 20 ++- tests/tasks.ts | 15 +- tests/utils.ts | 68 ++------ utils/mod.ts | 13 ++ 20 files changed, 554 insertions(+), 324 deletions(-) create mode 100644 examples/envs/ghjk.ts create mode 100644 examples/kitchen/ghjk.ts delete mode 100644 examples/protoc/ghjk.ts create mode 100644 hack.ts diff --git a/README.md b/README.md index 80bf19a1..63eb8ea1 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,9 @@ for tasks that are meant to be common dependencies of other tasks. ### Secure configs -Certain options are configured through the `secureConfig` object. +To improve ergonmoics, the typescript ghjkfile implementation exports simple functions and objects that mutate some global variable. +This also means that any script you import, if it knows the URL of the exact ghjk implementation you're using, can import this authoring module and mess with your ghjkfile. +Certain options for your file are thus only read from an export called `secureConfig` that'll host some of the more sensetive configurations. These include: ```ts import { env, stdSecureConfig } from "https://.../ghjk/mod.ts"; diff --git a/check.ts b/check.ts index 77a9dfd7..2ee8425b 100755 --- a/check.ts +++ b/check.ts @@ -6,6 +6,7 @@ import { $ } from "./utils/mod.ts"; const files = (await Array.fromAsync( $.path(import.meta.url).parentOrThrow().expandGlob("**/*.ts", { exclude: [ + ".git", "play.ts", ".ghjk/**", ".deno-dir/**", diff --git a/deno.jsonc b/deno.jsonc index a2187dd4..310d5082 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -6,6 +6,7 @@ }, "fmt": { "exclude": [ + "*.md", "**/*.md", ".ghjk/**", ".deno-dir/**", diff --git a/examples/envs/ghjk.ts b/examples/envs/ghjk.ts new file mode 100644 index 00000000..3a2e7c5d --- /dev/null +++ b/examples/envs/ghjk.ts @@ -0,0 +1,31 @@ +export { sophon } from "../../hack.ts"; +import { config, env, install, task } from "../../hack.ts"; +import * as ports from "../../ports/mod.ts"; + +config({ + // we can change which environment + // is activated by default for example + // when we enter the directory + defaultEnv: "main", + // set the env all others envs will by + // default inherit from + defaultBaseEnv: "main", +}); + +env("test", { + installs: [ports.unzip()], +}); + +env("ci") + .install(ports.opentofu_ghrel()); + +// top level `install` calls just +// go to an enviroment called "main" +install(ports.protoc()); + +// we can modify "main" directly +env("main") + // hooks execute when environments are + // activated/deactivated in interactive shells + .onEnter(task(($) => $`echo enter`)) + .onExit(task(($) => $`echo exit`)); diff --git a/examples/kitchen/ghjk.ts b/examples/kitchen/ghjk.ts new file mode 100644 index 00000000..a8eed12b --- /dev/null +++ b/examples/kitchen/ghjk.ts @@ -0,0 +1,100 @@ +import { stdDeps } from "../../files/mod.ts"; +import { file } from "../../mod.ts"; +import * as ports from "../../ports/mod.ts"; + +const ghjk = file({ + // configre an empty env so that no ports are avail by default in our workdir + defaultEnv: "empty", + envs: [{ name: "empty", inherit: false }], + // we wan't all other envs to start from empty unless they opt otherwise + defaultBaseEnv: "empty", + + // we won't use the following for now + // but they pretty much configure the "main" env + allowedBuildDeps: [], + installs: [], + stdDeps: true, + enableRuntimes: true, + // tasks aren't attached to envs + // but have their own env + tasks: [], +}); + +// we need this export for this file to be a valid ghjkfile +// it's the one thing used by the ghjk host implementation to +// interact with your ghjkfile +export const sophon = ghjk.sophon; + +const { install, env, task } = ghjk; + +// we can configure main like this as well +env("main") + // provision env vars to be acccessbile in the env + .var("RUST_LOG", "info,actix=warn") + // provision programs to be avail in the env + .install(ports.jq_ghrel()) + .allowedBuildDeps([ + // ports can use the following installs at build time + // very WIP mechanism but this is meant to prevent ports from + // pulling whatever dependency they want at build time unless + // explicityl allowed to do so + ports.cpy_bs({ version: "3.8.18", releaseTag: "20240224" }), + ports.node({}), + ports.rust({ version: "stable" }), + // add the std deps including the runtime ports. + // These includes node and python but still, precedence is given + // to our configuration of those ports above + ...stdDeps({ enableRuntimes: true }), + ]); + +// these top level installs go to the main env as well +install( + ports.rust({ version: "stable" }), + ports.protoc(), +); + +const ci = env("ci", { + // this inherits from main so it gets protoc and curl + inherit: "main", + // extra installs + installs: [ports.jq_ghrel()], + // it has extra allowed deps + allowedBuildDeps: [ports.node()], + // more env vars + vars: { + CI: 1, + }, + desc: "do ci stuff", +}); + +// tasks are invocable from the cli +task("install-app", ($) => $`cargo fetch`); + +task("build-app", { + dependsOn: "install-app", + // the task's env inherits from ci + inherit: ci.name, + // it can add more items to that env + installs: [], + // vars + vars: { + RUST_BACKTRACE: 1, + }, + // allowed build deps + allowedBuildDeps: [ports.zstd()], + desc: "build the app", + fn: async ($) => { + await $`cargo build -p app`; + // we can access tar here from the ci env + await $`tar xcv ./target/debug/app -o app.tar.gz`; + }, +}); + +env("dev") + .inherit("main") + // we can set tasks to run on activation/decativation + .onEnter(task(($) => $`echo enter`)) + .onEnter(task({ + workingDir: "..", + fn: ($) => $`ls`, + })); diff --git a/examples/protoc/ghjk.ts b/examples/protoc/ghjk.ts deleted file mode 100644 index a194ecf7..00000000 --- a/examples/protoc/ghjk.ts +++ /dev/null @@ -1,7 +0,0 @@ -export { ghjk } from "../../mod.ts"; -import { install } from "../../mod.ts"; -import protoc from "../../ports/protoc.ts"; - -install( - protoc(), -); diff --git a/examples/tasks/ghjk.ts b/examples/tasks/ghjk.ts index 225ed3a7..e4d5808a 100644 --- a/examples/tasks/ghjk.ts +++ b/examples/tasks/ghjk.ts @@ -1,5 +1,5 @@ -export { ghjk } from "../../mod.ts"; -import { logger, task } from "../../mod.ts"; +export { sophon } from "../../hack.ts"; +import { logger, task } from "../../hack.ts"; import * as ports from "../../ports/mod.ts"; task("greet", async ($, { argv: [name] }) => { @@ -9,7 +9,7 @@ task("greet", async ($, { argv: [name] }) => { const ha = task({ name: "ha", installs: [ports.protoc()], - envVars: { STUFF: "stuffier" }, + vars: { STUFF: "stuffier" }, async fn($) { await $`echo $STUFF; protoc --version; diff --git a/files/deno/worker.ts b/files/deno/worker.ts index 57c8201d..f6e641f0 100644 --- a/files/deno/worker.ts +++ b/files/deno/worker.ts @@ -37,7 +37,7 @@ async function serializeConfig(uri: string, envVars: Record) { const { setup: setupLogger } = await import("../../utils/logger.ts"); setupLogger(); const mod = await import(uri); - const rawConfig = await mod.ghjk.getConfig(uri, mod.secureConfig); + const rawConfig = await mod.sophon.getConfig(uri, mod.secureConfig); const config = JSON.parse(JSON.stringify(rawConfig)); return { config, diff --git a/files/mod.ts b/files/mod.ts index 24b04094..e6c565db 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -46,7 +46,7 @@ import type { export type EnvDefArgs = { name: string; installs?: InstallConfigFat[]; - allowedPortDeps?: AllowedPortDep[]; + allowedBuildDeps?: (InstallConfigFat | AllowedPortDep)[]; /** * If true or not set, will base the task's env on top * of the default env (usually `main`). If false, will build on @@ -55,7 +55,7 @@ export type EnvDefArgs = { */ inherit?: string | boolean; desc?: string; - vars?: Record; + vars?: Record; /** * Task to execute when environment is activated. */ @@ -84,10 +84,10 @@ export type TaskFn = ( export type TaskDefArgs = { name?: string; desc?: string; - dependsOn?: string[]; + dependsOn?: string | string[]; workingDir?: string | Path; - envVars?: Record; - allowedPortDeps?: AllowedPortDep[]; + vars?: Record; + allowedBuildDeps?: (InstallConfigFat | AllowedPortDep)[]; installs?: InstallConfigFat[]; inherit?: string | boolean; }; @@ -160,10 +160,13 @@ export class Ghjkfile { logger(import.meta).debug("install added", config); } - setAllowedPortDeps(setId: string, deps: AllowedPortDep[]) { + setAllowedPortDeps( + setId: string, + deps: (InstallConfigFat | AllowedPortDep)[], + ) { const set = this.#getSet(setId); set.allowedDeps = Object.fromEntries( - deps.map(( + reduceAllowedDeps(deps).map(( dep, ) => [dep.manifest.name, dep]), ); @@ -224,8 +227,8 @@ export class Ghjkfile { if (args.installs) { env.install(...args.installs); } - if (args.allowedPortDeps) { - env.allowedPortDeps(args.allowedPortDeps); + if (args.allowedBuildDeps) { + env.allowedBuildDeps(args.allowedBuildDeps); } if (args.desc) { env.desc(args.desc); @@ -259,22 +262,22 @@ export class Ghjkfile { } toConfig( - { defaultEnv, defaultBaseEnv, masterPortDepAllowList }: { + { defaultEnv, defaultBaseEnv }: { defaultEnv: string; defaultBaseEnv: string; ghjkfileUrl: string; - masterPortDepAllowList: AllowedPortDep[]; }, ) { + // make sure referenced envs exist + this.addEnv({ name: defaultEnv }); + this.addEnv({ name: defaultBaseEnv }); try { const envsConfig = this.#processEnvs(defaultEnv, defaultBaseEnv); const tasksConfig = this.#processTasks( envsConfig, defaultBaseEnv, ); - const portsConfig = this.#processInstalls( - masterPortDepAllowList ?? stdDeps(), - ); + const portsConfig = this.#processInstalls(); const config: SerializedConfig = { blackboard: Object.fromEntries(this.#bb.entries()), @@ -333,7 +336,7 @@ export class Ghjkfile { const final = finalizer(); const envBaseResolved = typeof final.inherit === "string" ? final.inherit - : final.inherit && defaultBaseEnv != final.name + : (final.inherit !== false) && defaultBaseEnv != final.name ? defaultBaseEnv : null; all[final.name] = { ...final, envBaseResolved }; @@ -408,10 +411,10 @@ export class Ghjkfile { }; const hooks = [ ...final.onEnterHookTasks.map( - (key) => [key, "hook.onEnter.posixExec"] as const, + (key) => [key, "hook.onEnter.ghjkTask"] as const, ), ...final.onExitHookTasks.map( - (key) => [key, "hook.onExit.posixExec"] as const, + (key) => [key, "hook.onExit.ghjkTask"] as const, ), ].map(([taskKey, ty]) => { const task = this.#tasks.get(taskKey); @@ -425,8 +428,7 @@ export class Ghjkfile { } if (task.ty == "denoFile@v1") { const prov: InlineTaskHookProvision = { - ty: "inline.hook.ghjkTask", - finalTy: ty, + ty, taskKey, }; return prov; @@ -489,9 +491,9 @@ export class Ghjkfile { ); for (const [key, args] of this.#tasks) { if (args.dependsOn && args.dependsOn.length > 0) { - const depKeys = args.dependsOn.map((nameOrKey) => - nameToKey[nameOrKey] ?? nameOrKey - ); + const depKeys = + (Array.isArray(args.dependsOn) ? args.dependsOn : [args.dependsOn]) + .map((nameOrKey) => nameToKey[nameOrKey] ?? nameOrKey); deps.set(key, depKeys); for (const depKey of depKeys) { const depRevDeps = revDeps.get(depKey); @@ -517,37 +519,45 @@ export class Ghjkfile { const args = this.#tasks.get(key)!; const { workingDir, desc, dependsOn, inherit } = args; - const envBaseResolved = typeof inherit === "string" - ? inherit - : inherit - ? defaultBaseEnv - : null; - - const envBaseRecipe = envBaseResolved - ? envsConfig.envs[envBaseResolved] - : null; - const taskEnvRecipe: EnvRecipe = { provides: [], }; - const taskInstallSet: InstallSet = { installs: args.installs ?? [], allowedDeps: Object.fromEntries( - (args.allowedPortDeps ?? []).map((dep) => [dep.manifest.name, dep]), + reduceAllowedDeps(args.allowedBuildDeps ?? []).map(( + dep, + ) => [dep.manifest.name, dep]), ), }; - const mergedEnvVars = args.envVars ?? {}; + const envBaseResolved = typeof inherit === "string" + ? inherit + : (inherit !== false) + ? defaultBaseEnv + : null; + + const envBaseRecipe = envBaseResolved + ? envsConfig.envs[envBaseResolved] + : null; + + const mergedEnvVars = args.vars ?? {}; if (envBaseRecipe) { for ( const prov of envBaseRecipe .provides as ( | WellKnownProvision | InstallSetRefProvision + | InlineTaskHookProvision )[] ) { - if (prov.ty == "posix.envVar") { + // task envs don't need hooks + if ( + prov.ty == "hook.onEnter.ghjkTask" || + prov.ty == "hook.onExit.ghjkTask" + ) { + continue; + } else if (prov.ty == "posix.envVar") { if (!mergedEnvVars[prov.key]) { mergedEnvVars[prov.key] = prov.val; } @@ -586,7 +596,11 @@ export class Ghjkfile { ...Object.entries(mergedEnvVars).map(( [key, val], ) => { - const prov: WellKnownProvision = { ty: "posix.envVar", key, val }; + const prov: WellKnownProvision = { + ty: "posix.envVar", + key, + val: val.toString(), + }; return prov; }), ); @@ -601,9 +615,14 @@ export class Ghjkfile { ? workingDir.toString() : workingDir, desc, - dependsOn: dependsOn?.map((keyOrHash) => - localToFinalKey[nameToKey[keyOrHash] ?? keyOrHash] - ), + ...dependsOn + ? { + dependsOn: (Array.isArray(dependsOn) ? dependsOn : [dependsOn]) + ?.map((keyOrHash) => + localToFinalKey[nameToKey[keyOrHash] ?? keyOrHash] + ), + } + : {}, envHash, }; const taskHash = objectHash(def); @@ -665,12 +684,16 @@ export class Ghjkfile { env.provides = env.provides.map( (prov) => { if ( - prov.ty == "inline.hook.ghjkTask" + prov.ty == "hook.onEnter.ghjkTask" || + prov.ty == "hook.onExit.ghjkTask" ) { + logger().warn("caught"); const inlineProv = prov as InlineTaskHookProvision; const taskKey = localToFinalKey[inlineProv.taskKey]; const out: WellKnownProvision = { - ty: inlineProv.finalTy, + ty: /onEnter/.test(prov.ty) + ? "hook.onEnter.posixExec" + : "hook.onExit.posixExec", program: "ghjk", arguments: ["x", taskKey], }; @@ -684,28 +707,13 @@ export class Ghjkfile { return moduleConfig; } - #processInstalls(masterAllowList: AllowedPortDep[]) { + #processInstalls() { const out: PortsModuleConfigHashed = { sets: {}, }; - const masterPortDepAllowList = Object.fromEntries( - masterAllowList.map((dep) => [dep.manifest.name, dep] as const), - ); for ( const [setId, set] of this.#installSets.entries() ) { - for (const [portName, _] of Object.entries(set.allowedDeps)) { - if (!masterPortDepAllowList[portName]) { - throw new Error( - `"${portName}" is in allowedPortDeps list of install set "${setId}" but not in the masterPortDepAllowList`, - ); - } - } - for (const [name, hash] of Object.entries(masterPortDepAllowList)) { - if (!set.allowedDeps[name]) { - set.allowedDeps[name] = hash; - } - } out.sets[setId] = { installs: set.installs.map((inst) => this.#addToBlackboard(inst)), allowedDeps: this.#addToBlackboard(Object.fromEntries( @@ -736,7 +744,7 @@ export class EnvBuilder { #installSetId: string; #file: Ghjkfile; #inherit: string | boolean = true; - #vars: Record = {}; + #vars: Record = {}; #desc?: string; #onEnterHookTasks: string[] = []; #onExitHookTasks: string[] = []; @@ -752,7 +760,9 @@ export class EnvBuilder { name: this.name, installSetId: this.#installSetId, inherit: this.#inherit, - vars: this.#vars, + vars: Object.fromEntries( + Object.entries(this.#vars).map(([key, val]) => [key, val.toString()]), + ), desc: this.#desc, onExitHookTasks: this.#onExitHookTasks, onEnterHookTasks: this.#onEnterHookTasks, @@ -775,9 +785,10 @@ export class EnvBuilder { } /** + * Configure the build time deps allowed to be used by ports. * This is treated as a single set and will replace previously any configured set. */ - allowedPortDeps(deps: AllowedPortDep[]) { + allowedBuildDeps(deps: (AllowedPortDep | InstallConfigFat)[]) { this.#file.setAllowedPortDeps(this.#installSetId, deps); return this; } @@ -793,7 +804,7 @@ export class EnvBuilder { /** * Add multiple environment variable. */ - vars(envVars: Record) { + vars(envVars: Record) { Object.assign(this.#vars, envVars); return this; } @@ -865,9 +876,22 @@ function task$( } type InlineTaskHookProvision = Provision & { - ty: "inline.hook.ghjkTask"; - finalTy: - | "hook.onEnter.posixExec" - | "hook.onExit.posixExec"; + ty: "hook.onExit.ghjkTask" | "hook.onEnter.ghjkTask"; taskKey: string; }; + +export function reduceAllowedDeps( + deps: (AllowedPortDep | InstallConfigFat)[], +): AllowedPortDep[] { + return deps.map( + (dep: any) => { + const res = portsValidators.allowedPortDep.safeParse(dep); + if (res.success) return res.data; + const out: AllowedPortDep = { + manifest: dep.port, + defaultInst: thinInstallConfig(dep), + }; + return portsValidators.allowedPortDep.parse(out); + }, + ); +} diff --git a/ghjk.ts b/ghjk.ts index e5589430..97244623 100644 --- a/ghjk.ts +++ b/ghjk.ts @@ -1,7 +1,12 @@ -export { ghjk } from "./mod.ts"; -import { $, env, install, stdSecureConfig, task } from "./mod.ts"; +export { sophon } from "./hack.ts"; +import { config, install } from "./hack.ts"; import * as ports from "./ports/mod.ts"; +config({ + defaultBaseEnv: "test", + enableRuntimes: true, +}); + // these are just for quick testing install(); @@ -11,16 +16,3 @@ install( ports.pipi({ packageName: "pre-commit" })[0], ports.cpy_bs(), ); - -env("main") - .onEnter(task(($) => $`echo enter`)) - .onExit(task(($) => $`echo exit`)); - -env("test", { - installs: [ports.protoc()], -}); - -export const secureConfig = stdSecureConfig({ - enableRuntimes: true, - defaultBaseEnv: "test", -}); diff --git a/hack.ts b/hack.ts new file mode 100644 index 00000000..38bf5d12 --- /dev/null +++ b/hack.ts @@ -0,0 +1,21 @@ +//! This file allows an easy way to start with the typescript ghjkfile +//! but is generally insecure for serious usage. +//! +//! If your ghjkfile imports a malicious module, the module could +//! import the functions defined herin and mess with your ghjkfile. +//! +//! For example, it could set `rm -rf / --no-preserve-root` to your +//! main env entry hook! + +export * from "./mod.ts"; +import { file } from "./mod.ts"; + +const { + sophon, + task, + env, + install, + config, +} = file(); + +export { config, env, install, sophon, task }; diff --git a/mod.ts b/mod.ts index 23aa2837..4f9771f4 100644 --- a/mod.ts +++ b/mod.ts @@ -1,165 +1,243 @@ -//! This module is intended to be re-exported by `ghjk.ts` config scripts. Please -//! avoid importing elsewhere at it has side-effects. +//! This module is intended to be re-exported by `ghjk.ts` config scripts. // TODO: harden most of the items in here import "./setup_logger.ts"; -import { zod } from "./deps/common.ts"; // ports specific imports -import portsValidators from "./modules/ports/types.ts"; import type { AllowedPortDep, InstallConfigFat, } from "./modules/ports/types.ts"; import logger from "./utils/logger.ts"; -import { $, thinInstallConfig } from "./utils/mod.ts"; -import { EnvBuilder, Ghjkfile, stdDeps } from "./files/mod.ts"; +import { $ } from "./utils/mod.ts"; +import { + EnvBuilder, + Ghjkfile, + reduceAllowedDeps, + stdDeps, +} from "./files/mod.ts"; import type { DenoTaskDefArgs, EnvDefArgs, TaskFn } from "./files/mod.ts"; // WARN: this module has side-effects and only ever import // types from it import type { ExecTaskArgs } from "./modules/tasks/deno.ts"; -const DEFAULT_BASE_ENV_NAME = "main"; +export type { DenoTaskDefArgs, EnvDefArgs, TaskFn } from "./files/mod.ts"; +export { $, logger, stdDeps }; -const file = new Ghjkfile(); -const mainEnv = file.addEnv({ - name: DEFAULT_BASE_ENV_NAME, - inherit: false, - allowedPortDeps: stdDeps(), - desc: "the default default environment.", -}); +export type AddEnv = { + (args: EnvDefArgs): EnvBuilder; + (name: string, args?: Omit): EnvBuilder; +}; -export type { DenoTaskDefArgs, EnvDefArgs, TaskFn } from "./files/mod.ts"; -export { $, logger, stdDeps, stdSecureConfig }; - -// FIXME: ses.lockdown to freeze primoridials -// freeze the object to prevent malicious tampering of the secureConfig -export const ghjk = Object.freeze({ - getConfig: Object.freeze( - ( - ghjkfileUrl: string, - secureConfig: DenoFileSecureConfig | undefined, - ) => { - const defaultEnv = secureConfig?.defaultEnv ?? DEFAULT_BASE_ENV_NAME; - const defaultBaseEnv = secureConfig?.defaultBaseEnv ?? - DEFAULT_BASE_ENV_NAME; - return file.toConfig({ - defaultEnv, - defaultBaseEnv, - ghjkfileUrl, - masterPortDepAllowList: secureConfig?.masterPortDepAllowList ?? - stdDeps(), - }); - }, - ), - execTask: Object.freeze( - // TODO: do we need to source the default base env from - // the secure config here? - (args: ExecTaskArgs) => file.execTask(args), - ), -}); - -/* - * Provision a port install in the `main` environment. +/** + * Provision a port install in the `main` env. */ -export function install(...configs: InstallConfigFat[]) { - mainEnv.install(...configs); -} +export type AddInstall = { + (...configs: InstallConfigFat[]): void; +}; /** * Define and register a task. */ -export function task(args: DenoTaskDefArgs): string; -export function task(name: string, args: Omit): string; -export function task( - name: string, - fn: TaskFn, - args?: Omit, -): string; -export function task(fn: TaskFn, args?: Omit): string; -export function task( - nameOrArgsOrFn: string | DenoTaskDefArgs | TaskFn, - argsOrFn?: Omit | TaskFn, - argsMaybe?: Omit, -): string { - let args: DenoTaskDefArgs; - if (typeof nameOrArgsOrFn == "object") { - args = nameOrArgsOrFn; - } else if (typeof nameOrArgsOrFn == "function") { - args = { - ...(argsOrFn ?? {}), - fn: nameOrArgsOrFn, - }; - } else if (typeof argsOrFn == "object") { - args = { ...argsOrFn, name: nameOrArgsOrFn }; - } else if (argsOrFn) { - args = { - ...(argsMaybe ?? {}), - name: nameOrArgsOrFn, - fn: argsOrFn, - }; - } else { - args = { - name: nameOrArgsOrFn, - }; - } - return file.addTask({ ...args, ty: "denoFile@v1" }); -} +export type AddTask = { + (args: DenoTaskDefArgs): string; + (name: string, args: Omit): string; + (fn: TaskFn, args?: Omit): string; + ( + name: string, + fn: TaskFn, + args?: Omit, + ): string; +}; -export function env(args: EnvDefArgs): EnvBuilder; -export function env(name: string, args?: Omit): EnvBuilder; -export function env( - nameOrArgs: string | EnvDefArgs, - argsMaybe?: Omit, -): EnvBuilder { - const args = typeof nameOrArgs == "object" - ? nameOrArgs - : { ...argsMaybe, name: nameOrArgs }; - return file.addEnv(args); -} +export type FileArgs = { + /** + * The env to activate by default. When entering the working + * directory for example. + */ + defaultEnv?: string; + /** + * The default env all envs inherit from. + */ + defaultBaseEnv?: string; + /** + * Additional ports that can be used as build time dependencies. + */ + allowedBuildDeps?: (InstallConfigFat | AllowedPortDep)[]; + /** + * Wether or not use the default set of allowed build dependencies. + * If set, {@link enableRuntimes} is ignored but {@link allowedBuildDeps} + * is still respected. + * True by default. + */ + stdDeps?: boolean; + /** + * (unstable) Allow runtimes from std deps to be used as build time dependencies. + */ + enableRuntimes?: boolean; + /** + * Installs to add to the main env. + */ + installs?: InstallConfigFat[]; + /** + * Tasks to expose to the CLI. + */ + tasks?: DenoTaskDefArgs[]; + /** + * Different envs availaible to the CLI. + */ + envs?: EnvDefArgs[]; +}; -const denoFileSecureConfig = zod.object({ - masterPortDepAllowList: zod.array(portsValidators.allowedPortDep).nullish(), - // TODO: move into envs/types - defaultEnv: zod.string().nullish(), - defaultBaseEnv: zod.string().nullish(), -}); -/* - * This is a secure sections of the config intended to be direct exports - * from the config script instead of the global variable approach the - * main [`GhjkConfig`] can take. - */ -export type DenoFileSecureConfig = zod.input< - typeof denoFileSecureConfig ->; -export type DenoFileSecureConfigX = zod.input< - typeof denoFileSecureConfig +type SecureConfigArgs = Omit< + FileArgs, + "envs" | "tasks" | "installs" >; -function stdSecureConfig( - args: { - additionalAllowedPorts?: (InstallConfigFat | AllowedPortDep)[]; - enableRuntimes?: boolean; - } & Pick, -) { - const { additionalAllowedPorts, enableRuntimes = false } = args; - const out: DenoFileSecureConfig = { - ...args, - masterPortDepAllowList: [ - ...stdDeps({ enableRuntimes }), - ...additionalAllowedPorts?.map( - (dep: any) => { - const res = portsValidators.allowedPortDep.safeParse(dep); - if (res.success) return res.data; - const out: AllowedPortDep = { - manifest: dep.port, - defaultInst: thinInstallConfig(dep), - }; - return portsValidators.allowedPortDep.parse(out); - }, - ) ?? [], - ], +export function file( + args: FileArgs = {}, +): { + sophon: Readonly; + install: AddInstall; + task: AddTask; + env: AddEnv; + config(args: SecureConfigArgs): void; +} { + const defaultBuildDepsSet: AllowedPortDep[] = []; + + // this replaces the allowedBuildDeps contents according to the + // args. Written to be called multilple times to allow + // replacement. + const replaceDefaultBuildDeps = (args: SecureConfigArgs) => { + // empty out the array first + defaultBuildDepsSet.length = 0; + defaultBuildDepsSet.push( + ...reduceAllowedDeps(args.allowedBuildDeps ?? []), + ); + const seenPorts = new Set( + defaultBuildDepsSet.map((dep) => dep.manifest.name), + ); + // if the user explicitly passes a port config, we let + // it override any ports of the same kind from the std library + for ( + const dep + of (args.stdDeps || args.stdDeps === undefined || args.stdDeps === null) + ? stdDeps({ enableRuntimes: args.enableRuntimes ?? false }) + : [] + ) { + if (seenPorts.has(dep.manifest.name)) { + continue; + } + defaultBuildDepsSet.push(dep); + } + }; + + // populate the bulid deps by the default args first + replaceDefaultBuildDeps(args); + + const DEFAULT_BASE_ENV_NAME = "main"; + + const file = new Ghjkfile(); + const mainEnv = file.addEnv({ + name: DEFAULT_BASE_ENV_NAME, + inherit: false, + installs: args.installs, + // the default build deps will be used + // as the allow set for the main env as well + // NOTE: this approach allows the main env to + // disassociate itself from the default set + // if the user invokes `allowedBuildDeps` + // on its EnvBuilder + allowedBuildDeps: defaultBuildDepsSet, + desc: "the default default environment.", + }); + for (const env of args.envs ?? []) { + file.addEnv(env); + } + for (const task of args.tasks ?? []) { + file.addTask({ ...task, ty: "denoFile@v1" }); + } + + // FIXME: ses.lockdown to freeze primoridials + // freeze the object to prevent malicious tampering of the secureConfig + const sophon = Object.freeze({ + getConfig: Object.freeze( + ( + ghjkfileUrl: string, + ) => { + return file.toConfig({ + ghjkfileUrl, + defaultEnv: args.defaultEnv ?? DEFAULT_BASE_ENV_NAME, + defaultBaseEnv: args.defaultBaseEnv ?? + DEFAULT_BASE_ENV_NAME, + }); + }, + ), + execTask: Object.freeze( + // TODO: do we need to source the default base env from + // the secure config here? + (args: ExecTaskArgs) => file.execTask(args), + ), + }); + + // we return a bunch of functions here + // to ease configuring the main environment + // including overloads + return { + sophon, + + install(...configs: InstallConfigFat[]) { + mainEnv.install(...configs); + }, + + task( + nameOrArgsOrFn: string | DenoTaskDefArgs | TaskFn, + argsOrFn?: Omit | TaskFn, + argsMaybe?: Omit, + ) { + let args: DenoTaskDefArgs; + if (typeof nameOrArgsOrFn == "object") { + args = nameOrArgsOrFn; + } else if (typeof nameOrArgsOrFn == "function") { + args = { + ...(argsOrFn ?? {}), + fn: nameOrArgsOrFn, + }; + } else if (typeof argsOrFn == "object") { + args = { ...argsOrFn, name: nameOrArgsOrFn }; + } else if (argsOrFn) { + args = { + ...(argsMaybe ?? {}), + name: nameOrArgsOrFn, + fn: argsOrFn, + }; + } else { + args = { + name: nameOrArgsOrFn, + }; + } + return file.addTask({ ...args, ty: "denoFile@v1" }); + }, + + env( + nameOrArgs: string | EnvDefArgs, + argsMaybe?: Omit, + ) { + const args = typeof nameOrArgs == "object" + ? nameOrArgs + : { ...argsMaybe, name: nameOrArgs }; + return file.addEnv(args); + }, + + config( + a: SecureConfigArgs, + ) { + replaceDefaultBuildDeps(a); + args = { + ...args, + ...a, + }; + }, }; - return out; } diff --git a/modules/tasks/deno.ts b/modules/tasks/deno.ts index e3c0003d..fe246012 100644 --- a/modules/tasks/deno.ts +++ b/modules/tasks/deno.ts @@ -61,7 +61,7 @@ async function importAndExec( args: ExecTaskArgs, ) { const mod = await import(uri); - await mod.ghjk.execTask(args); + await mod.sophon.execTask(args); return true; } diff --git a/tests/envHooks.ts b/tests/envHooks.ts index cf20e562..283ae22f 100644 --- a/tests/envHooks.ts +++ b/tests/envHooks.ts @@ -78,8 +78,8 @@ const cases: CustomE2eTestCase[] = [ harness(cases.map((testCase) => ({ ...testCase, tsGhjkfileStr: ` -export { ghjk } from "$ghjk/mod.ts"; -import { task, env } from "$ghjk/mod.ts"; +export { sophon } from "$ghjk/hack.ts"; +import { task, env } from "$ghjk/hack.ts"; env("main") .onEnter(task($ => $\`/bin/sh -c 'echo remark > marker'\`)) diff --git a/tests/envs.ts b/tests/envs.ts index 0be9b369..343c1b7e 100644 --- a/tests/envs.ts +++ b/tests/envs.ts @@ -5,9 +5,8 @@ import { genTsGhjkFile, harness, } from "./utils.ts"; -import { stdSecureConfig } from "../mod.ts"; import dummy from "../ports/dummy.ts"; -import type { DenoFileSecureConfig } from "../mod.ts"; +import type { FileArgs } from "../mod.ts"; type CustomE2eTestCase = & Omit @@ -18,7 +17,7 @@ type CustomE2eTestCase = & ( | { envs: EnvDefArgs[]; - secureConfig?: DenoFileSecureConfig; + secureConfig?: FileArgs; } | { ghjkTs: string; @@ -184,7 +183,7 @@ const cases: CustomE2eTestCase[] = [ name: "default_env_loader", ePoint: "fish", envs: envVarTestEnvs, - secureConfig: stdSecureConfig({ defaultEnv: "yuki" }), + secureConfig: { defaultEnv: "yuki" }, stdin: ` set fish_trace 1 # env base is false for "yuki" and thus no vars from "main" @@ -198,7 +197,12 @@ test "$HUMM" = "Soul Lady"; or exit 108 harness(cases.map((testCase) => ({ ...testCase, tsGhjkfileStr: "ghjkTs" in testCase ? testCase.ghjkTs : genTsGhjkFile( - { envDefs: testCase.envs, secureConf: testCase.secureConfig }, + { + secureConf: { + ...testCase.secureConfig, + envs: testCase.envs, + }, + }, ), ePoints: [{ cmd: testCase.ePoint, stdin: testCase.stdin }], name: `envs/${testCase.name}`, diff --git a/tests/ports.ts b/tests/ports.ts index 93605ec6..9e5f34a7 100644 --- a/tests/ports.ts +++ b/tests/ports.ts @@ -1,5 +1,5 @@ import "../setup_logger.ts"; -import { DenoFileSecureConfig, stdSecureConfig } from "../mod.ts"; +import { FileArgs } from "../mod.ts"; import { E2eTestCase, genTsGhjkFile, harness } from "./utils.ts"; import * as ports from "../ports/mod.ts"; import dummy from "../ports/dummy.ts"; @@ -9,7 +9,7 @@ import { testTargetPlatform } from "./utils.ts"; type CustomE2eTestCase = Omit & { ePoint: string; installConf: InstallConfigFat | InstallConfigFat[]; - secureConf?: DenoFileSecureConfig; + secureConf?: FileArgs; }; // order tests by download size to make failed runs less expensive const cases: CustomE2eTestCase[] = [ @@ -97,18 +97,18 @@ const cases: CustomE2eTestCase[] = [ name: "npmi-node-gyp", installConf: ports.npmi({ packageName: "node-gyp" }), ePoint: `node-gyp --version`, - secureConf: stdSecureConfig({ + secureConf: { enableRuntimes: true, - }), + }, }, // node + more megs { name: "npmi-jco", installConf: ports.npmi({ packageName: "@bytecodealliance/jco" }), ePoint: `jco --version`, - secureConf: stdSecureConfig({ + secureConf: { enableRuntimes: true, - }), + }, }, // 42 megs { @@ -159,9 +159,9 @@ const cases: CustomE2eTestCase[] = [ name: "pipi-poetry", installConf: ports.pipi({ packageName: "poetry" }), ePoint: `poetry --version`, - secureConf: stdSecureConfig({ + secureConf: { enableRuntimes: true, - }), + }, }, // rustup + 600 megs { @@ -202,9 +202,12 @@ harness(cases.map((testCase) => ({ ...testCase, tsGhjkfileStr: genTsGhjkFile( { - installConf: testCase.installConf, - secureConf: testCase.secureConf, - taskDefs: [], + secureConf: { + ...testCase.secureConf, + installs: Array.isArray(testCase.installConf) + ? testCase.installConf + : [testCase.installConf], + }, }, ), ePoints: [ @@ -212,14 +215,14 @@ harness(cases.map((testCase) => ({ cmd: [...`env ${sh}`.split(" "), `"${testCase.ePoint}"`], })), /* // FIXME: better tests for the `InstallDb` - // installs db means this shouldn't take too long - // as it's the second sync - { - cmd: [ - ..."env".split(" "), - "bash -c 'timeout 1 ghjk envs cook'", - ], - }, */ + // installs db means this shouldn't take too long + // as it's the second sync + { + cmd: [ + ..."env".split(" "), + "bash -c 'timeout 1 ghjk envs cook'", + ], + }, */ ], name: `ports/${testCase.name}`, }))); diff --git a/tests/reloadHooks.ts b/tests/reloadHooks.ts index b26f1e51..d9192a3a 100644 --- a/tests/reloadHooks.ts +++ b/tests/reloadHooks.ts @@ -202,15 +202,17 @@ harness(cases.map((testCase) => ({ ...testCase, tsGhjkfileStr: genTsGhjkFile( { - envDefs: [ - { - name: "main", - installs: testCase.installConf ? testCase.installConf : [dummy()], - }, - { - name: "test", - }, - ], + secureConf: { + envs: [ + { + name: "main", + installs: testCase.installConf ? testCase.installConf : [dummy()], + }, + { + name: "test", + }, + ], + }, }, ), ePoints: [{ cmd: testCase.ePoint, stdin: testCase.stdin }], diff --git a/tests/tasks.ts b/tests/tasks.ts index cb8cf352..c85f8bf1 100644 --- a/tests/tasks.ts +++ b/tests/tasks.ts @@ -2,7 +2,6 @@ import "../setup_logger.ts"; import { E2eTestCase, genTsGhjkFile, harness, type TaskDef } from "./utils.ts"; import * as ghjk from "../mod.ts"; import * as ports from "../ports/mod.ts"; -import { stdSecureConfig } from "../mod.ts"; type CustomE2eTestCase = & Omit @@ -36,7 +35,7 @@ test (ghjk x greet world) = "Hello world from $PWD!"`, name: "env_vars", tasks: [{ name: "greet", - envVars: { + vars: { LUNA: "moon", SOL: "sun", }, @@ -67,7 +66,7 @@ ghjk x protoc`, name: "test", // pipi depends on cpy_bs installs: [...ports.pipi({ packageName: "pre-commit" })], - allowedPortDeps: ghjk.stdDeps({ enableRuntimes: true }), + allowedBuildDeps: ghjk.stdDeps({ enableRuntimes: true }), fn: async ($) => { await $`pre-commit --version`; }, @@ -123,8 +122,8 @@ test (cat eddy) = 'ed edd eddy' { name: "anon", ghjkTs: ` -export { ghjk } from "$ghjk/mod.ts"; -import { task } from "$ghjk/mod.ts"; +export { sophon } from "$ghjk/hack.ts"; +import { task } from "$ghjk/hack.ts"; task({ dependsOn: [ @@ -151,10 +150,10 @@ harness(cases.map((testCase) => ({ ...testCase, tsGhjkfileStr: "ghjkTs" in testCase ? testCase.ghjkTs : genTsGhjkFile( { - taskDefs: testCase.tasks, - secureConf: stdSecureConfig({ + secureConf: { + tasks: testCase.tasks, enableRuntimes: testCase.enableRuntimesOnMasterPDAL, - }), + }, }, ), ePoints: [{ cmd: testCase.ePoint, stdin: testCase.stdin }], diff --git a/tests/utils.ts b/tests/utils.ts index ad15166a..766709ec 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -2,13 +2,8 @@ import { defaultInstallArgs, install } from "../install/mod.ts"; import { std_url } from "../deps/dev.ts"; import { std_async } from "../deps/dev.ts"; import { $, dbg, importRaw } from "../utils/mod.ts"; -import type { InstallConfigFat } from "../modules/ports/types.ts"; import logger from "../utils/logger.ts"; -import type { - DenoFileSecureConfig, - DenoTaskDefArgs, - EnvDefArgs, -} from "../mod.ts"; +import type { DenoTaskDefArgs, FileArgs } from "../mod.ts"; export type { EnvDefArgs } from "../mod.ts"; import { ALL_OS } from "../port.ts"; import { ALL_ARCH } from "../port.ts"; @@ -181,35 +176,23 @@ export type TaskDef = & Required>; export function genTsGhjkFile( - { installConf, secureConf, taskDefs, envDefs }: { - installConf?: InstallConfigFat | InstallConfigFat[]; - secureConf?: DenoFileSecureConfig; - taskDefs?: TaskDef[]; - envDefs?: EnvDefArgs[]; + { secureConf }: { + secureConf?: FileArgs; }, ) { - const installConfArray = installConf - ? Array.isArray(installConf) ? installConf : [installConf] - : []; - - const serializedPortsInsts = JSON.stringify( - installConfArray, - (_, val) => - typeof val == "string" - // we need to escape a json string embedded in a js string - // 2x - ? val.replaceAll(/\\/g, "\\\\") - : val, - ); - const serializedSecConf = JSON.stringify( // undefined is not recognized by JSON.parse // so we stub it with null - secureConf ?? null, + { + ...secureConf, + tasks: [], + }, + // we need to escape a json string embedded in a js string + // 2x (_, val) => typeof val == "string" ? val.replaceAll(/\\/g, "\\\\") : val, ); - const tasks = (taskDefs ?? []).map( + const tasks = (secureConf?.tasks ?? []).map( (def) => { const stringifiedSection = JSON.stringify( def, @@ -219,41 +202,24 @@ export function genTsGhjkFile( return $.dedent` ghjk.task({ ...JSON.parse(\`${stringifiedSection}\`), - fn: ${def.fn.toString()} - })`; - }, - ).join("\n"); - - const envs = (envDefs ?? []).map( - (def) => { - const stringifiedSection = JSON.stringify( - def, - (_, val) => - typeof val == "string" ? val.replaceAll(/\\/g, "\\\\") : val, - ); - return $.dedent` - ghjk.env({ - ...JSON.parse(\`${stringifiedSection}\`), + fn: ${def.fn?.toString()} })`; }, ).join("\n"); return ` -export { ghjk } from "$ghjk/mod.ts"; -import * as ghjk from "$ghjk/mod.ts"; +import { file } from "$ghjk/mod.ts"; + const confStr = \` -${serializedPortsInsts} +${serializedSecConf} \`; const confObj = JSON.parse(confStr); -ghjk.install(...confObj) +const ghjk = file(confObj); -const secConfStr = \` -${serializedSecConf} -\`; -export const secureConfig = JSON.parse(secConfStr); +export const sophon = ghjk.sophon; ${tasks} -${envs} + `; } diff --git a/utils/mod.ts b/utils/mod.ts index df1f8a6b..f9659be4 100644 --- a/utils/mod.ts +++ b/utils/mod.ts @@ -196,6 +196,19 @@ export const $ = dax.build$( requestBuilder: new dax.RequestBuilder() .showProgress(Deno.stderr.isTerminal()), extras: { + mapObject< + O, + V2, + >( + obj: O, + map: (key: keyof O, val: O[keyof O]) => [string, V2], + ): Record { + return Object.fromEntries( + Object.entries(obj as object).map(([key, val]) => + map(key as keyof O, val as O[keyof O]) + ), + ); + }, exponentialBackoff(initialDelayMs: number) { let delay = initialDelayMs; let attempt = 0;