diff --git a/SECURITY.md b/SECURITY.md index d19e0a731..5595fe113 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,6 +8,6 @@ We are only supporting the latest version at this point. Please email security@hofstadter.io -We appreciate your secure disclusures. We don't have much money right now but will make our best effort to find a reward or other consideration. +We appreciate your secure disclosures. We don't have much money right now but will make our best effort to find a reward or other consideration. Thank You diff --git a/docs/code/flow/examples/tasks/prompt/example.cue b/docs/code/flow/examples/tasks/prompt/example.cue new file mode 100644 index 000000000..370c0149d --- /dev/null +++ b/docs/code/flow/examples/tasks/prompt/example.cue @@ -0,0 +1,42 @@ +package prompt + +import "encoding/json" + +test: { + @flow() + + prompt: { + @task(prompt.Prompt) + + Input: { + name: string + repo: string + releases: bool | *false + } + + Questions: [{ + Name: "name" + Type: "input" + Prompt: "What is your project named" + Required: true + }, { + Name: "repo" + Type: "input" + Prompt: "Git repository" + Default: "github.com/user/repo" + }, { + Name: "releases" + Type: "confirm" + Prompt: "Enable release tooling" + Default: true + }] + + Output: _ + } + + output: { + @task(os.Stdout) + text: json.Indent(json.Marshal(prompt.Output), "", " ") + } + +} \ No newline at end of file diff --git a/docs/code/hof-schemas/flow/tasks/api/schema.cue b/docs/code/hof-schemas/flow/tasks/api/schema.cue new file mode 100644 index 000000000..3b9de43d6 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/api/schema.cue @@ -0,0 +1,75 @@ +package api + +Method: *"GET" | "POST" | "PUT" | "DELETE" | "OPTIONS" | "HEAD" | "CONNECT" | "TRACE" | "PATCH" + +Call: { + @task(api.Call) + + req: { + method: Method + host: string + path: string | *"" + auth?: string + headers?: [string]: string + query?: [string]: string + form?: [string]: string + data?: string | {...} + timeout?: string + // curl?: string + retry?: { + count: int | *3 + timer: string | *"6s" + codes: [...int] + } + } + + // filled by task + resp: { + status: string + statusCode: int + + body: *{} | bytes | string + header: [string]: string | [...string] + trailer: [string]: string | [...string] + } + +} + +Serve: { + @task(api.Serve) + + port: string + quitMailbox: string + + routes: [...{ + // @flow() is needed to run sub-tasks per request, which is more typical + // you can omit if you only need to reshape the data with CUE code + + // filled by hof/flow on each request + req: { + method: Method + url: string + + headers: [string]: string + query: [string]: string + + body: bytes | string | *{} // assumed json body if object + } + + // any tasks you may need to convert the req -> resp + // these will be run after the `req` fields has been filled + + // you construct the resp value which is sent back to the client + // (todo, make this include headers, code, etc + // for now, this is a value which will be turned into a JSON body for the response + resp: { + status: int + + // one of, if none -> NoContent + json?: {} + html?: string + body?: string + } + + }] +} diff --git a/docs/code/hof-schemas/flow/tasks/csp/schema.cue b/docs/code/hof-schemas/flow/tasks/csp/schema.cue new file mode 100644 index 000000000..0afbb58b9 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/csp/schema.cue @@ -0,0 +1,50 @@ +// CSP (Communicating Sequential Processes) +// Named channels (mailboxes) and send/recv messages +// Similar to Go / Erlang concurrency +// Can be accessed from different flows/tasks +package csp + +// Chan is a named mailbox +Chan: { + @task(csp.Chan) + + // the name of the channel + mailbox: string + + // how many messages can queue + // defaults to a blocking send / recv + buf: int | *0 +} + +// Send a message to a mailbox +Send: { + @task(csp.Send) + + // the name of the channel + mailbox: string + + // an optional key + key?: string + + // the message value + val: _ +} + +// Recv is a coroutine which runs indefinitely +Recv: { + @task(csp.Recv) + + // the name of the channel + mailbox: string + + // the name of a second mailbox to quit on any message received + quitMailbox?: string + + // the handler for a message, run as a flow per message + handler: { + // filled when a message is received + msg: _ + + // do whatever you like in here + } +} diff --git a/docs/code/hof-schemas/flow/tasks/cue/schema.cue b/docs/code/hof-schemas/flow/tasks/cue/schema.cue new file mode 100644 index 000000000..e8707b2ea --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/cue/schema.cue @@ -0,0 +1,21 @@ +package hof + +Format: { + @task(cue.Format) + + value: _ + + Package: string | *"" + Raw: bool | *false + Final: bool | *false + Concrete: bool | *true + Definitions: bool | *true + Optional: bool | *true + Hidden: bool | *true + Attributes: bool | *true + Docs: bool | *true + InlineImports: bool | *false + ErrorsAsValues: bool | *false + + out: string +} diff --git a/docs/code/hof-schemas/flow/tasks/db/schema.cue b/docs/code/hof-schemas/flow/tasks/db/schema.cue new file mode 100644 index 000000000..2161f651e --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/db/schema.cue @@ -0,0 +1,37 @@ +package db + +// need to figure out how to separate out the connection +// so it can be create once and reused by queries +// possibly how to pool connections as well + +// only SQLite supported currently + +// Call a database +Call: { + @task(db.Call) + + // db connection + conn: { + sqlite?: string // db name + postgres?: string + } + + // args to Call + args: [...] + + // Use only one of [query,exec,stmts] + query: string + exec: string + + stmts: [...{ + // Use only one of [query,exec,stmts] + query: string + exec: string + // args to statement, merged with top-level + args: string + + results: _ + }] + + results: _ +} diff --git a/docs/code/hof-schemas/flow/tasks/gen/schema.cue b/docs/code/hof-schemas/flow/tasks/gen/schema.cue new file mode 100644 index 000000000..601577a32 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/gen/schema.cue @@ -0,0 +1,35 @@ +package gen + +// Seed the Range +Seed: { + @task(gen.Seed) + + // only set to ensure consistent output while testing + seed?: int // defaults to time.Now() +} + +Int: { + @task(gen.Int) + max?: int // max value if set + + // the random val returned + val: int +} + +Str: { + @task(gen.Str) + + // number of runes to generate + n: int | *12 + + // possible runes, defaults to [a-zA-Z] + runes?: string +} + +// the other tasks don't really have schema or input +c: string @task(gen.CUID) // like UUID, but for cloud +f: float @task(gen.Float) +n: float @task(gen.Norm) +t: string @task(gen.Now) // RFC-3339 +s: string @task(gen.Slug) // related to CUID +u: string @task(gen.UUID) diff --git a/docs/code/hof-schemas/flow/tasks/hof/schema.cue b/docs/code/hof-schemas/flow/tasks/hof/schema.cue new file mode 100644 index 000000000..052f06e88 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/hof/schema.cue @@ -0,0 +1,13 @@ +package hof + +Tempate: { + @task(hof.Template) + + name: string | *"" + data: _ + + template: string + partials: [string]: string + + out: string +} diff --git a/docs/code/hof-schemas/flow/tasks/kv/schema.cue b/docs/code/hof-schemas/flow/tasks/kv/schema.cue new file mode 100644 index 000000000..830bb56af --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/kv/schema.cue @@ -0,0 +1,25 @@ +package kv + +// in memory cue.Value storage +Mem: { + @task(kv.Mem) + + // key to store under + key: string + + // if specified, value to store + // otherwise the value in memory as filled in + val?: _ + + // delete the key & value + delete: bool | *false + + // boolean for if the value was loaded + loaded: bool +} + +// redis, etcd + +// obj Stores (here?) +// mongo, couch, elastic +// s3 / gcs diff --git a/docs/code/hof-schemas/flow/tasks/msg/schema.cue b/docs/code/hof-schemas/flow/tasks/msg/schema.cue new file mode 100644 index 000000000..277f33417 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/msg/schema.cue @@ -0,0 +1,8 @@ +package msg + +// todo, abstract message type 'Any' +// add more like rabbit, kafka, nats + +IrcClient: { + // see github.com/verdverm/streamer-tools for an example +} diff --git a/docs/code/hof-schemas/flow/tasks/os/schema.cue b/docs/code/hof-schemas/flow/tasks/os/schema.cue new file mode 100644 index 000000000..c374ac03d --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/os/schema.cue @@ -0,0 +1,157 @@ +package os + +Exec: { + @task(os.Exec) + + cmd: string | [string, ...string] + + // dir specifies the working directory of the command. + // The default is the current working directory. + dir?: string + + // env defines the environment variables to use for this system. + // If the value is a list, the entries mus be of the form key=value, + // where the last value takes precedence in the case of multiple + // occurrences of the same key. + env: {[string]: string} | [...=~"="] + + // stdout captures the output from stdout if it is of type bytes or string. + // The default value of null indicates it is redirected to the stdout of the + // current process. + stdout?: null | bool | string | bytes + + // stderr is like stdout, but for errors. + stderr?: null | bool | string | bytes + + // stdin specifies the input for the process. If stdin is bool (true or false), the stdin + // of the current process is redirected to this command (the default). + // If it is of typ bytes or string, that input will be used instead. + stdin?: *null | bool | string | bytes + + // success is set to true when the process terminates with with a zero exit + // code or false otherwise. The user can explicitly specify the value + // force a fatal error if the desired success code is not reached. + success: bool + + // the exit code of the command + exitcode: int + + // error from cmd.Run() + error: string + + // exit flow when task fails + exitonerr: bool | *true +} + +// Get a filelock +FileLock: { + @task(os.FileLock) + + // lockfile name + filename: string + + // read-write (true for read-write, false for read-only) + rw: bool | *false + + // time.Duration for retries, zero means off + retry: string | *"0" +} + +// release a filelock +FileUnlock: { + @task(os.FileUnlock) + + // lockfile name + filename: string +} + +// A Value are all possible values allowed in flags. +// A null value unsets an environment variable. +Value: bool | number | *string | null + +// Name indicates a valid flag name. +Name: !="" & !~"^[$]" + +// Getenv gets and parses the specific command line variables. +Getenv: { + @task(os.Getenv) + + // if empty, get all + {[Name]: Value} +} + +Glob: { + @task(os.Glob) + + // glob patterns to match + globs: [...string] + + // filepaths found matching any of the globs + filepaths: [...string] +} + +// acts like 'mkdir -p' +Mkdir: { + @task(os.Mkdir) + + dir: string +} + +ReadFile: { + @task(os.ReadFile) + + // filename to read + filename: string + + // filled by flow + contents: *string | bytes +} + +Sleep: { + @task(os.Sleep) + + // time.Duration to sleep for + duration: string +} + +// read from stdin +Stdin: { + @task(os.Stdout) + + // optional message to user before reading input + prompt?: string + // user input + contents: string +} + +// print to stdout +Stdout: { + @task(os.Stdout) + + // text to write + text: string +} + +Watch: { + @task(os.Watch) + + // glob patterns to watch for events + globs: [...string] + + // todo, only handles write events + // should add event to handler before calling + + // a flow handler to run on each event + handler: {...} + + // (good first issue) + // debounce?: string // time.Duration +} + +WriteFile: { + @task(os.WriteFile) + + filename: string + contents: string | bytes + mode: int | *0o644 +} \ No newline at end of file diff --git a/docs/code/hof-schemas/flow/tasks/prompt/schema.cue b/docs/code/hof-schemas/flow/tasks/prompt/schema.cue new file mode 100644 index 000000000..876a0cd00 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/prompt/schema.cue @@ -0,0 +1,10 @@ +package prompt + +import ( + schema "github.com/hofstadter-io/hof/schema/prompt" +) + +// Same prompt from creators as a workflow task +Prompt: schema.Prompt & { + @task(prompt.Prompt) +} \ No newline at end of file diff --git a/docs/code/hof-schemas/flow/tasks/schema.cue b/docs/code/hof-schemas/flow/tasks/schema.cue new file mode 100644 index 000000000..0f3d7e11f --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/schema.cue @@ -0,0 +1,6 @@ +package tasks + +Noop: { + @task(noop) + $task: "noop" +} diff --git a/docs/code/hof-schemas/flow/tasks/st/schema.cue b/docs/code/hof-schemas/flow/tasks/st/schema.cue new file mode 100644 index 000000000..db0015c91 --- /dev/null +++ b/docs/code/hof-schemas/flow/tasks/st/schema.cue @@ -0,0 +1,59 @@ +package st + +Mask: { + @task(st.Mask) + + val: _ + mask: _ + out: _ +} + +Pick: { + @task(st.Pick) + + val: _ + pick: _ + out: _ +} + +Insert: { + @task(st.Insert) + + val: _ + insert: _ + out: _ +} + +Replace: { + @task(st.Replace) + + val: _ + replace: _ + out: _ +} + +Upsert: { + @task(st.Upsert) + + val: _ + upsert: _ + out: _ +} + +Diff: { + @task(st.Diff) + + orig: _ + patch: diff + next: _ + diff: _ +} + +Patch: { + @task(st.Patch) + + orig: _ + patch: _ + next: _ + diff: patch +} diff --git a/docs/code/hof-schemas/hof.cue b/docs/code/hof-schemas/hof.cue index 4cce49106..bbb3e9f5b 100644 --- a/docs/code/hof-schemas/hof.cue +++ b/docs/code/hof-schemas/hof.cue @@ -15,7 +15,6 @@ Hof: { // hof/datamodel datamodel?: { - // define the root of a datamodel root: bool | *false @@ -37,18 +36,16 @@ Hof: { // hof/gen gen?: { + // define the root of a generator root: bool | *false // name of the generator name: string | *"" - - // TODO, do we need this? aren't we... - // determining based on the existence of Create: {} - creator: bool | *false } // hof/flow, used for both flows & tasks flow?: { + // define the root of a workflow root: bool | *false // name of the flow or task @@ -56,14 +53,7 @@ Hof: { // if op is empty, it is a flow value // if op is not empty, it is a task value - // TODO, maybe we make this "flow" for flows? - op: string | *"flow" - } - - chat?: { - root: bool | *false - name: string | *"" - extra: string | *"" + op: string | *"" } } } @@ -82,4 +72,4 @@ Metadata: { } // depreciated -DHof: Hof +DHof: Hof \ No newline at end of file diff --git a/docs/config.cue b/docs/config.cue index 6ff392965..c05bfd090 100644 --- a/docs/config.cue +++ b/docs/config.cue @@ -25,7 +25,7 @@ params: { github_project_repo: "https://github.com/hofstadter-io/hof" github_repo: "https://github.com/hofstadter-io/hof" time_format_default: "January 2, 2006" - hofver: "v0.6.9-rc.1" + hofver: "v0.6.9-rc.2" cuever: "v0.8.2" cuedocBase: "https://pkg.go.dev/cuelang.org/go@\(cuever)" slackLink: "https://join.slack.com/t/hofstadter-io/shared_invite/zt-e5f90lmq-u695eJur0zE~AG~njNlT1A" diff --git a/docs/config.yaml b/docs/config.yaml index d1c6426d6..4dee1ff46 100644 --- a/docs/config.yaml +++ b/docs/config.yaml @@ -18,7 +18,7 @@ params: github_project_repo: https://github.com/hofstadter-io/hof github_repo: https://github.com/hofstadter-io/hof time_format_default: January 2, 2006 - hofver: v0.6.9-rc.1 + hofver: v0.6.9-rc.2 cuever: v0.8.2 cuedocBase: https://pkg.go.dev/cuelang.org/go@v0.8.2 slackLink: https://join.slack.com/t/hofstadter-io/shared_invite/zt-e5f90lmq-u695eJur0zE~AG~njNlT1A diff --git a/docs/content/_index.md b/docs/content/_index.md index df4a48ff0..3630c52d9 100755 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -35,17 +35,17 @@ __`hof` is a CLI tool you *add* to your workflow.__ #### There are two modes to use `hof` -1. creating applications (green boxes) -1. building reusable modules (blue boxes) +1. building reusable modules +1. creating applications Like most languages and frameworks, there are two types of users. Most users will build applications for some purpose, using libraries written by others. A smaller number will build reusable modules, like the packages and libraries you use today. -`hof` has the same for same relationship for code generators modules. -All modules exist outside of the `hof` tool and just need to be a git repository. +`hof` has the same relationship for its features, when packaged as modules. -## Designed to augment your workflows +### Augment your workflows +
__`hof` is a CLI tool you will add to your workflows.__ We know developers have their own preferences @@ -79,13 +79,6 @@ You can also add custom code directly in the output and `hof` will ensure it stays as you regenerate your application. -# We call this High Code development. - -{{}} -Creating code with higher levels of design, reuse, and implementation -{{}} - - ## What can you do with `hof`? diff --git a/docs/content/changelogs/v0.6.x/changelog.md b/docs/content/changelogs/v0.6.x/changelog.md index 7576c8121..138621c40 100644 --- a/docs/content/changelogs/v0.6.x/changelog.md +++ b/docs/content/changelogs/v0.6.x/changelog.md @@ -179,7 +179,6 @@ See the following to learn more - `hof gen -h` - [getting-started/code-generation](/getting-started/code-generation/) - [code-generation/adhoc](/code-generation/adhoc/) -- [reference/hof-gen/adhoc-file-gen](/reference/hof-gen/adhoc-file-gen/) ### other changes @@ -204,7 +203,7 @@ See the following to learn more: - `hof flow -h` - [getting-started](/getting-started/task-engine/) - [task-engine](/task-engine/) -- [reference](/reference/hof-flow/) +- [task reference](/task-engine/tasks/) ### other changes @@ -227,7 +226,7 @@ See the following to learn more: - `hof datamodel -h` - [getting-started/data-layer](/getting-started/data-layer/) -- [reference/hof-datamodel](/reference/hof-datamodel/) +- [data modeling section](/data-modeling/) ### other changes diff --git a/docs/content/code-generation/adhoc/_index.md b/docs/content/code-generation/adhoc/_index.md index 5341ccff2..ce4822851 100644 --- a/docs/content/code-generation/adhoc/_index.md +++ b/docs/content/code-generation/adhoc/_index.md @@ -1,6 +1,7 @@ --- Title: Ad-hoc Code Gen +draft: true weight: 59 --- diff --git a/docs/content/code-generation/formatting/_index.md b/docs/content/code-generation/formatting/_index.md index a2bf0522c..0df435631 100644 --- a/docs/content/code-generation/formatting/_index.md +++ b/docs/content/code-generation/formatting/_index.md @@ -26,6 +26,27 @@ see the [custom formatters](/code-generation/formatting/custom-formatters/) sect hof fmt ``` +{{}} +`hof/fmt` is a command which will +format any and all languages. +You can create your own formatters as well. +{{}} + + +`hof` needs a code formatter for the languages it generates. +It runs the pre-output through before applying diff and merging +with any custom code you added to output files. +This is simplifies the job of template authors, +but is also required to avoid unnecessary merge conflicts. + +You will need Docker available to use this feature. +Hof will pull and run containers in the background. +You can disable this by setting an environment variable. + +> `HOF_FMT_DISABLED=1` + + +{{}} ## Supported Languages diff --git a/docs/content/reference/hof-datamodel/commands.md b/docs/content/data-modeling/commands.md similarity index 100% rename from docs/content/reference/hof-datamodel/commands.md rename to docs/content/data-modeling/commands.md diff --git a/docs/content/reference/hof-datamodel/complexity.md b/docs/content/data-modeling/complexity.md similarity index 100% rename from docs/content/reference/hof-datamodel/complexity.md rename to docs/content/data-modeling/complexity.md diff --git a/docs/content/reference/hof-datamodel/composition.md b/docs/content/data-modeling/composition.md similarity index 100% rename from docs/content/reference/hof-datamodel/composition.md rename to docs/content/data-modeling/composition.md diff --git a/docs/content/reference/hof-datamodel/databases.md b/docs/content/data-modeling/databases.md similarity index 100% rename from docs/content/reference/hof-datamodel/databases.md rename to docs/content/data-modeling/databases.md diff --git a/docs/content/reference/hof-datamodel/extending.md b/docs/content/data-modeling/extending.md similarity index 100% rename from docs/content/reference/hof-datamodel/extending.md rename to docs/content/data-modeling/extending.md diff --git a/docs/content/reference/hof-datamodel/generators.md b/docs/content/data-modeling/generators.md similarity index 100% rename from docs/content/reference/hof-datamodel/generators.md rename to docs/content/data-modeling/generators.md diff --git a/docs/content/reference/hof-datamodel/languages.md b/docs/content/data-modeling/languages.md similarity index 100% rename from docs/content/reference/hof-datamodel/languages.md rename to docs/content/data-modeling/languages.md diff --git a/docs/content/reference/hof-datamodel/migrations-and-lenses.md b/docs/content/data-modeling/migrations-and-lenses.md similarity index 100% rename from docs/content/reference/hof-datamodel/migrations-and-lenses.md rename to docs/content/data-modeling/migrations-and-lenses.md diff --git a/docs/content/reference/hof-datamodel/schemas.md b/docs/content/data-modeling/schemas.md similarity index 100% rename from docs/content/reference/hof-datamodel/schemas.md rename to docs/content/data-modeling/schemas.md diff --git a/docs/content/reference/hof-datamodel/snapshots-and-history.md b/docs/content/data-modeling/snapshots-and-history.md similarity index 100% rename from docs/content/reference/hof-datamodel/snapshots-and-history.md rename to docs/content/data-modeling/snapshots-and-history.md diff --git a/docs/content/reference/hof-datamodel/with_cue.md b/docs/content/data-modeling/with_cue.md similarity index 100% rename from docs/content/reference/hof-datamodel/with_cue.md rename to docs/content/data-modeling/with_cue.md diff --git a/docs/content/first-example/data-layer/checkpointing.md b/docs/content/first-example/data-layer/checkpointing.md index 4df933fdc..9b8283f4f 100644 --- a/docs/content/first-example/data-layer/checkpointing.md +++ b/docs/content/first-example/data-layer/checkpointing.md @@ -21,7 +21,7 @@ Available Commands: {{}} Run `hof dm -h` for the full listing. -The top-level [data-modeling](/reference/hof-datamodel/) goes deeper into +The top-level [data-modeling](/data-modeling/) goes deeper into the command and many related topics. ### Checkpointing diff --git a/docs/content/first-example/data-layer/schema.md b/docs/content/first-example/data-layer/schema.md index 2288cb068..c404c5415 100644 --- a/docs/content/first-example/data-layer/schema.md +++ b/docs/content/first-example/data-layer/schema.md @@ -22,7 +22,7 @@ The following as the core of the `#Datamodel` schema. {{< codePane title="Core of #Datamodel" file="code/first-example/data-layer/content/schema/hof-dm.html">}} -See [hof datamodel schemas](/reference/hof-datamodel/schemas/) for the full schema. +See [hof datamodel schemas](/data-modeling/schemas/) for the full schema. ### example/schema.#Datamodel diff --git a/docs/content/reference/command-help.md b/docs/content/getting-started/command-help.md similarity index 89% rename from docs/content/reference/command-help.md rename to docs/content/getting-started/command-help.md index 886ae937b..36eed1bb7 100644 --- a/docs/content/reference/command-help.md +++ b/docs/content/getting-started/command-help.md @@ -24,7 +24,7 @@ Top level commands and help message Used for data model management (dm for short) -See the [Data Modeling section](/reference/hof-datamodel/) for details. +See the [data modeling section](/data-modeling/) for details.
hof help datamodel @@ -36,7 +36,7 @@ See the [Data Modeling section](/reference/hof-datamodel/) for details. Create one-liners to generate files with data, CUE, and templates or use composable generators to build out advanced applications. -See [the code gen docs](/reference/hof-gen/) to learn more +See [the code generation section](/code-generation/) to learn more
hof help flow @@ -47,7 +47,7 @@ See [the code gen docs](/reference/hof-gen/) to learn more Build workflows and scripts with CUE and a DAG engine -See [the flow docs](/reference/hof-flow/) to learn more +See [the task engine section](/task-engine/) to learn more
hof help flow diff --git a/docs/content/reference/concepts.md b/docs/content/getting-started/concepts.md similarity index 99% rename from docs/content/reference/concepts.md rename to docs/content/getting-started/concepts.md index f65883aff..f66f8e8e9 100644 --- a/docs/content/reference/concepts.md +++ b/docs/content/getting-started/concepts.md @@ -3,6 +3,7 @@ title: "Concepts" description: "Overview of the main concepts in Hofstadter around code generation, source-of-truth, reusability." brief: "in Hofstadter, a quick overview" +draft: true weight: 90 --- diff --git a/docs/content/getting-started/cue.md b/docs/content/getting-started/cue.md index 53062fe28..94b9a21e8 100644 --- a/docs/content/getting-started/cue.md +++ b/docs/content/getting-started/cue.md @@ -47,3 +47,58 @@ To further your CUE knowledge, be sure to check out these resources: - [CUE documentation](https://cuelang.org) (from the CUE Team) - [Cuetorials](https://cuetorials.com) (by Hofstadter) + + +### Hof's CUE commands + +Hof embeds CUE's `vet, def, eval, export` commands for your convenience. +The are mostly drop in alternatives, some codecs are not available and +several enhancements have been added. + +The enhancements are: + +- additional methods for data placement +- increased flexibility for environment variables +- @userfiles() to include any file +- `--tui` flag to open hof's TUI for the commands + + +### Hof & CUE Modules + +__hof__ has a preview version for __CUE modules__. +Hof & CUE's modules serve the same purpose as other languages, +allowing to to version, share, and reuse code. +CUE's module system is still largely experimental. +We will eventually migrate over once sufficient features +are in place, at which point we will provide automation to update. + +Most of hof's features can be used from the module system. +"`hof mod`" is the subcommand for working with modules and dependencies. +The implementation is based on Go modules. + +The name of a module should be the same the git repository. +`hof` talks directly to git repositories and many of +`hof`'s commands will accept modules as an input argument too. +There is also support for OCI based repositories. + +##### [To learn more, see the modules section](/modules/). + +
+ +{{}} +# create a new module +hof mod init github.com/hofstadter-io/example + +# add a dependency +hof mod get github.com/hofstadter-io/hof@v0.6.8 + or +hof mod get github.com/hofstadter-io/hof@latest + +# tidy dependencies +hof mod tidy + +# fetch dependencies +hof mod link + or +hof mod vendor +{{}} diff --git a/docs/content/getting-started/data-layer.md b/docs/content/getting-started/data-layer.md index da6d1fd7f..758596ed0 100644 --- a/docs/content/getting-started/data-layer.md +++ b/docs/content/getting-started/data-layer.md @@ -8,16 +8,21 @@ weight: 40 {{}} The data layer is a combination of schemas, config, and annotations -that are specially treated by `hof`. The two primary goals are +that `hof` understands and operates on. The primary goals are: -1. Provide consistent data models for downstream consumers -2. Enable history, diff, and migration features for version flexibility +- Support git-style checkpointing, history, and diff features, flexible to your datamodel schema +- Provide consistent data models for downstream consumers +- Enable downstream features like automatic database migrations, client/server version skew, and change detection for infrastructure as code + +The `hof datamodel` command and `schema/dm` schema form the foundations and +are designed so you can customize, extend, or replace as needed. + +- the built-in base models, fields, and enrichers +- the shape and hierarchy for diff and history tracking -The `hof datamodel` command and `schema/dm` are central to this, but are designed -in a way that allows you to customize and extend the built-in base. {{}} -_Note, `hof dm` is shorthand for `hof datamodel`. +_Note, `hof dm` is shorthand for `hof datamodel`_. ## Schemas diff --git a/docs/content/getting-started/hof-attributes.md b/docs/content/getting-started/hof-attributes.md new file mode 100644 index 000000000..af6ad39f9 --- /dev/null +++ b/docs/content/getting-started/hof-attributes.md @@ -0,0 +1,35 @@ +--- +title: "#hof & @attributes" + +weight: 70 +--- + +{{}} +The `#hof` definition is how generators, datamodels, and workflows are discovered. +Several attributes allow you to write shorthand for common settings. +{{}} + + +### The Attributes + +By now, you have seen hof's attributes in the previous getting-started sections. +Hof turns these CUE attributes into `#hof` configuration. + +- `@gen()` - the root of a generator +- `@datamodel()` - the root of a datamodel +- `@flow()` - the root of a workflow or a task type + +Datamodels and workflows have a few more attributes +that can be specified under their root. +They are covered in the respective sections on each. + +### Schema + +The following is the schema for `#hof`. +Many generators, workflows, and datamodels +you import and use will fill this in for you. + +You can embed the following schema in order to reference the content. +Hof will automatically inject it during the loading process to ensure correctness. + +{{< codePane title="hof/schema.Hof" file="code/hof-schemas/hof.html" >}} diff --git a/docs/content/getting-started/hof-tui.md b/docs/content/getting-started/hof-tui.md index ce2ee46c6..0cc271f27 100644 --- a/docs/content/getting-started/hof-tui.md +++ b/docs/content/getting-started/hof-tui.md @@ -6,7 +6,7 @@ brief: "Explore and develop CUE interactively" keywords: - TUI -weight: 45 +weight: 60 --- diff --git a/docs/content/getting-started/llm-chat.md b/docs/content/getting-started/llm-chat.md index 9b06a6e41..1f0564c1d 100644 --- a/docs/content/getting-started/llm-chat.md +++ b/docs/content/getting-started/llm-chat.md @@ -1,12 +1,11 @@ --- title: LLM Chat -description: "Combining LLMs, Bard, ChatGPT, and Hof." -brief: "Combining LLMs, ChatGPT, and hof." -draft: true +description: "Combining LLMs and Hof." +brief: "Combining LLMs and hof." keywords: - LLM -- Bard +- Gemini - ChatGPT - code gen @@ -15,16 +14,10 @@ weight: 50 {{}} Large Language Models (LLM) are an inflection point in computing. -The represent significant advancements and automation for tasks. -Generating code is among them and there are many interesting topics -at the intersection of LLMs and Hof. +`hof chat` is an experiment in combining LLMs with hof's features +to find where and how they can be used together for the best effect. {{}} -{{}} -We are only at the beginnings of our merging Hof with LLMs. -This page descibes the current state and where we are headed. -{{}} - ## hof chat @@ -32,7 +25,11 @@ The `hof chat` command is and early preview for interacting with hof using natur You can already use this to: 1. Talk with ChatGPT from the command line or vim -2. Talk with Hof data models (full demo coming soon :) +1. Talk with Hof data models (full demo coming soon :) + +_note, you can also call any LLM apis via hof/flow to build complex workflows_ + +
{{}} diff --git a/docs/content/getting-started/modules.md b/docs/content/getting-started/modules.md deleted file mode 100644 index 4bf4de876..000000000 --- a/docs/content/getting-started/modules.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Modules" -description: "Dependency management for CUE and code generators" -brief: "Dependency management for CUE and code generators" - -weight: 20 ---- - -### Hof & CUE Modules - -Every __hof generator__ is also a __CUE module__, -and in fact, many of hof's other features can -be used from the module system too. -Hof & CUE's modules serve the same purpose as other languages, -allowing to to version, share, and reuse code. -"`hof mod`" is the subcommand for working with modules and dependencies. -The implementation is based on Go modules. - -The name of a module should be the same the git repository. -`hof` talks directly to git repositories and many of -`hof`'s commands will accept modules as an input argument too. - -##### [To learn more, see the modules section](/modules/). - -
- -{{}} -# create a new module -hof mod init github.com/hofstadter-io/example - -# add a dependency -hof mod get github.com/hofstadter-io/hof@v0.6.8 - or -hof mod get github.com/hofstadter-io/hof@latest - -# tidy dependencies -hof mod tidy - -# fetch dependencies -hof mod link - or -hof mod vendor -{{}} - diff --git a/docs/content/getting-started/next-steps.md b/docs/content/getting-started/next-steps.md index e4c4be4a5..814238bef 100644 --- a/docs/content/getting-started/next-steps.md +++ b/docs/content/getting-started/next-steps.md @@ -3,6 +3,7 @@ title: "Next Steps" description: "Next steps to learning and using hof" brief: "To learning and using hof" weight: 100 +draft: true --- {{}} diff --git a/docs/content/getting-started/task-engine.md b/docs/content/getting-started/task-engine.md index c5b83913e..ba7e8aff7 100644 --- a/docs/content/getting-started/task-engine.md +++ b/docs/content/getting-started/task-engine.md @@ -4,5 +4,112 @@ description: "Build and run CUE based task workflows" brief: "Build and run CUE based task workflows" weight: 35 -draft: true --- + +{{}} +Hof's __task engine__ allows you to write dynamic workflows with CUE. +Build modular and composable DAGs that can + +- work with files, call APIs, can prompt for user input, and much more +- run scripts and containers, use any language for a custom task +- run a server that calls a workflow per request or bulk process a set of inputs +- create additional task based on the results of other tasks + +Hof's task engine is an extension of cue/flow with + +- extra tasks for key-value cache and advanced structural operations on CUE values +- additional control over parallel execution, synchronization of tasks, and message passing +- allows for workflows along side regular CUE files so they can be imported and shared as modules +{{}} + + +## Overview + + +### Command + +{{}} +run workflows and tasks powered by CUE + +Usage: + hof flow [cue files...] [@flow/name...] [+key=value] [flags] + hof flow [command] + +Aliases: + flow, f + +Available Commands: + list print available flows + +Flags: + -B, --bulk string exprs for inputs to run workflow in bulk mode + -F, --flow stringArray flow labels to match and run + -h, --help help for flow + -P, --parallel int global flow parallelism (default 1) + --progress print task progress as it happens + +Global Flags: + ... +{{}} + +### Tasks & Schemas + +You can find the schema and example for all tasks in +[the hof/flow reference section](/task-engine/tasks/) + +- [api](/task-engine/tasks/api/) + - Call + - Serve +- [`csp`](/task-engine/tasks/csp/) (communicating sequential processes) + - Chan + - Send + - Recv +- [`cue`](/task-engine/tasks/cue/) + - Format (print incomplete to concrete CUE values) +- [`gen`](/task-engine/tasks/gen/) (generate random values) + - Seed + - Now + - Str + - Int + - Float + - Norm + - UUID + - CUID + - Slug +- [`hof`](/task-engine/tasks/hof/) + - Template (render a hof text/template) +- [`kv`](/task-engine/tasks/kv/) + - Mem (in-memory cache) +- [`os`](/task-engine/tasks/os/) + - Exec + - FileLock + - FileUnlock + - Getenv + - Glob + - Mkdir + - ReadFile + - ReadGlobs + - Sleep + - Stdin + - Stdout + - Watch + - WriteFile +- [`prompt`](/task-engine/tasks/prompt/) + - Prompt (interactive user prompts, like creators) +- [`st`](/task-engine/tasks/st/) (structural) + - Mask + - Pick + - Insert + - Replace + - Upsert + - Diff + - Patch + +## Examples + +You can find many examples through hof's codebase and other projects + +- [the hof/flow task reference subsection](/task-engine/tasks/) +- [the hof/flow test directory](https://github.com/hofstadter-io/hof/tree/_dev/flow/testdata) +- [LLM chat examples](https://github.com/hofstadter-io/hof/tree/_dev/flow/chat) +- [event and server based processing](https://github.com/verdverm/streamer-tools) diff --git a/docs/content/reference/_index.md b/docs/content/reference/_index.md deleted file mode 100644 index 44217edf9..000000000 --- a/docs/content/reference/_index.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: "Reference" -weight: 90 ---- - -Reorganize as: - -- ENV VARS -- schemas/... -- cli/... -- projects/... - - -{{< childpages >}} diff --git a/docs/content/reference/harmony/_index.md b/docs/content/reference/harmony/_index.md deleted file mode 100644 index 0b2c994ea..000000000 --- a/docs/content/reference/harmony/_index.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -title: "harmony" -weight: 100 ---- - -{{}} -`harmony` is a framework for -testing a suite of repositories -across versions of dependencies. -Discover errors and regressions -in downstream projects before -releasing new code. -{{}} - -`harmony` makes it easy for both developers and users -to setup and register projects for usage in upstream testing. -Easily test against any commit or working code. -Easily register projects with a small amount of CUE or config. - -Built on [Dagger](https://dagger.io), `harmony` provides - -- simple setup for developers -- simple registration for user projects -- support for any languages or tools -- consistent environment for testing -- simplified version injection -- easily run specific or all cases - -_Note, while `harmony` uses Dagger, it is not required for downstream projects._ - -#### Examples - -- [harmony-cue](https://github.com/hofstadter-io/harmony-cue) for testing CUE base projects - -## Harmony Setup - -`harmony` is just a Dagger Plan, so to setup your project -you only need to add the following file to your project. - -```cue -package main - -import ( - "dagger.io/dagger" - "universe.dagger.io/docker" - - "github.com/hofstadter-io/harmony" - - // import your projects registry - "github.com/username/project/registry" -) - -// A dagger plan is used as the driver for testing -dagger.#Plan - -// add actions from Harmony -actions: harmony.Harmony - -// project specific actions & configuration -actions: { - - // global version config for this harmony - versions: { - go: "1.18" - } - - // the registry of downstream projects - // typically we put this in a subdir and import it - "registry": registry.Registry - - // the image test cases are run in - // typically parametrized so we can change dependencies or versions - // (you can also build any image you want here) - runner: docker.#Pull & { - source: "index.docker.io/golang:\(versions.go)-alpine" - } - - // where downstream project code is checked out - workdir: "/work" -} -``` - -Run registration cases or a single case with dagger: `dagger do [case]`. -Any cases found will be run in parallel. - -Use `./run.py` to run the full suite of registrations and cases sequentially, -or as a convenient way to set dependency versions. - -#### Registry - -You will typically want to provide a subdirectory -for registered projects. You can also provide -short codes to simplify user project registration further. - -Here we add a `_dagger` short code by -including a `schema.cue` in our `registry/` directory. - -```cue -package registry - -import ( - "strings" - - "universe.dagger.io/docker" - "github.com/hofstadter-io/harmony" -) - -// customized Registration schema built on harmony's -Registration: harmony.Registration & { - // add our short codes - cases: [string]: docker.#Run & { - _dagger?: string - if _dagger != _|_ { - command: { - name: "bash" - args: ["-c", _script] - _script: """ - dagger version - dagger project update - dagger do \(_dagger) - """ - } - } - } -} -``` - -Registrations then use with `cases: foo: { _dagger: "foo bar", workdir: "/work" }` - - -## Registration Setup - -To add a new registration for user projects, -add a CUE file to the upstream project. - -```cue -package registry - -// Note the 'Registry: : ...` needs to be unique -Registry: hof: Registration & { - // - remote: "github.com/hofstadter-io/hof" - ref: "main" - - cases: { - // these are docker.#Run from Dagger universe - cli: { - workdir: "/work" - command: { - name: "go" - args: ["build", "./cmd/hof"] - } - flow: { - workdir: "/work" - command: { - name: "go" - args: ["test", "./flow"] - } - } -} -``` - -## Base image using dirty code - -As a developer of a project, -you may wish to run harmony -before committing your code. -To do this, we need to - -- mount the local directory into dagger -- build a runner image from the local code -- mount the local directory into the runner (possibly) - -Refer to [harmony-cue](https://github.com/hofstadter-io/harmony-cue) for an example. -Look at how "local" is used: - -- harmony.cue -- testers/image.cue -- run.py - -The essence is to use CUE guards (if statements) - ---- - -`harmony` was inspired by the idea -of creating a [cue-unity](https://github.com/cue-unity/unity) -powered by [Dagger](https://dagger.io). -The goal of `cue-unity` is to collect community projects -and run them against new CUE language changes. -`cue-unity` was itself inspired by some work -Rob Pike did on Go, for the same purpose -of testing downstream projects using Go's stdlib. - -`harmony` is more generalized and -the CUE specific version is [harmony-cue](https://github.com/hofstadter-io/harmony-cue). - diff --git a/docs/content/reference/hof-create/_index.md b/docs/content/reference/hof-create/_index.md deleted file mode 100644 index c8d1fa2a7..000000000 --- a/docs/content/reference/hof-create/_index.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: "hof / create" -linkTitle: "hof / create" -weight: 15 ---- - -{{}} -`hof/create` is a command for one-line bootstrapping and application blueprints. -Add a Creator to any git repository to provide a "create-react-app" like experience. -{{}} - -{{}} diff --git a/docs/content/reference/hof-datamodel/_index.md b/docs/content/reference/hof-datamodel/_index.md deleted file mode 100644 index d0872ad9b..000000000 --- a/docs/content/reference/hof-datamodel/_index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: "hof / datamodel" -linkTitle: "hof / datamodel" -weight: 20 ---- - - -Data modeling is core to the development process. -As our understanding of the problem changes, -so must our implementation. - -`hof` has a data modeling system where you - -- define you data models in CUE -- generate types, database tables, and API resources -- checkpoint the data model and maintain a history -- have the entire history available for generating migrations and transformations - -{{< childpages >}} diff --git a/docs/content/reference/hof-flow/_index.md b/docs/content/reference/hof-flow/_index.md deleted file mode 100644 index a90ea5322..000000000 --- a/docs/content/reference/hof-flow/_index.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: "hof / flow" -linkTitle: "hof / flow" -weight: 35 ---- - -{{}} -`hof/flow` is a data and task engine -with automatic dependency detection -powered by `cue/flow`. -It has more task types and capabilities. -{{}} - -{{}} -run file(s) through the hof/flow DAG engine - -Use hof/flow to transform data, call APIs, work with DBs, -read and write files, call any program, handle events, -and much more. - -'hof flow' is very similar to 'cue cmd' and built on the same flow engine. -Tasks and dependencies are inferred. -Hof flow has a slightly different interface and more task types. - -Docs: https://docs.hofstadter.io/data-flow - -Example: - - @flow() - - call: { - @task(api.Call) - req: { ... } - resp: { - statusCode: 200 - body: string - } - } - - print: { - @task(os.Stdout) - test: call.resp - } - -Arguments: - cue entrypoints are the same as the cue cli - @path/name is shorthand for -f / --flow should match the @flow(path/name) - +key=value is shorthand for -t / --tags and are the same as CUE injection tags - - arguments can be in any order and mixed - -@flow() indicates a flow entrypoint - you can have many in a file or nested values - you can run one or many with the -f flag - -@task() represents a unit of work in the flow dag - intertask dependencies are autodetected and run appropriately - hof/flow provides many built in task types - you can reuse, combine, and share as CUE modules, packages, and values - -Usage: - hof flow [cue files...] [@flow/name...] [+key=value] [flags] - -Aliases: - flow, f - -Flags: - -d, --docs print pipeline docs - -f, --flow strings flow labels to match and run - -h, --help help for flow - -l, --list list available pipelines - --progress print task progress as it happens - -s, --stats Print final task statistics - -t, --tags strings data tags to inject before run - -Global Flags: - -p, --package string the Cue package context to use during execution - -q, --quiet turn off output and assume defaults at prompts - -v, --verbose int set the verbosity of output -{{< /codeInner >}} - - -## args & flags - -`hof/flow` accepts CUE entrypoints like the other commands. -There is CLI sugar for - -- flows: `@path/name` is sugar for `-f path/name` -- tags: `+key=value` is sugar for `-t key=value` - -Flags: - -- `-f`/`@` is used to select a flow by name in `@flow()` -- `-t`/`+` is used to inject strings into tags `@tag()` -- `-l`/`--list` prints the list of discovered flows -- `-d`/`--docs` prints additional flow details and docs -- `--progress` will print task progress for the events found, pre, & post -- `--stats` will print task times and dependencies at completion - - ---- - -{{< childpages >}} diff --git a/docs/content/reference/hof-flow/commands.md b/docs/content/reference/hof-flow/commands.md deleted file mode 100644 index 1e07ece6a..000000000 --- a/docs/content/reference/hof-flow/commands.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Commands" -description: "for hof flow" -brief: "for hof flow" - -weight: 10 ---- - -{{}} - diff --git a/docs/content/reference/hof-flow/examples/_index.md b/docs/content/reference/hof-flow/examples/_index.md deleted file mode 100644 index 97b34df06..000000000 --- a/docs/content/reference/hof-flow/examples/_index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: "Examples" -linkTitle: "Examples" -weight: 900 ---- - - -This is the examples section (work in progress) - -- https://github.com/hofstadter-io/hof/tree/_dev/flow/testdata -- https://github.com/hofstadter-io/flow-examples -- https://github.com/hofstadter-io/flows -- https://github.com/verdverm/streamer-tools - - -{{< childpages >}} - diff --git a/docs/content/reference/hof-flow/schemas/_index.md b/docs/content/reference/hof-flow/schemas/_index.md deleted file mode 100644 index b948a61ee..000000000 --- a/docs/content/reference/hof-flow/schemas/_index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: "Schemas" -linkTitle: "Schemas" -weight: 1000 ---- - - -{{}} -This is the schema section for `hof/flow` -{{}} - -{{< childpages >}} - diff --git a/docs/content/reference/hof-fmt/_index.md b/docs/content/reference/hof-fmt/_index.md deleted file mode 100644 index 481519bb5..000000000 --- a/docs/content/reference/hof-fmt/_index.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: "hof / fmt" -linkTitle: "hof / fmt" -weight: 40 ---- - -{{}} -`hof/fmt` is a command which will -format any and all languages. -You can create your own formatters as well. -{{}} - - -`hof` needs a code formatter for the languages it generates. -It runs the pre-output through before applying diff and merging -with any custom code you added to output files. -This is simplifies the job of template authors, -but is also required to avoid unnecessary merge conflicts. - -You will need Docker available to use this feature. -Hof will pull and run containers in the background. -You can disable this by setting an environment variable. - -> `HOF_FMT_DISABLED=1` - - -{{}} diff --git a/docs/content/reference/hof-gen/_index.md b/docs/content/reference/hof-gen/_index.md deleted file mode 100644 index 8a3f5445f..000000000 --- a/docs/content/reference/hof-gen/_index.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "hof / gen" -linkTitle: "hof / gen" -weight: 30 ---- - -Code generation is the core reason __hof__ was created. -It's how we turn single-source of truth designs into all the things. -It was needed so that when our datamodel changes, -we could edit just one or two design files -rather than dozens of source code, test files, and database schemas. - -We are working on this concurrently with the datamodels work. -Like that section, we will post links here as they become available. - -The goal here is to emit code from Cue in a general and language agnostic way. - -This is the best example right now: https://github.com/hofstadter-io/hofmod-cli (edited) - -These generators basically use the Go text/template package behind the scenes - -Schemas can be found in [./hof/schema/...](https://github.com/hofstadter-io/hof/tree/_dev/schema) - -Using a DSL on top of Cue will enable more context, and thus capabilities, for code generation. -A DSL is equivalent to the Schema found in the generators, which is the data input to the template rendering - -My intention for the datamodel work is to create a core which can be extended with context by domain schemas and fed into their generators. Another goal is to enable new code generators without modifying hof's code (the binary that you run) -This way, I can have a single data model and feed it into several generators, so that it is a "source of truth" to many outputs (i.e. client, server, sql) - - -Using Cue to specify the schema (DSL), introspect the user's design (using a schema / generator), and deciding what files should be created (gen dirs and templates). The actual process of pumping the data through the templates is in Go (lib/{gen.go,runtime.go,gen/}) - -There's also a shadow directory, so that we can 3-way diff merge between: - -1. previous code -2. next code -3. custom code in the generated output - -It was key for me to be able to write code in the output, change the design, regenerate, and have all things be OK `*` - -`*` changing datamodels may require you to change custom code using it - -There are edge cases around updating the generator and designs at the same time, would like to detect that situation and complain to the user more. - -Renaming certain things is hard too (while keeping custom code) i.e. a file name can change, which will create a new file and delete the old one. Git is your friend :] - -designs/ are filled in schema/ files which are the inputs to gen/ files which do some Cue magic to setup a data struct in hof, which itself just runs the Out field of generators through a process. Mostly Cue logic up front and the text/template logic in the templates/ and partials/ - - -{{< childpages >}} diff --git a/docs/content/reference/hof-gen/adhoc-file-gen.md b/docs/content/reference/hof-gen/adhoc-file-gen.md deleted file mode 100644 index 385504732..000000000 --- a/docs/content/reference/hof-gen/adhoc-file-gen.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Ad-hoc File Gen" -linkTitle: "Ad-hoc File Gen" -weight: 25 ---- - -{{}} -`hof gen` joins CUE with Go's text/template system and diff3. -This section focuses on the ad-hoc, one-liners -you can write to generate any file from any data. -{{}} - -
- -#### Learn about writing templates, with extra functions and helpers - -[Template writing docs](/code-generation/template-writing/) - -
- -#### Check the tests for complete examples - -https://github.com/hofstadter-io/hof/tree/_dev/test/render - -
- -#### Want to use and compose code gen modules and dependencies? - -Create and use generator modules. - -`hof gen app.cue -G frontend -G backend -G migrations` - -See the [first-example](/first-example/) to learn how. - -
- -#### Command Help - -
- -{{}} - -{{< childpages >}} - diff --git a/docs/content/reference/hof-gen/creating-generators.md b/docs/content/reference/hof-gen/creating-generators.md deleted file mode 100644 index 4030e7560..000000000 --- a/docs/content/reference/hof-gen/creating-generators.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: "Creating Generators" -linkTitle: "Creating Generators" -weight: 35 -description: > - Modules for the Hofstadter code generator. ---- - -- overview -- module layout -- schema -- generator -- templates -- subgeneration -- publishing diff --git a/docs/content/reference/hof-gen/using-generators.md b/docs/content/reference/hof-gen/using-generators.md deleted file mode 100644 index 1487399d0..000000000 --- a/docs/content/reference/hof-gen/using-generators.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: "Using Generators" -linkTitle: "Using Generators" -weight: 30 -description: > - Modules for the Hofstadter code generator. ---- - -- project setup -- importing -- designs and datamodels -- generating -- customizing -- iterating - diff --git a/docs/content/reference/hof-mod/_index.md b/docs/content/reference/hof-mod/_index.md deleted file mode 100644 index 9e7bf7d80..000000000 --- a/docs/content/reference/hof-mod/_index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: "hof / mod" -linkTitle: "hof / mod" -weight: 50 ---- - -{{}} -`hof/mod` is a command for managing CUE and Hof modules. -Use it to initialize a new module, set versioned dependencies, -and fetch them with a single command. -{{}} - -{{}} diff --git a/docs/content/reference/hof-tui/_index.md b/docs/content/reference/hof-tui/_index.md deleted file mode 100644 index ce20f371b..000000000 --- a/docs/content/reference/hof-tui/_index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: "hof / tui" -linkTitle: "hof / tui" -weight: 60 ---- - -{{}} -`hof/tui` is a command for working with CUE and Hof Flow dynamically. -{{}} - -{{< youtube id="XNBqBWO4y08" >}} - -
- -![hof tui help](/images/hof-tui-help.png) - diff --git a/docs/content/task-engine/_index.md b/docs/content/task-engine/_index.md index 6159fb94f..f730e98d5 100644 --- a/docs/content/task-engine/_index.md +++ b/docs/content/task-engine/_index.md @@ -21,3 +21,37 @@ as they are ready or needed. {{< childpages >}} + +--- + +{{}} +`hof/flow` is a data and task engine +with automatic dependency detection +powered by `cue/flow`. +It has more task types and capabilities. +{{}} + +{{< childpages >}} + +### Command Help + +
+hof flow -h +{{}} +
+ + +### args & flags + +`hof/flow` accepts CUE entrypoints like the other commands. +There is CLI sugar for + +- flows: `@path/name` is sugar for `-F path/name` +- tags: `+key=value` is sugar for `-t key=value` + +Useful Flags: + +- `-F`/`@` is used to select a flow by name in `@flow()` +- `-t`/`+` is used to inject strings into tags `@tag()` +- `--progress` will print task progress for the events found, pre, & post + diff --git a/docs/content/reference/hof-flow/schemas/middleware/_index.md b/docs/content/task-engine/middleware/_index.md similarity index 100% rename from docs/content/reference/hof-flow/schemas/middleware/_index.md rename to docs/content/task-engine/middleware/_index.md diff --git a/docs/content/reference/hof-flow/schemas/middleware/sync.md b/docs/content/task-engine/middleware/sync.md similarity index 100% rename from docs/content/reference/hof-flow/schemas/middleware/sync.md rename to docs/content/task-engine/middleware/sync.md diff --git a/docs/content/reference/hof-flow/schemas/tasks/_index.md b/docs/content/task-engine/tasks/_index.md similarity index 100% rename from docs/content/reference/hof-flow/schemas/tasks/_index.md rename to docs/content/task-engine/tasks/_index.md diff --git a/docs/content/reference/hof-flow/schemas/tasks/api.md b/docs/content/task-engine/tasks/api.md similarity index 80% rename from docs/content/reference/hof-flow/schemas/tasks/api.md rename to docs/content/task-engine/tasks/api.md index 5c2d95ccf..1a5975be8 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/api.md +++ b/docs/content/task-engine/tasks/api.md @@ -8,7 +8,7 @@ Call an API or run a REST server ### Schema -{{}} +{{}} ### Example: api.Call diff --git a/docs/content/reference/hof-flow/schemas/tasks/csp.md b/docs/content/task-engine/tasks/csp.md similarity index 76% rename from docs/content/reference/hof-flow/schemas/tasks/csp.md rename to docs/content/task-engine/tasks/csp.md index ca2deff07..6830571ce 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/csp.md +++ b/docs/content/task-engine/tasks/csp.md @@ -8,4 +8,4 @@ Concurrency message passing based on Go / Erlang ### Schema -{{}} +{{}} diff --git a/docs/content/task-engine/tasks/cue.md b/docs/content/task-engine/tasks/cue.md new file mode 100644 index 000000000..80f327cce --- /dev/null +++ b/docs/content/task-engine/tasks/cue.md @@ -0,0 +1,7 @@ +--- +title: cue +--- + +### Schema + +{{}} \ No newline at end of file diff --git a/docs/content/reference/hof-flow/schemas/tasks/gen.md b/docs/content/task-engine/tasks/gen.md similarity index 84% rename from docs/content/reference/hof-flow/schemas/tasks/gen.md rename to docs/content/task-engine/tasks/gen.md index c9da948d7..b7fb3c640 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/gen.md +++ b/docs/content/task-engine/tasks/gen.md @@ -10,7 +10,7 @@ Random generators ### Schema -{{}} +{{}} ### Example diff --git a/docs/content/task-engine/tasks/hof.md b/docs/content/task-engine/tasks/hof.md new file mode 100644 index 000000000..79d770a5e --- /dev/null +++ b/docs/content/task-engine/tasks/hof.md @@ -0,0 +1,7 @@ +--- +title: hof +--- + +### Schema + +{{}} \ No newline at end of file diff --git a/docs/content/reference/hof-flow/schemas/tasks/kv.md b/docs/content/task-engine/tasks/kv.md similarity index 72% rename from docs/content/reference/hof-flow/schemas/tasks/kv.md rename to docs/content/task-engine/tasks/kv.md index d877481f4..79fee4782 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/kv.md +++ b/docs/content/task-engine/tasks/kv.md @@ -8,5 +8,5 @@ Key/Value stores ### Schema -{{}} +{{}} diff --git a/docs/content/reference/hof-flow/schemas/tasks/os.md b/docs/content/task-engine/tasks/os.md similarity index 73% rename from docs/content/reference/hof-flow/schemas/tasks/os.md rename to docs/content/task-engine/tasks/os.md index e32e8fe0c..742cfa44e 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/os.md +++ b/docs/content/task-engine/tasks/os.md @@ -8,4 +8,4 @@ Interact with the OS ### Schema -{{}} +{{}} diff --git a/docs/content/task-engine/tasks/prompt.md b/docs/content/task-engine/tasks/prompt.md new file mode 100644 index 000000000..f630716d7 --- /dev/null +++ b/docs/content/task-engine/tasks/prompt.md @@ -0,0 +1,19 @@ +--- +title: prompt +--- + +{{}} +Prompt the user for input with interactive questions +{{}} + +### Schema + +{{}} + +{{}} + +### Example + +{{}} + + diff --git a/docs/content/reference/hof-flow/schemas/tasks/st.md b/docs/content/task-engine/tasks/st.md similarity index 82% rename from docs/content/reference/hof-flow/schemas/tasks/st.md rename to docs/content/task-engine/tasks/st.md index c20560f16..700333707 100644 --- a/docs/content/reference/hof-flow/schemas/tasks/st.md +++ b/docs/content/task-engine/tasks/st.md @@ -8,7 +8,7 @@ Structural helpers ### Schema -{{}} +{{}} ### Example diff --git a/docs/cue.mod/module.cue b/docs/cue.mod/module.cue index 04c8554d1..b37af8f0c 100644 --- a/docs/cue.mod/module.cue +++ b/docs/cue.mod/module.cue @@ -3,7 +3,7 @@ cue: "v0.8.2" require: { "github.com/hofstadter-io/cuelm": "v0.1.1" - "github.com/hofstadter-io/hof": "v0.6.9-rc.1" + "github.com/hofstadter-io/hof": "v0.6.9-rc.2" } indirect: { diff --git a/docs/cue.mod/sums.cue b/docs/cue.mod/sums.cue index 1b43fb944..19e6aa734 100644 --- a/docs/cue.mod/sums.cue +++ b/docs/cue.mod/sums.cue @@ -1,7 +1,7 @@ sums: { "github.com/hofstadter-io/cuelm": "v0.1.1": ["h1:G1tM1PcD9u+1fHjxXAAy8NH5AaRloUYc0DRW+yGBOVA="] "github.com/hofstadter-io/ghacue": "v0.2.0": ["h1:SRQO1Mj2w41v5cgGXYIZ6KP3XRGOv3TaBY0DwG2Slfg="] - "github.com/hofstadter-io/hof": "v0.6.9-rc.1": ["\u003cnever-fetched-only-replaced\u003e"] + "github.com/hofstadter-io/hof": "v0.6.9-rc.2": ["\u003cnever-fetched-only-replaced\u003e"] "github.com/hofstadter-io/hofmod-cli": "v0.9.0": ["h1:RV3IxqbOayB9obhhbI82CtzOU4NQ0x7GLgZYiMZYYis="] "github.com/hofstadter-io/supacode": "v0.0.7": ["h1:Ym52/vGKo7/msqjQ6WVCPjNtlfn9O01KD9JHyL1hE3s="] } diff --git a/docs/hack/make/extern.inc b/docs/hack/make/extern.inc index e3a094e34..5cc99fd75 100644 --- a/docs/hack/make/extern.inc +++ b/docs/hack/make/extern.inc @@ -4,6 +4,7 @@ extern: cmdhelp schemas schemas: # todo, get from github @ version rsync -ra ../schema/* code/hof-schemas + rsync -a --include='*.cue' --include='*/' --exclude='*' ../flow/tasks/* code/hof-schemas/flow/tasks .PHONY: cmdhelp cmdhelp: diff --git a/flow/tasks/api/schema.cue b/flow/tasks/api/schema.cue index f057efb7d..3b9de43d6 100644 --- a/flow/tasks/api/schema.cue +++ b/flow/tasks/api/schema.cue @@ -4,7 +4,6 @@ Method: *"GET" | "POST" | "PUT" | "DELETE" | "OPTIONS" | "HEAD" | "CONNECT" | "T Call: { @task(api.Call) - $task: "api.Call" req: { method: Method @@ -38,7 +37,6 @@ Call: { Serve: { @task(api.Serve) - $task: "api.Serve" port: string quitMailbox: string diff --git a/flow/tasks/csp/schema.cue b/flow/tasks/csp/schema.cue index 99f2c445c..0afbb58b9 100644 --- a/flow/tasks/csp/schema.cue +++ b/flow/tasks/csp/schema.cue @@ -7,7 +7,6 @@ package csp // Chan is a named mailbox Chan: { @task(csp.Chan) - $task: "csp.Chan" // the name of the channel mailbox: string @@ -20,7 +19,6 @@ Chan: { // Send a message to a mailbox Send: { @task(csp.Send) - $task: "csp.Send" // the name of the channel mailbox: string @@ -35,7 +33,6 @@ Send: { // Recv is a coroutine which runs indefinitely Recv: { @task(csp.Recv) - $task: "csp.Recv" // the name of the channel mailbox: string diff --git a/flow/tasks/cue/schema.cue b/flow/tasks/cue/schema.cue index 1f17fd828..e8707b2ea 100644 --- a/flow/tasks/cue/schema.cue +++ b/flow/tasks/cue/schema.cue @@ -2,7 +2,6 @@ package hof Format: { @task(cue.Format) - $task: "cue.Format" value: _ diff --git a/flow/tasks/db/schema.cue b/flow/tasks/db/schema.cue index 24cbe981b..2161f651e 100644 --- a/flow/tasks/db/schema.cue +++ b/flow/tasks/db/schema.cue @@ -9,7 +9,6 @@ package db // Call a database Call: { @task(db.Call) - $task: "db.Call" // db connection conn: { diff --git a/flow/tasks/gen/schema.cue b/flow/tasks/gen/schema.cue index 5698fa0cd..601577a32 100644 --- a/flow/tasks/gen/schema.cue +++ b/flow/tasks/gen/schema.cue @@ -3,7 +3,6 @@ package gen // Seed the Range Seed: { @task(gen.Seed) - $task: "gen.Seed" // only set to ensure consistent output while testing seed?: int // defaults to time.Now() @@ -11,7 +10,6 @@ Seed: { Int: { @task(gen.Int) - $task: "gen.Int" max?: int // max value if set // the random val returned @@ -20,7 +18,6 @@ Int: { Str: { @task(gen.Str) - $task: "gen.Str" // number of runes to generate n: int | *12 @@ -30,10 +27,9 @@ Str: { } // the other tasks don't really have schema or input - -// c: string @task(gen.CUID) // like UUID, but for cloud -// f: float @task(gen.Float) -// n: float @task(gen.Norm) -// n: string @task(gen.Now) // RFC-3339 -// s: string @task(gen.Slug) // related to CUID -// u: string @task(gen.UUID) +c: string @task(gen.CUID) // like UUID, but for cloud +f: float @task(gen.Float) +n: float @task(gen.Norm) +t: string @task(gen.Now) // RFC-3339 +s: string @task(gen.Slug) // related to CUID +u: string @task(gen.UUID) diff --git a/flow/tasks/hof/schema.cue b/flow/tasks/hof/schema.cue index f6138b734..052f06e88 100644 --- a/flow/tasks/hof/schema.cue +++ b/flow/tasks/hof/schema.cue @@ -2,7 +2,6 @@ package hof Tempate: { @task(hof.Template) - $task: "hof.Template" name: string | *"" data: _ diff --git a/flow/tasks/kv/schema.cue b/flow/tasks/kv/schema.cue index f503f5dec..830bb56af 100644 --- a/flow/tasks/kv/schema.cue +++ b/flow/tasks/kv/schema.cue @@ -3,7 +3,6 @@ package kv // in memory cue.Value storage Mem: { @task(kv.Mem) - $task: "kv.Mem" // key to store under key: string diff --git a/flow/tasks/os/schema.cue b/flow/tasks/os/schema.cue index 9a85b6dbf..c374ac03d 100644 --- a/flow/tasks/os/schema.cue +++ b/flow/tasks/os/schema.cue @@ -2,7 +2,6 @@ package os Exec: { @task(os.Exec) - $task: "os.Exec" cmd: string | [string, ...string] @@ -47,7 +46,6 @@ Exec: { // Get a filelock FileLock: { @task(os.FileLock) - $task: "os.FileLock" // lockfile name filename: string @@ -62,7 +60,6 @@ FileLock: { // release a filelock FileUnlock: { @task(os.FileUnlock) - $task: "os.FileUnlock" // lockfile name filename: string @@ -78,7 +75,6 @@ Name: !="" & !~"^[$]" // Getenv gets and parses the specific command line variables. Getenv: { @task(os.Getenv) - $task: "os.Getenv" // if empty, get all {[Name]: Value} @@ -86,7 +82,6 @@ Getenv: { Glob: { @task(os.Glob) - $task: "os.Glob" // glob patterns to match globs: [...string] @@ -98,14 +93,12 @@ Glob: { // acts like 'mkdir -p' Mkdir: { @task(os.Mkdir) - $task: "os.Mkdir" dir: string } ReadFile: { @task(os.ReadFile) - $task: "os.ReadFile" // filename to read filename: string @@ -116,7 +109,6 @@ ReadFile: { Sleep: { @task(os.Sleep) - $task: "os.Sleep" // time.Duration to sleep for duration: string @@ -125,7 +117,6 @@ Sleep: { // read from stdin Stdin: { @task(os.Stdout) - $task: "os.Stdout" // optional message to user before reading input prompt?: string @@ -136,15 +127,13 @@ Stdin: { // print to stdout Stdout: { @task(os.Stdout) - $task: "os.Stdout" // text to write text: string } Watch: { - @task(fs.Watch) - $task: "fs.Watch" + @task(os.Watch) // glob patterns to watch for events globs: [...string] @@ -161,9 +150,8 @@ Watch: { WriteFile: { @task(os.WriteFile) - $task: "os.WriteFile" filename: string contents: string | bytes mode: int | *0o644 -} +} \ No newline at end of file diff --git a/flow/tasks/prompt/schema.cue b/flow/tasks/prompt/schema.cue new file mode 100644 index 000000000..876a0cd00 --- /dev/null +++ b/flow/tasks/prompt/schema.cue @@ -0,0 +1,10 @@ +package prompt + +import ( + schema "github.com/hofstadter-io/hof/schema/prompt" +) + +// Same prompt from creators as a workflow task +Prompt: schema.Prompt & { + @task(prompt.Prompt) +} \ No newline at end of file diff --git a/flow/tasks/st/schema.cue b/flow/tasks/st/schema.cue index d4aa1cf2c..db0015c91 100644 --- a/flow/tasks/st/schema.cue +++ b/flow/tasks/st/schema.cue @@ -2,7 +2,7 @@ package st Mask: { @task(st.Mask) - $task: "st.Mask" + val: _ mask: _ out: _ @@ -10,7 +10,7 @@ Mask: { Pick: { @task(st.Pick) - $task: "st.Pick" + val: _ pick: _ out: _ @@ -18,7 +18,7 @@ Pick: { Insert: { @task(st.Insert) - $task: "st.Insert" + val: _ insert: _ out: _ @@ -26,7 +26,7 @@ Insert: { Replace: { @task(st.Replace) - $task: "st.Replace" + val: _ replace: _ out: _ @@ -34,7 +34,7 @@ Replace: { Upsert: { @task(st.Upsert) - $task: "st.Upsert" + val: _ upsert: _ out: _ @@ -42,7 +42,7 @@ Upsert: { Diff: { @task(st.Diff) - $task: "st.Diff" + orig: _ patch: diff next: _ @@ -51,7 +51,7 @@ Diff: { Patch: { @task(st.Patch) - $task: "st.Patch" + orig: _ patch: _ next: _ diff --git a/schema/hof.cue b/schema/hof.cue index 4cce49106..bbb3e9f5b 100644 --- a/schema/hof.cue +++ b/schema/hof.cue @@ -15,7 +15,6 @@ Hof: { // hof/datamodel datamodel?: { - // define the root of a datamodel root: bool | *false @@ -37,18 +36,16 @@ Hof: { // hof/gen gen?: { + // define the root of a generator root: bool | *false // name of the generator name: string | *"" - - // TODO, do we need this? aren't we... - // determining based on the existence of Create: {} - creator: bool | *false } // hof/flow, used for both flows & tasks flow?: { + // define the root of a workflow root: bool | *false // name of the flow or task @@ -56,14 +53,7 @@ Hof: { // if op is empty, it is a flow value // if op is not empty, it is a task value - // TODO, maybe we make this "flow" for flows? - op: string | *"flow" - } - - chat?: { - root: bool | *false - name: string | *"" - extra: string | *"" + op: string | *"" } } } @@ -82,4 +72,4 @@ Metadata: { } // depreciated -DHof: Hof +DHof: Hof \ No newline at end of file