From 53590815ece1ffa0ef8a763529cc597ac3e2a09f Mon Sep 17 00:00:00 2001 From: Tony Worm Date: Sun, 19 May 2024 12:10:07 -0400 Subject: [PATCH] docs: reorg references into relevant sections; add #hof & @attrs page --- docs/code/hof-schemas/hof.cue | 18 +- docs/content/code-generation/adhoc/_index.md | 1 + .../code-generation/formatting/_index.md | 21 ++ .../commands.md | 0 .../complexity.md | 0 .../composition.md | 0 .../databases.md | 0 .../extending.md | 0 .../generators.md | 0 .../languages.md | 0 .../migrations-and-lenses.md | 0 .../schemas.md | 0 .../snapshots-and-history.md | 0 .../with_cue.md | 0 .../command-help.md | 0 .../concepts.md | 1 + docs/content/getting-started/cue.md | 41 ++++ .../content/getting-started/hof-attributes.md | 16 ++ docs/content/getting-started/modules.md | 44 ---- docs/content/reference/_index.md | 14 -- docs/content/reference/harmony/_index.md | 196 ------------------ docs/content/reference/hof-create/_index.md | 12 -- .../content/reference/hof-datamodel/_index.md | 19 -- docs/content/reference/hof-flow/_index.md | 106 ---------- docs/content/reference/hof-fmt/_index.md | 27 --- docs/content/reference/hof-gen/_index.md | 50 ----- .../reference/hof-gen/adhoc-file-gen.md | 44 ---- .../reference/hof-gen/creating-generators.md | 15 -- .../reference/hof-gen/using-generators.md | 15 -- docs/content/reference/hof-mod/_index.md | 13 -- docs/content/reference/hof-tui/_index.md | 16 -- docs/content/task-engine/_index.md | 103 +++++++++ .../middleware/_index.md | 0 .../middleware/sync.md | 0 .../hof-flow => task-engine}/tasks/_index.md | 0 .../hof-flow => task-engine}/tasks/api.md | 0 .../hof-flow => task-engine}/tasks/csp.md | 0 .../hof-flow => task-engine}/tasks/cue.md | 0 .../hof-flow => task-engine}/tasks/gen.md | 0 .../hof-flow => task-engine}/tasks/hof.md | 0 .../hof-flow => task-engine}/tasks/kv.md | 0 .../hof-flow => task-engine}/tasks/os.md | 0 .../hof-flow => task-engine}/tasks/prompt.md | 0 .../hof-flow => task-engine}/tasks/st.md | 0 schema/hof.cue | 18 +- 45 files changed, 191 insertions(+), 599 deletions(-) rename docs/content/{reference/hof-datamodel => data-modeling}/commands.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/complexity.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/composition.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/databases.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/extending.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/generators.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/languages.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/migrations-and-lenses.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/schemas.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/snapshots-and-history.md (100%) rename docs/content/{reference/hof-datamodel => data-modeling}/with_cue.md (100%) rename docs/content/{reference => getting-started}/command-help.md (100%) rename docs/content/{reference => getting-started}/concepts.md (99%) create mode 100644 docs/content/getting-started/hof-attributes.md delete mode 100644 docs/content/getting-started/modules.md delete mode 100644 docs/content/reference/_index.md delete mode 100644 docs/content/reference/harmony/_index.md delete mode 100644 docs/content/reference/hof-create/_index.md delete mode 100644 docs/content/reference/hof-datamodel/_index.md delete mode 100644 docs/content/reference/hof-flow/_index.md delete mode 100644 docs/content/reference/hof-fmt/_index.md delete mode 100644 docs/content/reference/hof-gen/_index.md delete mode 100644 docs/content/reference/hof-gen/adhoc-file-gen.md delete mode 100644 docs/content/reference/hof-gen/creating-generators.md delete mode 100644 docs/content/reference/hof-gen/using-generators.md delete mode 100644 docs/content/reference/hof-mod/_index.md delete mode 100644 docs/content/reference/hof-tui/_index.md rename docs/content/{reference/hof-flow => task-engine}/middleware/_index.md (100%) rename docs/content/{reference/hof-flow => task-engine}/middleware/sync.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/_index.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/api.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/csp.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/cue.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/gen.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/hof.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/kv.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/os.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/prompt.md (100%) rename docs/content/{reference/hof-flow => task-engine}/tasks/st.md (100%) diff --git a/docs/code/hof-schemas/hof.cue b/docs/code/hof-schemas/hof.cue index 4cce49106..bbb3e9f5b 100644 --- a/docs/code/hof-schemas/hof.cue +++ b/docs/code/hof-schemas/hof.cue @@ -15,7 +15,6 @@ Hof: { // hof/datamodel datamodel?: { - // define the root of a datamodel root: bool | *false @@ -37,18 +36,16 @@ Hof: { // hof/gen gen?: { + // define the root of a generator root: bool | *false // name of the generator name: string | *"" - - // TODO, do we need this? aren't we... - // determining based on the existence of Create: {} - creator: bool | *false } // hof/flow, used for both flows & tasks flow?: { + // define the root of a workflow root: bool | *false // name of the flow or task @@ -56,14 +53,7 @@ Hof: { // if op is empty, it is a flow value // if op is not empty, it is a task value - // TODO, maybe we make this "flow" for flows? - op: string | *"flow" - } - - chat?: { - root: bool | *false - name: string | *"" - extra: string | *"" + op: string | *"" } } } @@ -82,4 +72,4 @@ Metadata: { } // depreciated -DHof: Hof +DHof: Hof \ No newline at end of file diff --git a/docs/content/code-generation/adhoc/_index.md b/docs/content/code-generation/adhoc/_index.md index 5341ccff2..ce4822851 100644 --- a/docs/content/code-generation/adhoc/_index.md +++ b/docs/content/code-generation/adhoc/_index.md @@ -1,6 +1,7 @@ --- Title: Ad-hoc Code Gen +draft: true weight: 59 --- diff --git a/docs/content/code-generation/formatting/_index.md b/docs/content/code-generation/formatting/_index.md index a2bf0522c..0df435631 100644 --- a/docs/content/code-generation/formatting/_index.md +++ b/docs/content/code-generation/formatting/_index.md @@ -26,6 +26,27 @@ see the [custom formatters](/code-generation/formatting/custom-formatters/) sect hof fmt ``` +{{}} +`hof/fmt` is a command which will +format any and all languages. +You can create your own formatters as well. +{{}} + + +`hof` needs a code formatter for the languages it generates. +It runs the pre-output through before applying diff and merging +with any custom code you added to output files. +This is simplifies the job of template authors, +but is also required to avoid unnecessary merge conflicts. + +You will need Docker available to use this feature. +Hof will pull and run containers in the background. +You can disable this by setting an environment variable. + +> `HOF_FMT_DISABLED=1` + + +{{}} ## Supported Languages diff --git a/docs/content/reference/hof-datamodel/commands.md b/docs/content/data-modeling/commands.md similarity index 100% rename from docs/content/reference/hof-datamodel/commands.md rename to docs/content/data-modeling/commands.md diff --git a/docs/content/reference/hof-datamodel/complexity.md b/docs/content/data-modeling/complexity.md similarity index 100% rename from docs/content/reference/hof-datamodel/complexity.md rename to docs/content/data-modeling/complexity.md diff --git a/docs/content/reference/hof-datamodel/composition.md b/docs/content/data-modeling/composition.md similarity index 100% rename from docs/content/reference/hof-datamodel/composition.md rename to docs/content/data-modeling/composition.md diff --git a/docs/content/reference/hof-datamodel/databases.md b/docs/content/data-modeling/databases.md similarity index 100% rename from docs/content/reference/hof-datamodel/databases.md rename to docs/content/data-modeling/databases.md diff --git a/docs/content/reference/hof-datamodel/extending.md b/docs/content/data-modeling/extending.md similarity index 100% rename from docs/content/reference/hof-datamodel/extending.md rename to docs/content/data-modeling/extending.md diff --git a/docs/content/reference/hof-datamodel/generators.md b/docs/content/data-modeling/generators.md similarity index 100% rename from docs/content/reference/hof-datamodel/generators.md rename to docs/content/data-modeling/generators.md diff --git a/docs/content/reference/hof-datamodel/languages.md b/docs/content/data-modeling/languages.md similarity index 100% rename from docs/content/reference/hof-datamodel/languages.md rename to docs/content/data-modeling/languages.md diff --git a/docs/content/reference/hof-datamodel/migrations-and-lenses.md b/docs/content/data-modeling/migrations-and-lenses.md similarity index 100% rename from docs/content/reference/hof-datamodel/migrations-and-lenses.md rename to docs/content/data-modeling/migrations-and-lenses.md diff --git a/docs/content/reference/hof-datamodel/schemas.md b/docs/content/data-modeling/schemas.md similarity index 100% rename from docs/content/reference/hof-datamodel/schemas.md rename to docs/content/data-modeling/schemas.md diff --git a/docs/content/reference/hof-datamodel/snapshots-and-history.md b/docs/content/data-modeling/snapshots-and-history.md similarity index 100% rename from docs/content/reference/hof-datamodel/snapshots-and-history.md rename to docs/content/data-modeling/snapshots-and-history.md diff --git a/docs/content/reference/hof-datamodel/with_cue.md b/docs/content/data-modeling/with_cue.md similarity index 100% rename from docs/content/reference/hof-datamodel/with_cue.md rename to docs/content/data-modeling/with_cue.md diff --git a/docs/content/reference/command-help.md b/docs/content/getting-started/command-help.md similarity index 100% rename from docs/content/reference/command-help.md rename to docs/content/getting-started/command-help.md diff --git a/docs/content/reference/concepts.md b/docs/content/getting-started/concepts.md similarity index 99% rename from docs/content/reference/concepts.md rename to docs/content/getting-started/concepts.md index f65883aff..f66f8e8e9 100644 --- a/docs/content/reference/concepts.md +++ b/docs/content/getting-started/concepts.md @@ -3,6 +3,7 @@ title: "Concepts" description: "Overview of the main concepts in Hofstadter around code generation, source-of-truth, reusability." brief: "in Hofstadter, a quick overview" +draft: true weight: 90 --- diff --git a/docs/content/getting-started/cue.md b/docs/content/getting-started/cue.md index 53062fe28..a4e7372bf 100644 --- a/docs/content/getting-started/cue.md +++ b/docs/content/getting-started/cue.md @@ -47,3 +47,44 @@ To further your CUE knowledge, be sure to check out these resources: - [CUE documentation](https://cuelang.org) (from the CUE Team) - [Cuetorials](https://cuetorials.com) (by Hofstadter) + + +### Hof & CUE Modules + +__hof__ has a preview version for __CUE modules__. +Hof & CUE's modules serve the same purpose as other languages, +allowing to to version, share, and reuse code. +CUE's module system is still largely experimental. +We will eventually migrate over once sufficient features +are in place, at which point we will provide automation to update. + +Most of hof's features can be used from the module system. +"`hof mod`" is the subcommand for working with modules and dependencies. +The implementation is based on Go modules. + +The name of a module should be the same the git repository. +`hof` talks directly to git repositories and many of +`hof`'s commands will accept modules as an input argument too. +There is also support for OCI based repositories. + +##### [To learn more, see the modules section](/modules/). + +
+ +{{}} +# create a new module +hof mod init github.com/hofstadter-io/example + +# add a dependency +hof mod get github.com/hofstadter-io/hof@v0.6.8 + or +hof mod get github.com/hofstadter-io/hof@latest + +# tidy dependencies +hof mod tidy + +# fetch dependencies +hof mod link + or +hof mod vendor +{{}} diff --git a/docs/content/getting-started/hof-attributes.md b/docs/content/getting-started/hof-attributes.md new file mode 100644 index 000000000..ba91d8ddf --- /dev/null +++ b/docs/content/getting-started/hof-attributes.md @@ -0,0 +1,16 @@ +--- +title: "#hof & @attributes" + +weight: 70 +--- + +{{}} +Hof denotes major features with a `#hof` definition +and has several attributes you can use as shorthand. +{{}} + +- @gen +- @flow +- @datamodel + +{{< codePane title="hof/schema.Hof" file="code/hof-schemas/hof.html" >}} diff --git a/docs/content/getting-started/modules.md b/docs/content/getting-started/modules.md deleted file mode 100644 index 4bf4de876..000000000 --- a/docs/content/getting-started/modules.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Modules" -description: "Dependency management for CUE and code generators" -brief: "Dependency management for CUE and code generators" - -weight: 20 ---- - -### Hof & CUE Modules - -Every __hof generator__ is also a __CUE module__, -and in fact, many of hof's other features can -be used from the module system too. -Hof & CUE's modules serve the same purpose as other languages, -allowing to to version, share, and reuse code. -"`hof mod`" is the subcommand for working with modules and dependencies. -The implementation is based on Go modules. - -The name of a module should be the same the git repository. -`hof` talks directly to git repositories and many of -`hof`'s commands will accept modules as an input argument too. - -##### [To learn more, see the modules section](/modules/). - -
- -{{}} -# create a new module -hof mod init github.com/hofstadter-io/example - -# add a dependency -hof mod get github.com/hofstadter-io/hof@v0.6.8 - or -hof mod get github.com/hofstadter-io/hof@latest - -# tidy dependencies -hof mod tidy - -# fetch dependencies -hof mod link - or -hof mod vendor -{{}} - diff --git a/docs/content/reference/_index.md b/docs/content/reference/_index.md deleted file mode 100644 index 44217edf9..000000000 --- a/docs/content/reference/_index.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: "Reference" -weight: 90 ---- - -Reorganize as: - -- ENV VARS -- schemas/... -- cli/... -- projects/... - - -{{< childpages >}} diff --git a/docs/content/reference/harmony/_index.md b/docs/content/reference/harmony/_index.md deleted file mode 100644 index 0b2c994ea..000000000 --- a/docs/content/reference/harmony/_index.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -title: "harmony" -weight: 100 ---- - -{{}} -`harmony` is a framework for -testing a suite of repositories -across versions of dependencies. -Discover errors and regressions -in downstream projects before -releasing new code. -{{}} - -`harmony` makes it easy for both developers and users -to setup and register projects for usage in upstream testing. -Easily test against any commit or working code. -Easily register projects with a small amount of CUE or config. - -Built on [Dagger](https://dagger.io), `harmony` provides - -- simple setup for developers -- simple registration for user projects -- support for any languages or tools -- consistent environment for testing -- simplified version injection -- easily run specific or all cases - -_Note, while `harmony` uses Dagger, it is not required for downstream projects._ - -#### Examples - -- [harmony-cue](https://github.com/hofstadter-io/harmony-cue) for testing CUE base projects - -## Harmony Setup - -`harmony` is just a Dagger Plan, so to setup your project -you only need to add the following file to your project. - -```cue -package main - -import ( - "dagger.io/dagger" - "universe.dagger.io/docker" - - "github.com/hofstadter-io/harmony" - - // import your projects registry - "github.com/username/project/registry" -) - -// A dagger plan is used as the driver for testing -dagger.#Plan - -// add actions from Harmony -actions: harmony.Harmony - -// project specific actions & configuration -actions: { - - // global version config for this harmony - versions: { - go: "1.18" - } - - // the registry of downstream projects - // typically we put this in a subdir and import it - "registry": registry.Registry - - // the image test cases are run in - // typically parametrized so we can change dependencies or versions - // (you can also build any image you want here) - runner: docker.#Pull & { - source: "index.docker.io/golang:\(versions.go)-alpine" - } - - // where downstream project code is checked out - workdir: "/work" -} -``` - -Run registration cases or a single case with dagger: `dagger do [case]`. -Any cases found will be run in parallel. - -Use `./run.py` to run the full suite of registrations and cases sequentially, -or as a convenient way to set dependency versions. - -#### Registry - -You will typically want to provide a subdirectory -for registered projects. You can also provide -short codes to simplify user project registration further. - -Here we add a `_dagger` short code by -including a `schema.cue` in our `registry/` directory. - -```cue -package registry - -import ( - "strings" - - "universe.dagger.io/docker" - "github.com/hofstadter-io/harmony" -) - -// customized Registration schema built on harmony's -Registration: harmony.Registration & { - // add our short codes - cases: [string]: docker.#Run & { - _dagger?: string - if _dagger != _|_ { - command: { - name: "bash" - args: ["-c", _script] - _script: """ - dagger version - dagger project update - dagger do \(_dagger) - """ - } - } - } -} -``` - -Registrations then use with `cases: foo: { _dagger: "foo bar", workdir: "/work" }` - - -## Registration Setup - -To add a new registration for user projects, -add a CUE file to the upstream project. - -```cue -package registry - -// Note the 'Registry: : ...` needs to be unique -Registry: hof: Registration & { - // - remote: "github.com/hofstadter-io/hof" - ref: "main" - - cases: { - // these are docker.#Run from Dagger universe - cli: { - workdir: "/work" - command: { - name: "go" - args: ["build", "./cmd/hof"] - } - flow: { - workdir: "/work" - command: { - name: "go" - args: ["test", "./flow"] - } - } -} -``` - -## Base image using dirty code - -As a developer of a project, -you may wish to run harmony -before committing your code. -To do this, we need to - -- mount the local directory into dagger -- build a runner image from the local code -- mount the local directory into the runner (possibly) - -Refer to [harmony-cue](https://github.com/hofstadter-io/harmony-cue) for an example. -Look at how "local" is used: - -- harmony.cue -- testers/image.cue -- run.py - -The essence is to use CUE guards (if statements) - ---- - -`harmony` was inspired by the idea -of creating a [cue-unity](https://github.com/cue-unity/unity) -powered by [Dagger](https://dagger.io). -The goal of `cue-unity` is to collect community projects -and run them against new CUE language changes. -`cue-unity` was itself inspired by some work -Rob Pike did on Go, for the same purpose -of testing downstream projects using Go's stdlib. - -`harmony` is more generalized and -the CUE specific version is [harmony-cue](https://github.com/hofstadter-io/harmony-cue). - diff --git a/docs/content/reference/hof-create/_index.md b/docs/content/reference/hof-create/_index.md deleted file mode 100644 index c8d1fa2a7..000000000 --- a/docs/content/reference/hof-create/_index.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -title: "hof / create" -linkTitle: "hof / create" -weight: 15 ---- - -{{}} -`hof/create` is a command for one-line bootstrapping and application blueprints. -Add a Creator to any git repository to provide a "create-react-app" like experience. -{{}} - -{{}} diff --git a/docs/content/reference/hof-datamodel/_index.md b/docs/content/reference/hof-datamodel/_index.md deleted file mode 100644 index d0872ad9b..000000000 --- a/docs/content/reference/hof-datamodel/_index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: "hof / datamodel" -linkTitle: "hof / datamodel" -weight: 20 ---- - - -Data modeling is core to the development process. -As our understanding of the problem changes, -so must our implementation. - -`hof` has a data modeling system where you - -- define you data models in CUE -- generate types, database tables, and API resources -- checkpoint the data model and maintain a history -- have the entire history available for generating migrations and transformations - -{{< childpages >}} diff --git a/docs/content/reference/hof-flow/_index.md b/docs/content/reference/hof-flow/_index.md deleted file mode 100644 index f2ca18f91..000000000 --- a/docs/content/reference/hof-flow/_index.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: "hof / flow" -linkTitle: "hof / flow" -weight: 35 ---- - -{{}} -`hof/flow` is a data and task engine -with automatic dependency detection -powered by `cue/flow`. -It has more task types and capabilities. -{{}} - -{{< childpages >}} - -### Command Help - -
-hof flow -h -{{}} -run file(s) through the hof/flow DAG engine - -Use hof/flow to transform data, call APIs, work with DBs, -read and write files, call any program, handle events, -and much more. - -'hof flow' is very similar to 'cue cmd' and built on the same flow engine. -Tasks and dependencies are inferred. -Hof flow has a slightly different interface and more task types. - -Docs: https://docs.hofstadter.io/data-flow - -Example: - - @flow() - - call: { - @task(api.Call) - req: { ... } - resp: { - statusCode: 200 - body: string - } - } - - print: { - @task(os.Stdout) - test: call.resp - } - -Arguments: - cue entrypoints are the same as the cue cli - @path/name is shorthand for -f / --flow should match the @flow(path/name) - +key=value is shorthand for -t / --tags and are the same as CUE injection tags - - arguments can be in any order and mixed - -@flow() indicates a flow entrypoint - you can have many in a file or nested values - you can run one or many with the -f flag - -@task() represents a unit of work in the flow dag - intertask dependencies are autodetected and run appropriately - hof/flow provides many built in task types - you can reuse, combine, and share as CUE modules, packages, and values - -Usage: - hof flow [cue files...] [@flow/name...] [+key=value] [flags] - -Aliases: - flow, f - -Flags: - -d, --docs print pipeline docs - -f, --flow strings flow labels to match and run - -h, --help help for flow - -l, --list list available pipelines - --progress print task progress as it happens - -s, --stats Print final task statistics - -t, --tags strings data tags to inject before run - -Global Flags: - -p, --package string the Cue package context to use during execution - -q, --quiet turn off output and assume defaults at prompts - -v, --verbose int set the verbosity of output -{{< /codeInner >}} -
- - -### args & flags - -`hof/flow` accepts CUE entrypoints like the other commands. -There is CLI sugar for - -- flows: `@path/name` is sugar for `-f path/name` -- tags: `+key=value` is sugar for `-t key=value` - -Flags: - -- `-f`/`@` is used to select a flow by name in `@flow()` -- `-t`/`+` is used to inject strings into tags `@tag()` -- `-l`/`--list` prints the list of discovered flows -- `-d`/`--docs` prints additional flow details and docs -- `--progress` will print task progress for the events found, pre, & post -- `--stats` will print task times and dependencies at completion - diff --git a/docs/content/reference/hof-fmt/_index.md b/docs/content/reference/hof-fmt/_index.md deleted file mode 100644 index 481519bb5..000000000 --- a/docs/content/reference/hof-fmt/_index.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -title: "hof / fmt" -linkTitle: "hof / fmt" -weight: 40 ---- - -{{}} -`hof/fmt` is a command which will -format any and all languages. -You can create your own formatters as well. -{{}} - - -`hof` needs a code formatter for the languages it generates. -It runs the pre-output through before applying diff and merging -with any custom code you added to output files. -This is simplifies the job of template authors, -but is also required to avoid unnecessary merge conflicts. - -You will need Docker available to use this feature. -Hof will pull and run containers in the background. -You can disable this by setting an environment variable. - -> `HOF_FMT_DISABLED=1` - - -{{}} diff --git a/docs/content/reference/hof-gen/_index.md b/docs/content/reference/hof-gen/_index.md deleted file mode 100644 index 8a3f5445f..000000000 --- a/docs/content/reference/hof-gen/_index.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "hof / gen" -linkTitle: "hof / gen" -weight: 30 ---- - -Code generation is the core reason __hof__ was created. -It's how we turn single-source of truth designs into all the things. -It was needed so that when our datamodel changes, -we could edit just one or two design files -rather than dozens of source code, test files, and database schemas. - -We are working on this concurrently with the datamodels work. -Like that section, we will post links here as they become available. - -The goal here is to emit code from Cue in a general and language agnostic way. - -This is the best example right now: https://github.com/hofstadter-io/hofmod-cli (edited) - -These generators basically use the Go text/template package behind the scenes - -Schemas can be found in [./hof/schema/...](https://github.com/hofstadter-io/hof/tree/_dev/schema) - -Using a DSL on top of Cue will enable more context, and thus capabilities, for code generation. -A DSL is equivalent to the Schema found in the generators, which is the data input to the template rendering - -My intention for the datamodel work is to create a core which can be extended with context by domain schemas and fed into their generators. Another goal is to enable new code generators without modifying hof's code (the binary that you run) -This way, I can have a single data model and feed it into several generators, so that it is a "source of truth" to many outputs (i.e. client, server, sql) - - -Using Cue to specify the schema (DSL), introspect the user's design (using a schema / generator), and deciding what files should be created (gen dirs and templates). The actual process of pumping the data through the templates is in Go (lib/{gen.go,runtime.go,gen/}) - -There's also a shadow directory, so that we can 3-way diff merge between: - -1. previous code -2. next code -3. custom code in the generated output - -It was key for me to be able to write code in the output, change the design, regenerate, and have all things be OK `*` - -`*` changing datamodels may require you to change custom code using it - -There are edge cases around updating the generator and designs at the same time, would like to detect that situation and complain to the user more. - -Renaming certain things is hard too (while keeping custom code) i.e. a file name can change, which will create a new file and delete the old one. Git is your friend :] - -designs/ are filled in schema/ files which are the inputs to gen/ files which do some Cue magic to setup a data struct in hof, which itself just runs the Out field of generators through a process. Mostly Cue logic up front and the text/template logic in the templates/ and partials/ - - -{{< childpages >}} diff --git a/docs/content/reference/hof-gen/adhoc-file-gen.md b/docs/content/reference/hof-gen/adhoc-file-gen.md deleted file mode 100644 index 385504732..000000000 --- a/docs/content/reference/hof-gen/adhoc-file-gen.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: "Ad-hoc File Gen" -linkTitle: "Ad-hoc File Gen" -weight: 25 ---- - -{{}} -`hof gen` joins CUE with Go's text/template system and diff3. -This section focuses on the ad-hoc, one-liners -you can write to generate any file from any data. -{{}} - -
- -#### Learn about writing templates, with extra functions and helpers - -[Template writing docs](/code-generation/template-writing/) - -
- -#### Check the tests for complete examples - -https://github.com/hofstadter-io/hof/tree/_dev/test/render - -
- -#### Want to use and compose code gen modules and dependencies? - -Create and use generator modules. - -`hof gen app.cue -G frontend -G backend -G migrations` - -See the [first-example](/first-example/) to learn how. - -
- -#### Command Help - -
- -{{}} - -{{< childpages >}} - diff --git a/docs/content/reference/hof-gen/creating-generators.md b/docs/content/reference/hof-gen/creating-generators.md deleted file mode 100644 index 4030e7560..000000000 --- a/docs/content/reference/hof-gen/creating-generators.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: "Creating Generators" -linkTitle: "Creating Generators" -weight: 35 -description: > - Modules for the Hofstadter code generator. ---- - -- overview -- module layout -- schema -- generator -- templates -- subgeneration -- publishing diff --git a/docs/content/reference/hof-gen/using-generators.md b/docs/content/reference/hof-gen/using-generators.md deleted file mode 100644 index 1487399d0..000000000 --- a/docs/content/reference/hof-gen/using-generators.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: "Using Generators" -linkTitle: "Using Generators" -weight: 30 -description: > - Modules for the Hofstadter code generator. ---- - -- project setup -- importing -- designs and datamodels -- generating -- customizing -- iterating - diff --git a/docs/content/reference/hof-mod/_index.md b/docs/content/reference/hof-mod/_index.md deleted file mode 100644 index 9e7bf7d80..000000000 --- a/docs/content/reference/hof-mod/_index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: "hof / mod" -linkTitle: "hof / mod" -weight: 50 ---- - -{{}} -`hof/mod` is a command for managing CUE and Hof modules. -Use it to initialize a new module, set versioned dependencies, -and fetch them with a single command. -{{}} - -{{}} diff --git a/docs/content/reference/hof-tui/_index.md b/docs/content/reference/hof-tui/_index.md deleted file mode 100644 index ce20f371b..000000000 --- a/docs/content/reference/hof-tui/_index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: "hof / tui" -linkTitle: "hof / tui" -weight: 60 ---- - -{{}} -`hof/tui` is a command for working with CUE and Hof Flow dynamically. -{{}} - -{{< youtube id="XNBqBWO4y08" >}} - -
- -![hof tui help](/images/hof-tui-help.png) - diff --git a/docs/content/task-engine/_index.md b/docs/content/task-engine/_index.md index 6159fb94f..7ba55d017 100644 --- a/docs/content/task-engine/_index.md +++ b/docs/content/task-engine/_index.md @@ -21,3 +21,106 @@ as they are ready or needed. {{< childpages >}} + +--- + +{{}} +`hof/flow` is a data and task engine +with automatic dependency detection +powered by `cue/flow`. +It has more task types and capabilities. +{{}} + +{{< childpages >}} + +### Command Help + +
+hof flow -h +{{}} +run file(s) through the hof/flow DAG engine + +Use hof/flow to transform data, call APIs, work with DBs, +read and write files, call any program, handle events, +and much more. + +'hof flow' is very similar to 'cue cmd' and built on the same flow engine. +Tasks and dependencies are inferred. +Hof flow has a slightly different interface and more task types. + +Docs: https://docs.hofstadter.io/data-flow + +Example: + + @flow() + + call: { + @task(api.Call) + req: { ... } + resp: { + statusCode: 200 + body: string + } + } + + print: { + @task(os.Stdout) + test: call.resp + } + +Arguments: + cue entrypoints are the same as the cue cli + @path/name is shorthand for -f / --flow should match the @flow(path/name) + +key=value is shorthand for -t / --tags and are the same as CUE injection tags + + arguments can be in any order and mixed + +@flow() indicates a flow entrypoint + you can have many in a file or nested values + you can run one or many with the -f flag + +@task() represents a unit of work in the flow dag + intertask dependencies are autodetected and run appropriately + hof/flow provides many built in task types + you can reuse, combine, and share as CUE modules, packages, and values + +Usage: + hof flow [cue files...] [@flow/name...] [+key=value] [flags] + +Aliases: + flow, f + +Flags: + -d, --docs print pipeline docs + -f, --flow strings flow labels to match and run + -h, --help help for flow + -l, --list list available pipelines + --progress print task progress as it happens + -s, --stats Print final task statistics + -t, --tags strings data tags to inject before run + +Global Flags: + -p, --package string the Cue package context to use during execution + -q, --quiet turn off output and assume defaults at prompts + -v, --verbose int set the verbosity of output +{{< /codeInner >}} +
+ + +### args & flags + +`hof/flow` accepts CUE entrypoints like the other commands. +There is CLI sugar for + +- flows: `@path/name` is sugar for `-f path/name` +- tags: `+key=value` is sugar for `-t key=value` + +Flags: + +- `-f`/`@` is used to select a flow by name in `@flow()` +- `-t`/`+` is used to inject strings into tags `@tag()` +- `-l`/`--list` prints the list of discovered flows +- `-d`/`--docs` prints additional flow details and docs +- `--progress` will print task progress for the events found, pre, & post +- `--stats` will print task times and dependencies at completion + diff --git a/docs/content/reference/hof-flow/middleware/_index.md b/docs/content/task-engine/middleware/_index.md similarity index 100% rename from docs/content/reference/hof-flow/middleware/_index.md rename to docs/content/task-engine/middleware/_index.md diff --git a/docs/content/reference/hof-flow/middleware/sync.md b/docs/content/task-engine/middleware/sync.md similarity index 100% rename from docs/content/reference/hof-flow/middleware/sync.md rename to docs/content/task-engine/middleware/sync.md diff --git a/docs/content/reference/hof-flow/tasks/_index.md b/docs/content/task-engine/tasks/_index.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/_index.md rename to docs/content/task-engine/tasks/_index.md diff --git a/docs/content/reference/hof-flow/tasks/api.md b/docs/content/task-engine/tasks/api.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/api.md rename to docs/content/task-engine/tasks/api.md diff --git a/docs/content/reference/hof-flow/tasks/csp.md b/docs/content/task-engine/tasks/csp.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/csp.md rename to docs/content/task-engine/tasks/csp.md diff --git a/docs/content/reference/hof-flow/tasks/cue.md b/docs/content/task-engine/tasks/cue.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/cue.md rename to docs/content/task-engine/tasks/cue.md diff --git a/docs/content/reference/hof-flow/tasks/gen.md b/docs/content/task-engine/tasks/gen.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/gen.md rename to docs/content/task-engine/tasks/gen.md diff --git a/docs/content/reference/hof-flow/tasks/hof.md b/docs/content/task-engine/tasks/hof.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/hof.md rename to docs/content/task-engine/tasks/hof.md diff --git a/docs/content/reference/hof-flow/tasks/kv.md b/docs/content/task-engine/tasks/kv.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/kv.md rename to docs/content/task-engine/tasks/kv.md diff --git a/docs/content/reference/hof-flow/tasks/os.md b/docs/content/task-engine/tasks/os.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/os.md rename to docs/content/task-engine/tasks/os.md diff --git a/docs/content/reference/hof-flow/tasks/prompt.md b/docs/content/task-engine/tasks/prompt.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/prompt.md rename to docs/content/task-engine/tasks/prompt.md diff --git a/docs/content/reference/hof-flow/tasks/st.md b/docs/content/task-engine/tasks/st.md similarity index 100% rename from docs/content/reference/hof-flow/tasks/st.md rename to docs/content/task-engine/tasks/st.md diff --git a/schema/hof.cue b/schema/hof.cue index 4cce49106..bbb3e9f5b 100644 --- a/schema/hof.cue +++ b/schema/hof.cue @@ -15,7 +15,6 @@ Hof: { // hof/datamodel datamodel?: { - // define the root of a datamodel root: bool | *false @@ -37,18 +36,16 @@ Hof: { // hof/gen gen?: { + // define the root of a generator root: bool | *false // name of the generator name: string | *"" - - // TODO, do we need this? aren't we... - // determining based on the existence of Create: {} - creator: bool | *false } // hof/flow, used for both flows & tasks flow?: { + // define the root of a workflow root: bool | *false // name of the flow or task @@ -56,14 +53,7 @@ Hof: { // if op is empty, it is a flow value // if op is not empty, it is a task value - // TODO, maybe we make this "flow" for flows? - op: string | *"flow" - } - - chat?: { - root: bool | *false - name: string | *"" - extra: string | *"" + op: string | *"" } } } @@ -82,4 +72,4 @@ Metadata: { } // depreciated -DHof: Hof +DHof: Hof \ No newline at end of file