docs: update readme with better examples

divnix · Jun 29, 2023 · dfaca3f · dfaca3f
1 parent c2ca7a2
commit dfaca3f
Show file tree

Hide file tree

Showing 2 changed files with 194 additions and 103 deletions.
diff --git a/README.md b/README.md
@@ -1,155 +1,246 @@
-# Usage
+# Standard Action
 
-- Works with https://github.com/divnix/std
-- Since GitHub CI doesn't support yaml anchors, explode your file with: `yq '. | explode(.)' ci.raw.yaml > ci.yaml`
-- To set up AWS Credentials for an S3 Cache, find details [here](https://github.com/aws-actions/configure-aws-credentials)
-- **Warning:** This is still under active development and testing. You're likely better off waiting a little while, still.
-  - But it's already being used with success :smile:
+_for [Standard] & [Paisano]_.
+
+[Paisano]: https://github.com/paisano-nix
+[Standard]: https://github.com/divnix/std
+
+Don't waste any time on extra work. Use Standard Action to automatically
+detect CI targets that need re-doing; implemented on top of familiar GH Actions.
+
+## Features
+
+- Evaluate once and distribute final build instructions to workers
+- Once configured, `discovery` picks up new targets automatically
+- Optional `proviso` script can detect if work needs to be done
+
+> **Note on `proviso`**: one example is the oci block type which
+> [checks if the image] is already in the registry and only schedules
+> a build if its missing. If `proviso` queries private remote state
+> then the `discovery` environment must provide all authentication
+> prior to running the discovery step.
+
+[checks if the image]: https://github.com/divnix/std/blob/main/src/std/fwlib/blockTypes/containers-proviso.sh
+
+## Usage
+
+**Minimumn nix version `v2.16.1`**
+
+Tip! Since GitHub CI doesn't support `yaml` anchors, explode your file with:
+
+```
+yq '. | explode(.)' ci.raw.yaml > ci.yaml
+```
+
+### Standalone
+
+```nix
+{
+  /* ... */
+  outputs = {std, ...}@inputs: std.growOn {
+  /* ... */
+    cellBlocks = with std.blockTypes; [
+      (installables "packages" {ci.build = true;})
+      (containers "oci-images" {ci.publish = true;})
+      (kubectl "deployments" {ci.apply = true;})
+    ];
+  /* ... */
+  };
+}
+```
+
+<details><summary><h4>GH Action file</h4></summary>
 
 ```yaml
-# .github/workflows/ci.yml
-name: Standard CI
+# yq '. | explode(.)' this.yml > .github/workflows/std.yml
+name: CI/CD
 
 on:
+  pull_request:
+    branches:
+      - main
   push:
     branches:
       - main
-  workflow_dispatch:
 
 permissions:
+  id-token: write
   contents: read
 
+concurrency:
+  group: std-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   discover:
     outputs:
       hits: ${{ steps.discovery.outputs.hits }}
-      nix_conf: ${{ steps.discovery.outputs.nix_conf }}
-
     runs-on: ubuntu-latest
-    concurrency:
-      group: ${{ github.workflow }}
     steps:
-      - name: Standard Discovery
-        uses: divnix/std-action/discover@main
-        id: discovery
+      # Important: use this as it also detects flake configuration
+      - uses: blaggacao/nix-quick-install-action@detect-nix-flakes-config
+      # if you want to use nixbuild
+      - uses: nixbuild/nixbuild-action@v17
         with:
-          github_pat: ${{ secrets.HUB_PAT }}
+          nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }}
+          generate_summary_for: job
+      # significantly speeds up things in small projects
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - uses: divnix/std-action/discover@main
+        id: discovery
 
-  build-packages: &run-job
+  build: &job
     needs: discover
+    name: ${{ matrix.target.jobName }}
+    runs-on: ubuntu-latest
+    if: fromJSON(needs.discover.outputs.hits).packages.build != '{}'
     strategy:
       matrix:
         target: ${{ fromJSON(needs.discover.outputs.hits).packages.build }}
-    name: ${{ matrix.target.cell }} - ${{ matrix.target.name }}
-    runs-on: ubuntu-latest
     steps:
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1-node16
+      # Important: use this as it also detects flake configuration
+      - uses: blaggacao/nix-quick-install-action@detect-nix-flakes-config
+      # if you want to use nixbuild
+      - uses: nixbuild/nixbuild-action@v17
         with:
-          role-to-assume: arn:aws:iam::123456789100:role/my-github-actions-role
-          aws-region: us-east-2
+          nixbuild_ssh_key: ${{ secrets.SSH_PRIVATE_KEY }}
+          generate_summary_for: job
+      - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: divnix/std-action/run@main
-        with:
-          extra_nix_config: |
-            ${{ needs.discover.outputs.nix_conf }}
-          json: ${{ toJSON(matrix.target) }}
-          # optional:
-          github_pat: ${{ secrets.HUB_PAT }}
-          nix_key: ${{ secrets.NIX_SECRET_KEY }}
-          nix_ssh_key: ${{ secrets.NIXBUILD_SSH }}
-          cache: s3://nix?endpoint=sfo3.digitaloceanspaces.com
-          builder: ssh-ng://eu.nixbuild.net
-          ssh_known_hosts: "eu.nixbuild.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPIQCZc54poJ8vqawd8TraNryQeJnvH1eLpIDgbiqymM"
-
-  build-devshells:
-    <<: *run-job
+
+  images:
+    <<: *job
+    needs: [discover, build]
+    if: fromJSON(needs.discover.outputs.hits).oci-images.publish != '{}'
     strategy:
       matrix:
-        target: ${{ fromJSON(needs.discover.outputs.hits).devshells.build }}
-
-  publish-containers:
-    <<: *run-job
+        target: ${{ fromJSON(needs.discover.outputs.hits).oci-images.publish }}
+
+  deploy:
+    <<: *job
+    needs: [discover, images]
+    environment:
+      name: development
+      url: https://my.dev.example.com
+    if: fromJSON(needs.discover.outputs.hits).deployments.apply != '{}'
     strategy:
       matrix:
-        target: ${{ fromJSON(needs.discover.outputs.hits).containers.publish }}
+        target: ${{ fromJSON(needs.discover.outputs.hits).deployments.apply }}
 ```
 
-## Notes & Explanation
+</details>
 
-### Notes on the Build Matrix
+### Persistent Discovery Host
 
-Hits from the discovery phase are namespaced by Block and Action.
+#### Requirements
 
-That means:
+- `nix` >= v2.16.1
+- `zstd`
+- (gnu) `parallel`
+- `jq`
+- `base64`
+- `bash` > v5
 
-- In: `target: ${{ fromJSON(needs.discover.outputs.hits).packages.build }}`
-  - `packages` is the name of a Standard Block
-  - `build` is the name of an Action of that Block
+The persistent host must also implement the `nixConfig` detection capabilities
+implemented by [this script][script].
 
-This example would be defined in `flake.nix` as such
+[script]: https://github.com/nixbuild/nix-quick-install-action/blob/5752d21669438be20da4de77327ae963e98c82a3/read-nix-config-from-flake.sh
 
 ```nix
 {
   /* ... */
   outputs = {std, ...}@inputs: std.growOn {
   /* ... */
     cellBlocks = with std.blockTypes; [
-      (installables "packages" {ci.build = true;})
-      (containers "containers" {ci.publish = true;})
+      (devshells "envs" {ci.build = true;})
+      (containers "oci-images" {ci.publish = true;})
     ];
   /* ... */
   };
 }
 ```
 
-An example schema of the json returned by the dicovery phase:
+<details><summary><h4>GH Action file</h4></summary>
 
-```json
-{
-  "containers": {
-    "publish": [
-      {
-        "action": "publish",
-        "actionDrv": "/nix/store/6b0i2ww5drcdfa6hgxijx39zbcq57rwl-publish.drv",
-        "actionFragment": "\"__std\".\"actions\".\"x86_64-linux\".\"_automation\".\"containers\".\"vscode\".\"publish",
-        "block": "containers",
-        "blockType": "containers",
-        "cell": "_automation",
-        "name": "vscode",
-        "targetDrv": "/nix/store/4hs8x5lgb9nkvjfrxj7azv95hi77avxn-image-std-vscode.json.drv",
-        "targetFragment": "\"x86_64-linux\".\"_automation\".\"containers\".\"vscode\""
-      }
-    ]
-  },
-  "devshells": {
-    "build": [
-      {
-        "action": "build",
-        "actionDrv": "/nix/store/zmlva6xlngzj098znyy47p72rxjzgka3-build.drv",
-        "actionFragment": "\"__std\".\"actions\".\"x86_64-linux\".\"_automation\".\"devshells\".\"default\".\"build",
-        "block": "devshells",
-        "blockType": "devshells",
-        "cell": "_automation",
-        "name": "default",
-        "targetDrv": "/nix/store/xq4sl7pf51gp0a036garz56kkr160n5c-Standard.drv",
-        "targetFragment": "\"x86_64-linux\".\"_automation\".\"devshells\".\"default\""
-      }
-    ]
-  },
-  "packages": {
-    "build": [
-      {
-        "action": "build",
-        "actionDrv": "/nix/store/l4y4gzpgym5wbvn42avsaf24nqj0d27y-build.drv",
-        "actionFragment": "\"__std\".\"actions\".\"x86_64-linux\".\"std\".\"packages\".\"adrgen\".\"build",
-        "block": "packages",
-        "blockType": "installables",
-        "cell": "std",
-        "name": "adrgen",
-        "targetDrv": "/nix/store/mwidj7li8b7zypq83ap0fmmwxqx58qn6-adrgen-2022-08-08.drv",
-        "targetFragment": "\"x86_64-linux\".\"std\".\"packages\".\"adrgen\""
-      }
-    ]
-  }
-}
+```yaml
+# yq '. | explode(.)' this.yml > .github/workflows/std.yml
+name: CI/CD
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+env:
+  DISCOVERY_USER_NAME: gha-runner
+  DISCOVERY_KNOWN_HOSTS_ENTRY: "10.10.10.10 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEOVVDZydvD+diYa6A3EtA3WGw5NfN0wv7ckQxa/fX1O"
+
+permissions:
+  id-token: write
+  contents: read
+
+concurrency:
+  group: ${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  discover:
+    outputs:
+      hits: ${{ steps.discovery.outputs.hits }}
+    runs-on: [self-hosted, discovery]
+    steps:
+      - name: Standard Discovery
+        uses: divnix/std-action/discover@main
+        id: discovery
+        # avoids transporting derivations via GH Cache
+        with: { ffBuildInstructions: true }
+
+  image: &run-job
+    needs: discover
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ${{ fromJSON(needs.discover.outputs.hits).oci-images.publish }}
+    if: fromJSON(needs.discover.outputs.hits).oci-images.publish != '{}'
+    name: ${{ matrix.target.jobName }}
+    runs-on: ubuntu-latest
+    steps:
+      # sets up ssh credentials for `ssh discovery ...`
+      - uses: divnix/std-action/setup-discovery-ssh@main
+        with:
+          ssh_key: ${{ secrets.SSH_PRIVATE_KEY_CI }}
+          user_name: ${{ env.DISCOVERY_USER_NAME }}
+          ssh_known_hosts_entry: ${{ env.DISCOVERY_KNOWN_HOSTS_ENTRY }}
+      - uses: divnix/std-action/run@main
+        # avoids retreiving derivations via GH Cache and uses `ssh discovery ...` instead
+        with: { ffBuildInstructions: true }
+
+  build:
+    <<: *run-job
+    strategy:
+      matrix:
+        target: ${{ fromJSON(needs.discover.outputs.hits).envs.build }}
+    if: fromJSON(needs.discover.outputs.hits).envs.build != '{}'
 ```
+
+</details>
+
+## Notes & Explanation
+
+### Notes on the Build Matrix
+
+Hits from the discovery phase are namespaced by Block and Action.
+
+That means:
+
+- In: `target: ${{ fromJSON(needs.discover.outputs.hits).packages.build }}`
+  - `packages` is the name of a Standard Block
+  - `build` is the name of an Action of that Block
+
+### Debugging
+
+Watch out for `base64`-encoded blobs in the logs, you can inspect the
+working data of that context by doing: `base64 -d <<< copy-blob-here | jq`.
diff --git a/discover/eval.sh b/discover/eval.sh
@@ -142,7 +142,7 @@ echo "::group::📞️ Pass artifacts to the build matrix ..."
   if [[ "$SKIP_DRV_EXPORT" == "false" ]]; then
     command mkdir -p "$EVALSTORE_EXPORT"
     for drv in $(command jq --compact-output --raw-output '.[].actionDrv' <<<"$PROVISIONED"); do
-       nix-store --query --requisites "$drv" | command nix-store --stdin --export | command zstd > "$EVALSTORE_EXPORT/$(basename $drv).zst"
+       command nix-store --query --requisites "$drv" | command nix-store --stdin --export | command zstd > "$EVALSTORE_EXPORT/$(basename $drv).zst"
     done
   fi
 }