Improve CI (#111) #388
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CI" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| pull_request: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| env: | |
| FLOX_DISABLE_METRICS: "true" | |
| jobs: | |
| envs: | |
| name: "Find environments" | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 30 | |
| outputs: | |
| envs_per_system: "${{ steps.envs.outputs.envs_per_system }}" | |
| envs_only: "${{ steps.envs.outputs.envs_only }}" | |
| steps: | |
| - name: "Checkout" | |
| uses: "actions/checkout@v4" | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: "Find environment" | |
| id: "envs" | |
| run: | | |
| envs_per_system="[" | |
| envs_only="[" | |
| update_all=${{ github.event_name == 'schedule' && 'true' || '' }} | |
| BASE_SHA="${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || 'HEAD~1' }}" | |
| if git diff --name-only $BASE_SHA HEAD -- | grep -E "flake.nix|flake.lock|.github" ; then | |
| echo "detected major change" | |
| update_all=true | |
| fi | |
| while IFS= read manifest_path; do | |
| env_path=$(realpath -s $(dirname $manifest_path)/../..) | |
| rel_env_path="${env_path#$PWD/}" | |
| echo "env_path=$env_path" | |
| echo "rel_env_path=$rel_env_path" | |
| if [ -f "$env_path/test.sh" ] && [ "$update_all" == "true" ] || ( git diff --name-only $BASE_SHA HEAD | grep -q "$rel_env_path/" ; ); then | |
| name=$(basename $env_path) | |
| num_of_services=$(yq -oy '.services | length' $manifest_path) | |
| start_services="true" | |
| if [ "$num_of_services" -eq 0 ]; then | |
| start_services="false" | |
| fi | |
| readarray systems < <(yq e -o=j -I=0 '.options.systems[]' $manifest_path) | |
| comma_per_system="" | |
| if [ "$envs_per_system" != "[" ]; then comma_per_system=","; fi | |
| for system in "${systems[@]}"; do | |
| system=$(echo $system | xargs) | |
| envs_per_system="$envs_per_system$comma_per_system{\"example\":\"$name\",\"system\":\"$system\",\"start_services\":$start_services}" | |
| comma_per_system="," | |
| done | |
| comma_only="" | |
| if [ "$name" == "flaim" ]; then continue; fi | |
| if [ "$envs_only" != "[" ]; then comma_only=","; fi | |
| envs_only="$envs_only$comma_only{\"example\":\"$name\"}" | |
| fi | |
| done <<< "$(find $PWD -name manifest.toml)" | |
| envs_per_system="$envs_per_system]" | |
| envs_only="$envs_only]" | |
| echo "-- envs_per_system ---------" | |
| echo "$envs_per_system" | jq | |
| echo "----------------------------" | |
| echo "-- envs_only ---------------" | |
| echo "$envs_only" | jq | |
| echo "----------------------------" | |
| echo "envs_per_system=$envs_per_system" >> "$GITHUB_OUTPUT" | |
| echo "envs_only=$envs_only" >> "$GITHUB_OUTPUT" | |
| test: | |
| name: "Test '${{ matrix.example }}' example on '${{ matrix.system }}'" | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 30 | |
| needs: | |
| - "envs" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| include: ${{ fromJSON(needs.envs.outputs.envs_per_system ) }} | |
| steps: | |
| - name: "Setup SSH" | |
| uses: "webfactory/[email protected]" | |
| with: | |
| ssh-private-key: "${{ secrets.MANAGED_FLOXBOT_SSH_KEY }}" | |
| - name: "Setup Tailscale" | |
| uses: "tailscale/github-action@v2" | |
| with: | |
| args: "--timeout 30s --login-server ${{ vars.MANAGED_TAILSCALE_URL }}" | |
| tags: "tag:ci" | |
| authkey: "${{ secrets.MANAGED_TAILSCALE_AUTH_KEY }}" | |
| - name: "Find remote server to run tests on" | |
| run: | | |
| set -eo pipefail | |
| echo "${{ vars.MANAGED_REMOTE_BUILDERS }}" > machines | |
| export REMOTE_SERVER=$(cat machines | grep ${{ matrix.system }} | cut -f1 -d' ' | cut -f3 -d'/' | head -1 | sed 's/nixbld@//' ; ) | |
| export REMOTE_SERVER_USER_KNOWN_HOSTS_FILE=$(mktemp) | |
| export REMOTE_PUBLIC_HOST_KEY=$(cat machines | grep ${{ matrix.system }} | tr -s ' ' | cut -f8 -d' ' | base64 -d ; ) | |
| printf "%s %s\n" "$REMOTE_SERVER" "$REMOTE_PUBLIC_HOST_KEY" > "$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE" | |
| echo "REMOTE_SERVER: $REMOTE_SERVER" | |
| echo "REMOTE_SERVER_USER_KNOWN_HOSTS_FILE: $REMOTE_SERVER_USER_KNOWN_HOSTS_FILE" | |
| cat $REMOTE_SERVER_USER_KNOWN_HOSTS_FILE | |
| echo "REMOTE_SERVER=$REMOTE_SERVER" >> $GITHUB_ENV | |
| echo "REMOTE_SERVER_USER_KNOWN_HOSTS_FILE=$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE" >> $GITHUB_ENV | |
| - name: "Test environment" | |
| run: | | |
| ssh github@$REMOTE_SERVER \ | |
| -oUserKnownHostsFile=$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE \ | |
| nix run \ | |
| --accept-flake-config \ | |
| --extra-experimental-features '"nix-command flakes"' \ | |
| --option access-tokens "github.com=${{ secrets.MANAGED_FLOXBOT_GITHUB_ACCESS_TOKEN_REPO_SCOPE }}" \ | |
| github:flox/floxenvs/${{ github.sha }}#apps.${{ matrix.system }}.test-${{ matrix.example }} -- ${{ matrix.start_services }} | |
| containarize: | |
| name: "Containarize '${{ matrix.example }}'" | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 30 | |
| if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' | |
| needs: | |
| - "envs" | |
| - "test" | |
| env: | |
| FLOX_BIN: "flox -vvv" | |
| permissions: | |
| contents: "read" | |
| packages: "write" | |
| attestations: "write" | |
| id-token: "write" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| include: ${{ fromJSON(needs.envs.outputs.envs_only ) }} | |
| steps: | |
| - name: "Checkout" | |
| uses: "actions/checkout@v4" | |
| - name: "Install flox" | |
| uses: "flox/install-flox-action@main" | |
| - name: "Login to Github Container Registry" | |
| uses: "docker/login-action@v3" | |
| with: | |
| registry: "ghcr.io" | |
| username: "${{ github.actor }}" | |
| password: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: "Containarize" | |
| run: | | |
| flox containerize -d ./${{ matrix.example }} | |
| - name: "Tag & Push" | |
| run: | | |
| docker tag ${{ matrix.example }}:latest ghcr.io/flox/floxenvs:${{ matrix.example }}-latest | |
| docker push ghcr.io/flox/floxenvs:${{ matrix.example }}-latest | |
| push: | |
| name: "Sync '${{ matrix.example }}' manifest" | |
| runs-on: "ubuntu-latest" | |
| timeout-minutes: 30 | |
| if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' | |
| needs: | |
| - "envs" | |
| - "test" | |
| env: | |
| FLOX_BIN: "flox -vvv" | |
| FLOX_REMOTE_OWNER: "flox" | |
| FLOX_AUTH0_URL: "https://auth.flox.dev" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| include: ${{ fromJSON(needs.envs.outputs.envs_only ) }} | |
| steps: | |
| - name: "Checkout" | |
| uses: "actions/checkout@v4" | |
| - name: "Install flox" | |
| uses: "flox/install-flox-action@main" | |
| - name: "Get FloxHub token" | |
| run: | | |
| echo "FLOX_FLOXHUB_TOKEN=$( | |
| curl --request POST \ | |
| --url $FLOX_AUTH0_URL/oauth/token \ | |
| --header 'content-type: application/x-www-form-urlencoded' \ | |
| --data "client_id=${{ secrets.MANAGED_FLOXENVS_AUTH0_CLIENT_ID }}" \ | |
| --data "audience=https://hub.flox.dev/api" \ | |
| --data "grant_type=client_credentials" \ | |
| --data "client_secret=${{ secrets.MANAGED_FLOXENVS_AUTH0_CLIENT_SECRET }}" \ | |
| | jq .access_token -r)" >> $GITHUB_ENV | |
| - name: "Pull or Create remote environment" | |
| run: | | |
| pushd ./${{ matrix.example }} | |
| if flox list --config --remote "$FLOX_REMOTE_OWNER/${{ matrix.example }}" >/dev/null; then | |
| $FLOX_BIN pull --remote "$FLOX_REMOTE_OWNER/${{ matrix.example }}" --dir "remote" | |
| else | |
| echo "WARN: No environment $FLOX_REMOTE_OWNER/${{ matrix.example }} found on FloxHub" | |
| echo "WARN: Creating a new environment ${{ matrix.example }}" | |
| $FLOX_BIN init --name ${{ matrix.example }} --dir "remote" | |
| $FLOX_BIN push --dir "remote" | |
| fi | |
| popd | |
| - name: "Sync to remote environment" | |
| run: | | |
| pushd ./${{ matrix.example }} | |
| cp -rf .flox/env/* remote/.flox/env/ | |
| $FLOX_BIN edit --sync --dir "remote" | |
| popd | |
| - name: "Push to remote environment" | |
| run: | | |
| pushd ./${{ matrix.example }} | |
| $FLOX_BIN push --dir "remote" | |
| popd | |
| report-failure: | |
| name: "Report Failure" | |
| runs-on: "ubuntu-latest" | |
| if: ${{ failure() && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'schedule') }} | |
| needs: | |
| - "test" | |
| - "push" | |
| - "containarize" | |
| steps: | |
| - name: "Slack Notification" | |
| uses: "rtCamp/action-slack-notify@v2" | |
| env: | |
| SLACK_CHANNEL: "team-content" | |
| SLACK_TITLE: "Something broke CI for floxenvs" | |
| SLACK_FOOTER: "Thank you for caring" | |
| SLACK_WEBHOOK: "${{ secrets.MANAGED_SLACK_WEBHOOK }}" | |
| SLACK_USERNAME: "GitHub" | |
| SLACK_ICON_EMOJI: ":poop:" | |
| SLACK_COLOR: "#ff2800" # ferrari red -> https://encycolorpedia.com/ff2800 | |
| SLACK_LINK_NAMES: true |