Skip to content
CI

Improve CI #387

Sign in to view logs

Improve CI

Improve CI #387

Workflow file for this run

.github/workflows/ci.yml at fa3b8e2
name: "CI"
on:
workflow_dispatch:
push:
branches:
- "main"
pull_request:
schedule:
- cron: "0 0 * * *"
env:
FLOX_DISABLE_METRICS: "true"
jobs:
envs:
name: "Find environments"
runs-on: "ubuntu-latest"
timeout-minutes: 30
outputs:
envs_per_system: "${{ steps.envs.outputs.envs_per_system }}"
envs_only: "${{ steps.envs.outputs.envs_only }}"
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- name: "Find environment"
id: "envs"
run: |
envs_per_system="["
envs_only="["
update_all=${{ github.event_name == 'schedule' && 'true' || '' }}
BASE_SHA="${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || 'HEAD~1' }}"
if git diff --name-only $BASE_SHA HEAD -- | grep -E "flake.nix|flake.lock|.github" ; then
echo "detected major change"
update_all=true
fi
while IFS= read manifest_path; do
env_path=$(realpath -s $(dirname $manifest_path)/../..)
rel_env_path="${env_path#$PWD/}"
echo "env_path=$env_path"
echo "rel_env_path=$rel_env_path"
if [ -f "$env_path/test.sh" ] && [ "$update_all" == "true" ] || ( git diff --name-only $BASE_SHA HEAD | grep -q "$rel_env_path/" ; ); then
name=$(basename $env_path)
num_of_services=$(yq -oy '.services | length' $manifest_path)
start_services="true"
if [ "$num_of_services" -eq 0 ]; then
start_services="false"
fi
readarray systems < <(yq e -o=j -I=0 '.options.systems[]' $manifest_path)
comma_per_system=""
if [ "$envs_per_system" != "[" ]; then comma_per_system=","; fi
for system in "${systems[@]}"; do
system=$(echo $system | xargs)
envs_per_system="$envs_per_system$comma_per_system{\"example\":\"$name\",\"system\":\"$system\",\"start_services\":$start_services}"
comma_per_system=","
done
comma_only=""
if [ "$name" == "flaim" ]; then continue; fi
if [ "$envs_only" != "[" ]; then comma_only=","; fi
envs_only="$envs_only$comma_only{\"example\":\"$name\"}"
fi
done <<< "$(find $PWD -name manifest.toml)"
envs_per_system="$envs_per_system]"
envs_only="$envs_only]"
echo "-- envs_per_system ---------"
echo "$envs_per_system" | jq
echo "----------------------------"
echo "-- envs_only ---------------"
echo "$envs_only" | jq
echo "----------------------------"
echo "envs_per_system=$envs_per_system" >> "$GITHUB_OUTPUT"
echo "envs_only=$envs_only" >> "$GITHUB_OUTPUT"
test:
name: "Test '${{ matrix.example }}' example on '${{ matrix.system }}'"
runs-on: "ubuntu-latest"
timeout-minutes: 30
needs:
- "envs"
strategy:
fail-fast: false
max-parallel: 8
matrix:
include: ${{ fromJSON(needs.envs.outputs.envs_per_system ) }}
steps:
- name: "Setup SSH"
uses: "webfactory/[email protected]"
with:
ssh-private-key: "${{ secrets.MANAGED_FLOXBOT_SSH_KEY }}"
- name: "Setup Tailscale"
uses: "tailscale/github-action@v2"
with:
args: "--timeout 30s --login-server ${{ vars.MANAGED_TAILSCALE_URL }}"
tags: "tag:ci"
authkey: "${{ secrets.MANAGED_TAILSCALE_AUTH_KEY }}"
- name: "Find remote server to run tests on"
run: |
set -eo pipefail
echo "${{ vars.MANAGED_REMOTE_BUILDERS }}" > machines
export REMOTE_SERVER=$(cat machines | grep ${{ matrix.system }} | cut -f1 -d' ' | cut -f3 -d'/' | head -1 | sed 's/nixbld@//' ; )
export REMOTE_SERVER_USER_KNOWN_HOSTS_FILE=$(mktemp)
export REMOTE_PUBLIC_HOST_KEY=$(cat machines | grep ${{ matrix.system }} | tr -s ' ' | cut -f8 -d' ' | base64 -d ; )
printf "%s %s\n" "$REMOTE_SERVER" "$REMOTE_PUBLIC_HOST_KEY" > "$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE"
echo "REMOTE_SERVER: $REMOTE_SERVER"
echo "REMOTE_SERVER_USER_KNOWN_HOSTS_FILE: $REMOTE_SERVER_USER_KNOWN_HOSTS_FILE"
cat $REMOTE_SERVER_USER_KNOWN_HOSTS_FILE
echo "REMOTE_SERVER=$REMOTE_SERVER" >> $GITHUB_ENV
echo "REMOTE_SERVER_USER_KNOWN_HOSTS_FILE=$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE" >> $GITHUB_ENV
- name: "Test environment"
run: |
ssh github@$REMOTE_SERVER \
-oUserKnownHostsFile=$REMOTE_SERVER_USER_KNOWN_HOSTS_FILE \
nix run \
--accept-flake-config \
--extra-experimental-features '"nix-command flakes"' \
--option access-tokens "github.com=${{ secrets.MANAGED_FLOXBOT_GITHUB_ACCESS_TOKEN_REPO_SCOPE }}" \
github:flox/floxenvs/${{ github.sha }}#apps.${{ matrix.system }}.test-${{ matrix.example }} -- ${{ matrix.start_services }}
containarize:
name: "Containarize '${{ matrix.example }}'"
runs-on: "ubuntu-latest"
timeout-minutes: 30
if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
needs:
- "envs"
- "test"
env:
FLOX_BIN: "flox -vvv"
permissions:
contents: "read"
packages: "write"
attestations: "write"
id-token: "write"
strategy:
fail-fast: false
max-parallel: 8
matrix:
include: ${{ fromJSON(needs.envs.outputs.envs_only ) }}
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
- name: "Install flox"
uses: "flox/install-flox-action@main"
- name: "Login to Github Container Registry"
uses: "docker/login-action@v3"
with:
registry: "ghcr.io"
username: "${{ github.actor }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: "Containarize"
run: |
flox containerize -d ./${{ matrix.example }}
- name: "Tag & Push"
run: |
docker tag ${{ matrix.example }}:latest ghcr.io/flox/floxenvs:${{ matrix.example }}-latest
docker push ghcr.io/flox/floxenvs:${{ matrix.example }}-latest
push:
name: "Sync '${{ matrix.example }}' manifest"
runs-on: "ubuntu-latest"
timeout-minutes: 30
if: (github.event_name == 'push' && github.ref_name == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
needs:
- "envs"
- "test"
env:
FLOX_BIN: "flox -vvv"
FLOX_REMOTE_OWNER: "flox"
FLOX_AUTH0_URL: "https://auth.flox.dev"
strategy:
fail-fast: false
max-parallel: 8
matrix:
include: ${{ fromJSON(needs.envs.outputs.envs_only ) }}
steps:
- name: "Checkout"
uses: "actions/checkout@v4"
- name: "Install flox"
uses: "flox/install-flox-action@main"
- name: "Get FloxHub token"
run: |
echo "FLOX_FLOXHUB_TOKEN=$(
curl --request POST \
--url $FLOX_AUTH0_URL/oauth/token \
--header 'content-type: application/x-www-form-urlencoded' \
--data "client_id=${{ secrets.MANAGED_FLOXENVS_AUTH0_CLIENT_ID }}" \
--data "audience=https://hub.flox.dev/api" \
--data "grant_type=client_credentials" \
--data "client_secret=${{ secrets.MANAGED_FLOXENVS_AUTH0_CLIENT_SECRET }}" \
| jq .access_token -r)" >> $GITHUB_ENV
- name: "Pull or Create remote environment"
run: |
pushd ./${{ matrix.example }}
if flox list --config --remote "$FLOX_REMOTE_OWNER/${{ matrix.example }}" >/dev/null; then
$FLOX_BIN pull --remote "$FLOX_REMOTE_OWNER/${{ matrix.example }}" --dir "remote"
else
echo "WARN: No environment $FLOX_REMOTE_OWNER/${{ matrix.example }} found on FloxHub"
echo "WARN: Creating a new environment ${{ matrix.example }}"
$FLOX_BIN init --name ${{ matrix.example }} --dir "remote"
$FLOX_BIN push --dir "remote"
fi
popd
- name: "Sync to remote environment"
run: |
pushd ./${{ matrix.example }}
cp -rf .flox/env/* remote/.flox/env/
$FLOX_BIN edit --sync --dir "remote"
popd
- name: "Push to remote environment"
run: |
pushd ./${{ matrix.example }}
$FLOX_BIN push --dir "remote"
popd
report-failure:
name: "Report Failure"
runs-on: "ubuntu-latest"
if: ${{ failure() && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'schedule') }}
needs:
- "test"
- "push"
- "containarize"
steps:
- name: "Slack Notification"
uses: "rtCamp/action-slack-notify@v2"
env:
SLACK_CHANNEL: "team-content"
SLACK_TITLE: "Something broke CI for floxenvs"
SLACK_FOOTER: "Thank you for caring"
SLACK_WEBHOOK: "${{ secrets.MANAGED_SLACK_WEBHOOK }}"
SLACK_USERNAME: "GitHub"
SLACK_ICON_EMOJI: ":poop:"
SLACK_COLOR: "#ff2800" # ferrari red -> https://encycolorpedia.com/ff2800
SLACK_LINK_NAMES: true