From e1a28c9c8ce3e2202a8b30eb114432943ea4eb40 Mon Sep 17 00:00:00 2001 From: Gautier DI FOLCO Date: Mon, 28 Aug 2023 20:19:57 +0200 Subject: [PATCH] fix(ci): populate cache when it expired --- .github/workflows/call-check-advisories.yml | 94 +++++++++++++ .github/workflows/call-nix.yml | 38 ++++++ .../workflows/check-advisories-standalone.yml | 61 --------- .github/workflows/check-advisories.yml | 129 +++++++----------- .github/workflows/nix.yml | 37 +---- 5 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/call-check-advisories.yml create mode 100644 .github/workflows/call-nix.yml delete mode 100644 .github/workflows/check-advisories-standalone.yml diff --git a/.github/workflows/call-check-advisories.yml b/.github/workflows/call-check-advisories.yml new file mode 100644 index 00000000..e100a13b --- /dev/null +++ b/.github/workflows/call-check-advisories.yml @@ -0,0 +1,94 @@ +name: Check and publish security advisories +on: + workflow_call: + inputs: + fetch-key: + required: true + type: string + is-artifact: + required: true + type: boolean + changed-advisories: + required: false + type: string + default: '[]' +jobs: + check-advisories: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + with: + path: source + # We need to retrieve full history to determine the correct + # `published` and `modified` timestamps + fetch-depth: 0 + - run: mkdir -p ~/.local/dockerImages + - name: Fetch artifact + if: ${{ inputs.is-artifact }} + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.fetch-key }} + path: ~/.local/dockerImages + - name: Fetch cache + id: fetch-binaries + if: ${{ !inputs.is-artifact }} + uses: actions/cache/restore@v3 + with: + key: ${{ inputs.fetch-key }} + path: ~/.local/dockerImages + - name: Populate cache on cache miss + if: ${{ !inputs.is-artifact && steps.fetch-binaries.outputs.cache-hit != 'true' }} + uses: ./.github/workflows/call-nix.yml + - name: Fetch cache (second attempt after cache miss) + if: ${{ !inputs.is-artifact && steps.fetch-binaries.outputs.cache-hit != 'true' }} + uses: actions/cache/restore@v3 + with: + key: ${{ inputs.fetch-key }} + path: ~/.local/dockerImages + fail-on-cache-miss: true + - run: docker load -i ~/.local/dockerImages/hsec-tools + - name: Run advisory syntax checks + env: + CHANGED_ADVISORIES_JSON: ${{ inputs.changed-advisories }} + run: | + CHANGED_ADVISORIES=( $(printenv CHANGED_ADVISORIES_JSON | jq -r '.[]') ) + cd source + RESULT=0 + # Remove the begining of the README to extract the example. + (echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md + while read FILE ; do + [ "$(dirname "$FILE")" != advisories/reserved ] || continue + echo -n "$FILE: " + docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1 + done < <([ ${#CHANGED_ADVISORIES[@]} -gt 0 ] && printf "%s\n" "${CHANGED_ADVISORIES[@]}" || find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md") + exit $RESULT + - name: Run advisory uniqueness checks + run: | + ! find source/advisories -type f -name '*.md' -print0 \ + | xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 ' + - name: Publish OSV data + if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }} + env: + GITHUB_SHA: ${{ github.sha }} + run: | + DATA_DIR=$PWD/osv + mkdir "$DATA_DIR" + cd source + while read FILE ; do + ID=$(basename "$FILE" .md) + YEAR=$(echo "$ID" | cut -d - -f 2) + mkdir -p $DATA_DIR/$YEAR + docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json + done < <(find advisories -type f -name "*.md") + BRANCH=generated/osv-export + REF=refs/remotes/origin/$BRANCH + export GIT_WORK_TREE=$DATA_DIR + git read-tree "$REF" + git add --all --intent-to-add + git diff --quiet && exit + git add --all + TREE=$(git write-tree) + git config user.email security-advisories@haskell.org + git config user.name "Haskell Security Response Team" + COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)") + git push origin $COMMIT:$BRANCH diff --git a/.github/workflows/call-nix.yml b/.github/workflows/call-nix.yml new file mode 100644 index 00000000..244636ed --- /dev/null +++ b/.github/workflows/call-nix.yml @@ -0,0 +1,38 @@ +name: nix build +on: + workflow_call: +jobs: + check_nix: + name: Check nix build + runs-on: ubuntu-22.04 + steps: + - name: git checkout + uses: actions/checkout@v3 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: system-features = nixos-test benchmark big-parallel kvm + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Check Nix flake inputs + uses: DeterminateSystems/flake-checker-action@v4 + - name: Build executable + run: nix -L build + - name: Build docker image + run: nix build -L '.#packages.x86_64-linux.hsec-tools-image' + - run: mkdir -p ~/.local/dockerImages + - run: cp result ~/.local/dockerImages/hsec-tools + - id: code-hash + name: Compute code directory hash + run: | + code_hash=$(git rev-parse HEAD:code) + echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT" + - uses: actions/cache/save@v3 + if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + with: + key: hsec-tools-${{ steps.code-hash.outputs.code-hash}} + path: ~/.local/dockerImages + - name: upload executable + uses: actions/upload-artifact@v3 + with: + name: hsec-tools-${{ github.sha }} + path: ~/.local/dockerImages diff --git a/.github/workflows/check-advisories-standalone.yml b/.github/workflows/check-advisories-standalone.yml deleted file mode 100644 index a57d29d2..00000000 --- a/.github/workflows/check-advisories-standalone.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Check advisories standalone -on: - - pull_request -jobs: - tools_changed: - continue-on-error: true - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5.3.0 - with: - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - paths: '["code/**"]' - do_not_skip: '["push", "workflow_dispatch", "schedule"]' - advisories_changed: - continue-on-error: true - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip_check.outputs.should_skip }} - changed_files: ${{ steps.process-changed-files.outputs.out }} - steps: - - id: skip_check - uses: fkirc/skip-duplicate-actions@v5.3.0 - with: - concurrent_skipping: "never" - skip_after_successful_duplicate: "true" - paths: '["advisories/**", "EXAMPLE_ADVISORY.md"]' - do_not_skip: '["push", "workflow_dispatch", "schedule"]' - - id: process-changed-files - name: Extract matched files list - env: - PATHS_RESULT: ${{ steps.skip_check.outputs.paths_result }} - run: | - echo -n 'out=' >> "$GITHUB_OUTPUT" - # See https://github.com/fkirc/skip-duplicate-actions#paths_result - printenv PATHS_RESULT \ - | jq --compact-output .global.matched_files >> "$GITHUB_OUTPUT" - code_hash: - name: Compute code directory hash - runs-on: ubuntu-22.04 - outputs: - code_hash: ${{ steps.code-hash.outputs.code-hash }} - steps: - - name: git checkout - uses: actions/checkout@v3 - - id: code-hash - run: | - code_hash=$(git rev-parse HEAD:code) - echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT" - check_advisories: - name: Invoke check-advisories workflow - needs: [tools_changed, advisories_changed, code_hash] - if: ${{ needs.tools_changed.outputs.should_skip == 'true' && needs.advisories_changed.outputs.should_skip != 'true' }} - uses: ./.github/workflows/check-advisories.yml - with: - fetch-key: hsec-tools-${{ needs.code_hash.outputs.code_hash }} - is-artifact: false - changed-advisories: ${{ needs.advisories_changed.outputs.changed_files }} diff --git a/.github/workflows/check-advisories.yml b/.github/workflows/check-advisories.yml index e0b06146..47adfe21 100644 --- a/.github/workflows/check-advisories.yml +++ b/.github/workflows/check-advisories.yml @@ -1,84 +1,61 @@ -name: Check and publish security advisories +name: Check advisories on: - workflow_call: - inputs: - fetch-key: - required: true - type: string - is-artifact: - required: true - type: boolean - changed-advisories: - required: false - type: string - default: '[]' + - pull_request jobs: - check-advisories: - runs-on: ubuntu-20.04 + tools_changed: + continue-on-error: true + runs-on: ubuntu-22.04 + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} steps: - - uses: actions/checkout@v3 + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5.3.0 with: - path: source - # We need to retrieve full history to determine the correct - # `published` and `modified` timestamps - fetch-depth: 0 - - run: mkdir -p ~/.local/dockerImages - - name: Fetch artifact - if: ${{ inputs.is-artifact }} - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.fetch-key }} - path: ~/.local/dockerImages - - name: Fetch cache - if: ${{ !inputs.is-artifact }} - uses: actions/cache/restore@v3 + concurrent_skipping: "never" + skip_after_successful_duplicate: "true" + paths: '["code/**"]' + do_not_skip: '["push", "workflow_dispatch", "schedule"]' + advisories_changed: + continue-on-error: true + runs-on: ubuntu-22.04 + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + changed_files: ${{ steps.process-changed-files.outputs.out }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5.3.0 with: - key: ${{ inputs.fetch-key }} - path: ~/.local/dockerImages - fail-on-cache-miss: true - - run: docker load -i ~/.local/dockerImages/hsec-tools - - name: Run advisory syntax checks + concurrent_skipping: "never" + skip_after_successful_duplicate: "true" + paths: '["advisories/**", "EXAMPLE_ADVISORY.md"]' + do_not_skip: '["push", "workflow_dispatch", "schedule"]' + - id: process-changed-files + name: Extract matched files list env: - CHANGED_ADVISORIES_JSON: ${{ inputs.changed-advisories }} - run: | - CHANGED_ADVISORIES=( $(printenv CHANGED_ADVISORIES_JSON | jq -r '.[]') ) - cd source - RESULT=0 - # Remove the begining of the README to extract the example. - (echo '```toml'; sed -e '1,/```toml/d' README.md) > EXAMPLE_README.md - while read FILE ; do - [ "$(dirname "$FILE")" != advisories/reserved ] || continue - echo -n "$FILE: " - docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools check "$FILE" || RESULT=1 - done < <([ ${#CHANGED_ADVISORIES[@]} -gt 0 ] && printf "%s\n" "${CHANGED_ADVISORIES[@]}" || find advisories EXAMPLE_README.md EXAMPLE_ADVISORY.md -type f -name "*.md") - exit $RESULT - - name: Run advisory uniqueness checks + PATHS_RESULT: ${{ steps.skip_check.outputs.paths_result }} run: | - ! find source/advisories -type f -name '*.md' -print0 \ - | xargs -0n1 basename | sort | uniq -c | grep -E -v '[[:space:]]*1 ' - - name: Publish OSV data - if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'haskell/security-advisories' }} - env: - GITHUB_SHA: ${{ github.sha }} + echo -n 'out=' >> "$GITHUB_OUTPUT" + # See https://github.com/fkirc/skip-duplicate-actions#paths_result + printenv PATHS_RESULT \ + | jq --compact-output .global.matched_files >> "$GITHUB_OUTPUT" + code_hash: + name: Compute code directory hash + runs-on: ubuntu-22.04 + outputs: + code_hash: ${{ steps.code-hash.outputs.code-hash }} + steps: + - name: git checkout + uses: actions/checkout@v3 + - id: code-hash run: | - DATA_DIR=$PWD/osv - mkdir "$DATA_DIR" - cd source - while read FILE ; do - ID=$(basename "$FILE" .md) - YEAR=$(echo "$ID" | cut -d - -f 2) - mkdir -p $DATA_DIR/$YEAR - docker run --rm -v $PWD:/repo --workdir /repo haskell/hsec-tools:latest /bin/hsec-tools osv "$FILE" > $DATA_DIR/$YEAR/$ID.json - done < <(find advisories -type f -name "*.md") - BRANCH=generated/osv-export - REF=refs/remotes/origin/$BRANCH - export GIT_WORK_TREE=$DATA_DIR - git read-tree "$REF" - git add --all --intent-to-add - git diff --quiet && exit - git add --all - TREE=$(git write-tree) - git config user.email security-advisories@haskell.org - git config user.name "Haskell Security Response Team" - COMMIT=$(git commit-tree "$TREE" -p "$REF" -m "$(date --utc --rfc-3339=seconds) ($GITHUB_SHA)") - git push origin $COMMIT:$BRANCH + code_hash=$(git rev-parse HEAD:code) + echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT" + check_advisories: + name: Invoke check-advisories workflow + needs: [tools_changed, advisories_changed, code_hash] + if: ${{ needs.tools_changed.outputs.should_skip == 'true' && needs.advisories_changed.outputs.should_skip != 'true' }} + uses: ./.github/workflows/call-check-advisories.yml + with: + fetch-key: hsec-tools-${{ needs.code_hash.outputs.code_hash }} + is-artifact: false + changed-advisories: ${{ needs.advisories_changed.outputs.changed_files }} diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 8235b627..b49ed610 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -1,6 +1,4 @@ -# Our desired pipeline using only a Nix shell environment name: nix build - on: - push - pull_request @@ -22,43 +20,12 @@ jobs: name: Check nix build needs: tools_changed if: ${{ needs.tools_changed.outputs.should_skip != 'true' }} - runs-on: ubuntu-22.04 - steps: - - name: git checkout - uses: actions/checkout@v3 - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - with: - extra-conf: system-features = nixos-test benchmark big-parallel kvm - - uses: DeterminateSystems/magic-nix-cache-action@main - - name: Check Nix flake inputs - uses: DeterminateSystems/flake-checker-action@v4 - - name: Build executable - run: nix -L build - - name: Bild docker image - run: nix build -L '.#packages.x86_64-linux.hsec-tools-image' - - run: mkdir -p ~/.local/dockerImages - - run: cp result ~/.local/dockerImages/hsec-tools - - id: code-hash - name: Compute code directory hash - run: | - code_hash=$(git rev-parse HEAD:code) - echo "code-hash=$code_hash" >> "$GITHUB_OUTPUT" - - uses: actions/cache/save@v3 - if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} - with: - key: hsec-tools-${{ steps.code-hash.outputs.code-hash}} - path: ~/.local/dockerImages - - name: upload executable - uses: actions/upload-artifact@v3 - with: - name: hsec-tools-${{ github.sha }} - path: ~/.local/dockerImages + uses: ./.github/workflows/call-nix.yml check-advisories: name: Invoke check-advisories workflow if: ${{ needs.tools_changed.outputs.should_skip != 'true' }} needs: check_nix - uses: ./.github/workflows/check-advisories.yml + uses: ./.github/workflows/call-check-advisories.yml with: fetch-key: hsec-tools-${{ github.sha }} is-artifact: true