diff --git a/.github/actions/decision/action.yml b/.github/actions/decision/action.yml index cb83677a..9370d1a2 100644 --- a/.github/actions/decision/action.yml +++ b/.github/actions/decision/action.yml @@ -1,9 +1,15 @@ name: Decision description: Decision outputs: - data: - description: Decision data - value: ${{ steps.filtered.outputs.result }} + matrix: + description: Decision matrix + value: ${{ steps.matrix.outputs.result }} + artifacts: + description: Artifacts data + value: ${{ steps.decision.outputs.artifacts }} + mounts: + description: Mounts data + value: ${{ steps.decision.outputs.mounts }} runs: using: "composite" steps: @@ -13,27 +19,27 @@ runs: run: | python3 CI/decision.py >> $GITHUB_OUTPUT - uses: actions/setup-node@v3 - if: ${{ steps.decision.outputs.data }} + if: ${{ steps.decision.outputs.matrix }} with: node-version: '20.x' - shell: bash - if: ${{ steps.decision.outputs.data }} + if: ${{ steps.decision.outputs.matrix }} run: npm install @actions/cache - name: Check dependencies - id: filtered - if: ${{ steps.decision.outputs.data }} + id: matrix + if: ${{ steps.decision.outputs.matrix }} uses: actions/github-script@v7 with: script: | const cache = require('@actions/cache'); - const data = ${{ toJSON(fromJSON(steps.decision.outputs.data)) }}; + const matrix = ${{ toJSON(fromJSON(steps.decision.outputs.matrix)) }}; + const artifacts = ${{ toJSON(fromJSON(steps.decision.outputs.artifacts)) }}; const filtered = await Promise.all( - Object.entries(data).map(async ([name, items]) => { + Object.entries(matrix).map(async ([name, items]) => { const filtered_items = []; for (const item of items) { - if (!('key' in item) || !await cache.restoreCache(item.paths, item.key, [], { lookupOnly: true }, true)) { - const { key, paths, ...filtered_item } = item; - filtered_items.push(filtered_item); + if (!(item.task in artifacts) || !await cache.restoreCache(artifacts[item.task].paths, artifacts[item.task].key, [], { lookupOnly: true }, true)) { + filtered_items.push(item); } } return [name, filtered_items]; diff --git a/.github/workflows/env-tools.yml b/.github/workflows/env-tools.yml index 796255de..854df1b7 100644 --- a/.github/workflows/env-tools.yml +++ b/.github/workflows/env-tools.yml @@ -4,19 +4,22 @@ jobs: decision: runs-on: ubuntu-latest outputs: - data: ${{ steps.decision.outputs.data }} + data: ${{ steps.decision.outputs.data }} + matrix: ${{ steps.decision.outputs.matrix }} + env: + TC_IS_PUSH: $${ (github.event_name == "push") && '1' || '0' }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/decision id: decision docker-base: - if: ${{ fromJSON(needs.decision.outputs.data)['docker-base'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['docker-base'] && !cancelled() }} needs: decision strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['docker-base'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['docker-base'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -25,12 +28,12 @@ jobs: name: "${{ matrix.task }}" docker: - if: ${{ fromJSON(needs.decision.outputs.data)['docker'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['docker'] && !cancelled() }} needs: [decision, docker-base] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['docker'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['docker'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -39,12 +42,12 @@ jobs: name: "${{ matrix.task }}" msys2-base: - if: ${{ fromJSON(needs.decision.outputs.data)['msys2-base'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['msys2-base'] && !cancelled() }} needs: [decision, docker-base] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['msys2-base'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['msys2-base'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -53,12 +56,12 @@ jobs: name: "${{ matrix.task }}" msys2: - if: ${{ fromJSON(needs.decision.outputs.data)['msys2'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['msys2'] && !cancelled() }} needs: [decision, msys2-base] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['msys2'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['msys2'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -67,12 +70,12 @@ jobs: name: "${{ matrix.task }}" git: - if: ${{ fromJSON(needs.decision.outputs.data)['git'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['git'] && !cancelled() }} needs: [decision, docker] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['git'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['git'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -81,12 +84,12 @@ jobs: name: "${{ matrix.task }}" hg: - if: ${{ fromJSON(needs.decision.outputs.data)['hg'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['hg'] && !cancelled() }} needs: [decision, docker, msys2] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['hg'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['hg'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -95,12 +98,12 @@ jobs: name: "${{ matrix.task }}" build: - if: ${{ fromJSON(needs.decision.outputs.data)['build'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['build'] && !cancelled() }} needs: [decision, docker] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['build'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['build'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -109,12 +112,12 @@ jobs: name: "${{ matrix.task }}" cram: - if: ${{ fromJSON(needs.decision.outputs.data)['cram'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['cram'] && !cancelled() }} needs: [decision, docker, git, hg, build] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['cram'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['cram'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -123,12 +126,12 @@ jobs: name: "${{ matrix.task }}" download: - if: ${{ fromJSON(needs.decision.outputs.data)['download'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['download'] && !cancelled() }} needs: [decision, docker, git, build, msys2] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['download'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['download'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -137,12 +140,12 @@ jobs: name: "${{ matrix.task }}" hg-clone: - if: ${{ fromJSON(needs.decision.outputs.data)['hg-clone'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['hg-clone'] && !cancelled() }} needs: [decision, docker, hg] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['hg-clone'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['hg-clone'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -151,12 +154,12 @@ jobs: name: "${{ matrix.task }}" clone: - if: ${{ fromJSON(needs.decision.outputs.data)['clone'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['clone'] && !cancelled() }} needs: [decision, git, build, hg-clone] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['clone'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['clone'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -165,12 +168,12 @@ jobs: name: "${{ matrix.task }}" graft: - if: ${{ fromJSON(needs.decision.outputs.data)['graft'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['graft'] && !cancelled() }} needs: [decision, docker, git, build, hg, clone, hg-clone] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['graft'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['graft'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -179,12 +182,12 @@ jobs: name: "${{ matrix.task }}" test: - if: ${{ fromJSON(needs.decision.outputs.data)['test'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['test'] && !cancelled() }} needs: [decision, docker, git, build, hg, clone, hg-clone] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['test'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['test'] }} runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -193,12 +196,12 @@ jobs: name: "${{ matrix.task }}" upload: - if: ${{ fromJSON(needs.decision.outputs.data)['upload'] && !cancelled() }} + if: ${{ fromJSON(needs.decision.outputs.matrix)['upload'] && !cancelled() }} needs: [decision, docker, build, cram, graft, test] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.decision.outputs.data)['upload'] }} + include: ${{ fromJSON(needs.decision.outputs.matrix)['upload'] }} runs-on: ${{ matrix.runner }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/CI/decision.py b/CI/decision.py index 7c58a0be..933d3e62 100644 --- a/CI/decision.py +++ b/CI/decision.py @@ -464,7 +464,7 @@ def tasks(): ], ) ), - **kwargs + **kwargs, ) @@ -484,7 +484,9 @@ def main_gh(): "windows": "windows-latest", } matrix = {} - for t in Task.by_id.values(): + artifacts = {} + mounts = {} + for id, t in Task.by_id.items(): task = t.task payload = task.get("payload", {}) name = task.get("metadata", {})["name"] @@ -493,20 +495,38 @@ def main_gh(): job_name = "hg-clone" if job_name in ("docker", "msys2") and "base" in name: job_name = f"{job_name}-base" - item = { - "task": name, - "runner": RUNNER[task["workerType"]], - } - if "routes" in task: - item["paths"] = [ - os.path.basename(artifact["name"]) - for artifact in payload.get("artifacts", []) - ] - item["key"] = task.get("routes")[0] - matrix.setdefault(job_name, []).append(item) + matrix.setdefault(job_name, []).append( + { + "task": name, + "runner": RUNNER[task["workerType"]], + } + ) + for mount in payload.get("mounts", []): + content = mount["content"] + dep_id = content["taskId"] + routes = Task.by_id[dep_id].task.get("routes") + mounts.setdefault(name, []).append( + { + "artifact": content["artifact"], + "key": routes[0] if routes else dep_id, + } + ) + + if payload.get("artifacts"): + artifacts.setdefault(name, []).append( + { + "paths": [ + os.path.basename(artifact["name"]) + for artifact in payload.get("artifacts", []) + ], + "key": task["routes"][0] if "routes" in task else id, + } + ) for m in matrix.values(): m.sort(key=lambda x: x["task"]) - print_output("data", matrix) + print_output("matrix", matrix) + print_output("artifacts", artifacts) + print_output("mounts", mounts) def main_tc():