[CI] Duplicate the taskcluster workflow to github

glandium · Feb 1, 2025 · f7bc281 · f7bc281
1 parent e4d02f4
commit f7bc281
Show file tree

Hide file tree

Showing 9 changed files with 620 additions and 12 deletions.
diff --git a/.github/actions/decision/action.yml b/.github/actions/decision/action.yml
@@ -0,0 +1,48 @@
+name: Decision
+description: Decision
+outputs:
+  matrix:
+    description: Decision matrix
+    value: ${{ steps.matrix.outputs.result }}
+  artifacts:
+    description: Artifacts data
+    value: ${{ steps.decision.outputs.artifacts }}
+  mounts:
+    description: Mounts data
+    value: ${{ steps.decision.outputs.mounts }}
+runs:
+  using: "composite"
+  steps:
+    - name: Decision
+      id: decision
+      shell: bash
+      run: |
+        python3 CI/decision.py >> $GITHUB_OUTPUT
+    - uses: actions/setup-node@v3
+      if: ${{ steps.decision.outputs.matrix }}
+      with:
+        node-version: '20.x'
+    - shell: bash
+      if: ${{ steps.decision.outputs.matrix }}
+      run: npm install @actions/cache
+    - name: Check dependencies
+      id: matrix
+      if: ${{ steps.decision.outputs.matrix }}
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const cache = require('@actions/cache');
+          const matrix = ${{ toJSON(fromJSON(steps.decision.outputs.matrix)) }};
+          const artifacts = ${{ toJSON(fromJSON(steps.decision.outputs.artifacts)) }};
+          const filtered = await Promise.all(
+            Object.entries(matrix).map(async ([name, items]) => {
+              const filtered_items = [];
+              for (const item of items) {
+                if (!(item.task in artifacts) || !await cache.restoreCache(artifacts[item.task].paths, artifacts[item.task].key, [], { lookupOnly: true }, true)) {
+                  filtered_items.push(item);
+                }
+              }
+              return [name, filtered_items];
+            })
+          );
+          return Object.fromEntries(filtered.filter(([_, items]) => items.length > 0));
diff --git a/.github/actions/mounts/action.yml b/.github/actions/mounts/action.yml
@@ -0,0 +1,35 @@
+name: Mount dependencies
+description: Mount dependencies
+inputs:
+  mounts:
+    description: mounts
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: '20.x'
+    - shell: bash
+      run: npm install @actions/cache
+    - name: Mount dependencies
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const fs = require('fs');
+          const path = require('path');
+          const cache = require('@actions/cache');
+          const mounts = ${{ toJSON(fromJSON(inputs.mounts)) }};
+          const pwd = process.cwd();
+          // Make actions/cache's getWorkingDirectory() return process.cwd.
+          delete process.env.GITHUB_WORKSPACE;
+          for (const mount of mounts) {
+            const dir = path.join(pwd, 'cache', mount.key, path.dirname(mount.artifact));
+            fs.mkdirSync(dir, { recursive: true });
+            process.chdir(dir);
+            if (await cache.restoreCache([path.basename(mount.artifact)], mount.key, [], {}, true)) {
+              console.log(`Cache restored from key: ${mount.key}`);
+            } else {
+              core.setFailed(`Failed to restore cache from key: ${mount.key}`);
+            }
+          }
diff --git a/.github/actions/task/action.yml b/.github/actions/task/action.yml
@@ -0,0 +1,40 @@
+name: Task
+description: Run task
+inputs:
+  name:
+    description: task name
+    required: true
+  mounts:
+    description: mounts for the task
+    required: false
+  artifacts:
+    description: artifacts from the task
+    required: false
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v5
+      if: ${{ runner.os == 'macOS' }}
+      with:
+        python-version: ${{ runner.arch == 'ARM64' && '3.11.7' || '3.9.14' }}
+    - name: Finish python setup
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        python3 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade
+    - uses: actions/cache@v4
+      if: ${{ fromJSON(inputs.artifacts) }}
+      id: cache
+      with:
+        path: ${{ join(fromJSON(inputs.artifacts).paths, '\n') }}
+        key: ${{ fromJSON(inputs.artifacts).key }}
+        enableCrossOsArchive: true
+    - uses: ./.github/actions/mounts
+      if: ${{ steps.cache.outputs.cache-hit != 'true' && fromJSON(inputs.mounts) }}
+      with:
+        mounts: ${{ inputs.mounts }}
+    - name: ${{ inputs.name }}
+      shell: bash
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      run: |
+        python3 CI/run.py --cache cache --no-recurse --out . "${{ inputs.name }}"
diff --git a/.github/workflows/env-tools.yml b/.github/workflows/env-tools.yml
@@ -0,0 +1,242 @@
+name: environment and tools
+on:
+  push: []
+  pull_request: []
+jobs:
+  decision:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.decision.outputs.matrix }}
+      artifacts: ${{ steps.decision.outputs.artifacts }}
+      mounts: ${{ steps.decision.outputs.mounts }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/decision
+        id: decision
+
+  docker-base:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['docker-base'] && !cancelled() }}
+    needs: decision
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['docker-base'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  docker:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['docker'] && !cancelled() }}
+    needs: [decision, docker-base]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['docker'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  msys2-base:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['msys2-base'] && !cancelled() }}
+    needs: [decision, docker-base]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['msys2-base'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  msys2:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['msys2'] && !cancelled() }}
+    needs: [decision, msys2-base]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['msys2'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  git:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['git'] && !cancelled() }}
+    needs: [decision, docker]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['git'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  hg:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['hg'] && !cancelled() }}
+    needs: [decision, docker, msys2]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['hg'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  build:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['build'] && !cancelled() }}
+    needs: [decision, docker]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['build'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  cram:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['cram'] && !cancelled() }}
+    needs: [decision, docker, git, hg, build]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['cram'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  download:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['download'] && !cancelled() }}
+    needs: [decision, docker, git, build, msys2]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['download'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  hg-clone:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['hg-clone'] && !cancelled() }}
+    needs: [decision, docker, hg]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['hg-clone'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  clone:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['clone'] && !cancelled() }}
+    needs: [decision, git, build, hg-clone]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['clone'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  graft:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['graft'] && !cancelled() }}
+    needs: [decision, docker, git, build, hg, clone, hg-clone]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['graft'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  test:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['test'] && !cancelled() }}
+    needs: [decision, docker, git, build, hg, clone, hg-clone]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['test'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
+  upload:
+    # Disabled until we turn it off on taskcluster.
+    if: ${{ false && fromJSON(needs.decision.outputs.matrix)['upload'] && !failure() }}
+    needs: [decision, docker, build, cram, graft, test]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['upload'] }}
+    runs-on: ${{ matrix.runner }}
+    env:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}