diff --git a/.github/actions/decision/action.yml b/.github/actions/decision/action.yml new file mode 100644 index 000000000..6006c94db --- /dev/null +++ b/.github/actions/decision/action.yml @@ -0,0 +1,40 @@ +name: Decision +description: Decision +outputs: + data: + description: Decision data + value: ${{ steps.decision.outputs.data }} +runs: + using: "composite" + steps: + - name: Decision + id: decision + shell: bash + run: | + python3 CI/run.py --decision | tee -a $GITHUB_OUTPUT + - run: | + cat<<'EOF' + ${{ toJSON(steps.decision.outputs) }} + EOF + shell: bash + - uses: actions/setup-node@v3 + if: ${{ steps.dependencies.outputs.data }} + with: + node-version: '20.x' + - shell: bash + if: ${{ steps.dependencies.outputs.data }} + run: npm install @actions/cache + - name: Check dependencies + if: ${{ steps.dependencies.outputs.data }} + uses: actions/github-script@v7 + with: + script: | + const cache = require('@actions/cache'); + const data = ${{ toJSON(fromJSON(steps.decision.outputs.data)) }}; + for (const mount of data) { + if (await cache.restoreCache([path.basename(mount.artifact)], mount.key, [], { lookupOnly: true }, true)) { + console.log(`Cache restored from key: ${mount.key}`); + } else { + core.setFailed(`Failed to restore cache from key: ${mount.key}`); + } + } diff --git a/.github/actions/mounts/action.yml b/.github/actions/mounts/action.yml new file mode 100644 index 000000000..0271a0ef3 --- /dev/null +++ b/.github/actions/mounts/action.yml @@ -0,0 +1,43 @@ +name: Mount dependencies +description: Mount dependencies +inputs: + name: + description: task name + required: true +runs: + using: "composite" + steps: + - name: Compute dependencies + id: dependencies + shell: bash + run: | + python3 CI/run.py --mounts "${{ inputs.name }}" >> $GITHUB_OUTPUT + - uses: actions/setup-node@v3 + if: ${{ steps.dependencies.outputs.data }} + with: + node-version: '20.x' + - shell: bash + if: ${{ steps.dependencies.outputs.data }} + run: npm install @actions/cache + - name: Mount dependencies + if: ${{ steps.dependencies.outputs.data }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const cache = require('@actions/cache'); + const data = ${{ toJSON(fromJSON(steps.dependencies.outputs.data)) }}; + const pwd = process.cwd(); + // Make actions/cache's getWorkingDirectory() return process.cwd. + delete process.env.GITHUB_WORKSPACE; + for (const mount of data) { + const dir = path.join(pwd, 'cache', mount.key, path.dirname(mount.artifact)); + fs.mkdirSync(dir, { recursive: true }); + process.chdir(dir); + if (await cache.restoreCache([path.basename(mount.artifact)], mount.key, [], {}, true)) { + console.log(`Cache restored from key: ${mount.key}`); + } else { + core.setFailed(`Failed to restore cache from key: ${mount.key}`); + } + } diff --git a/.github/actions/task/action.yml b/.github/actions/task/action.yml new file mode 100644 index 000000000..23e56b45e --- /dev/null +++ b/.github/actions/task/action.yml @@ -0,0 +1,39 @@ +name: Task +description: Run task +inputs: + name: + description: task name + required: true +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + if: ${{ runner.os == 'macOS' }} + with: + python-version: ${{ runner.arch == 'ARM64' && '3.11.7' || '3.9.14' }} + - name: Finish python setup + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + python3 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade + - name: Artifact + id: artifacts + shell: bash + run: | + python3 CI/run.py --artifacts "${{ inputs.name }}" >> $GITHUB_OUTPUT + - uses: actions/cache@v4 + if: ${{ steps.artifacts.outputs.key }} + id: cache + with: + path: ${{ join(fromJSON(steps.artifacts.outputs.path), '\n') }} + key: ${{ steps.artifacts.outputs.key }} + enableCrossOsArchive: true + - uses: ./.github/actions/mounts + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + with: + name: ${{ inputs.name }} + - name: ${{ inputs.name }} + shell: bash + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + python3 CI/run.py --cache cache --no-recurse --out . "${{ inputs.name }}" diff --git a/.github/workflows/env-tools.yml b/.github/workflows/env-tools.yml new file mode 100644 index 000000000..32c7b6c14 --- /dev/null +++ b/.github/workflows/env-tools.yml @@ -0,0 +1,261 @@ +# TODO: https://stackoverflow.com/questions/59180385/using-an-array-of-values-to-repeat-a-step-in-github-actions-workflow +name: environment and tools +on: [push] +jobs: + decision: + runs-on: ubuntu-latest + outputs: + data: ${{ steps.decision.outputs.data }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/decision + id: decision + + docker-base: + if: ${{ github.sha == '1' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/task + with: + name: "docker image: base" + + docker: + if: ${{ !cancelled() }} + needs: [decision, docker-base] + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.decision.outputs.data)[github.job] }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/task + with: + name: "${{ matrix.task }}" + +# msys-base: +# if: ${{ github.sha == '1' }} +# runs-on: ubuntu-latest +# needs: docker-base +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: "msys2 image: base x86_64" +# +# msys: +# if: ${{ github.sha == '1' }} +# strategy: +# fail-fast: false +# matrix: +# image: +# - build +# - test +# runs-on: windows-latest +# needs: msys-base +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: "msys2 image: ${{ matrix.image }} x86_64" +# +# git: +# if: ${{ github.sha == '1' }} +# strategy: +# fail-fast: false +# matrix: +# version: +# - 1.8.5 +# - 2.7.4 +# - 2.47.1 +# - 2.47.1.windows.1 windows x86_64 +# runner: [ubuntu-latest] +# include: +# - version: 2.47.1 macos x86_64 +# runner: macos-13 +# - version: 2.47.1 macos arm64 +# runner: macos-14 +# runs-on: ${{ matrix.runner }} +# needs: docker +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: git v${{ matrix.version }} +# +# mercurial: +# if: ${{ github.sha == '1' }} +# strategy: +# fail-fast: false +# matrix: +# version: +# - 1.9.3 +# - 2.5.4 +# - 3.4.2 +# - 6.8 +# runner: [ubuntu-latest] +# include: +# - version: 6.8 windows x86_64 +# runner: windows-latest +# - version: 6.8 macos x86_64 +# runner: macos-13 +# - version: 6.8 macos arm64 +# runner: macos-14 +# runs-on: ${{ matrix.runner }} +# needs: [docker, msys] +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: hg v${{ matrix.version }} +# +# git-cinnabar: +# if: ${{ !cancelled() }} +# strategy: +# fail-fast: false +# matrix: +# type: +# - linux arm64 +# - linux x86_64 +# - linux x86_64 asan +# - linux x86_64 coverage +# - windows x86_64 +# runner: [ubuntu-latest] +# include: +# - type: macos x86_64 +# runner: macos-13 +# - type: macos arm64 +# runner: macos-14 +# runs-on: ${{ matrix.runner }} +# needs: docker +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: build ${{ matrix.type }} +# +# cram: +# if: ${{ !cancelled() }} +# strategy: +# fail-fast: false +# matrix: +# type: +# - git-2.47.1 hg-6.8 +# - git-2.47.1 hg-6.8 asan +# runner: [ubuntu-latest] +# include: +# - type: git-2.47.1 hg-6.8 macos x86_64 +# runner: macos-13 +# - type: git-2.47.1 hg-6.8 macos arm64 +# runner: macos-14 +# runs-on: ${{ matrix.runner }} +# needs: [docker, git, mercurial, git-cinnabar] +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: cram w/ ${{ matrix.type }} +# +# download: +# if: ${{ !cancelled() }} +# strategy: +# fail-fast: false +# matrix: +# include: +# - type: linux x86_64 +# runner: ubuntu-latest +# - type: macos x86_64 +# runner: macos-13 +# - type: macos arm64 +# runner: macos-14 +# - type: windows x86_64 +# runner: windows-latest +# runs-on: ${{ matrix.runner }} +# needs: [docker, git, git-cinnabar, msys] +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: download build ${{ matrix.type }} +# +# hg-clone: +# if: ${{ github.sha == '1' }} +# needs: [docker, mercurial] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: hg clone w/ 6.8 +# +# clone: +# if: ${{ !cancelled() }} +# needs: [git, git-cinnabar, hg-clone] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: clone w/ ${{ github.sha }} +# +# graft-tests: +# if: ${{ !cancelled() }} +# strategy: +# fail-fast: false +# matrix: +# type: +# - git-2.47.1 hg-6.8 +# - git-2.47.1 hg-6.8 asan +# runner: [ubuntu-latest] +# include: +# - type: git-2.47.1 hg-6.8 macos x86_64 +# runner: macos-13 +# - type: git-2.47.1 hg-6.8 macos arm64 +# runner: macos-14 +# runs-on: ${{ matrix.runner }} +# needs: [docker, git, git-cinnabar, mercurial, clone, hg-clone] +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: graft tests w/ ${{ matrix.type }} +# +# tests: +# if: ${{ !cancelled() }} +# strategy: +# fail-fast: false +# matrix: +# type: +# - git-2.47.1 hg-6.8 +# - git-2.47.1 hg-6.8 asan +# - git-2.47.1 hg-3.4.2 +# - git-2.47.1 hg-2.5.4 +# - git-2.47.1 hg-1.9.3 +# - git-2.7.4 hg-6.8 asan +# - git-1.8.5 hg-6.8 +# runner: [ubuntu-latest] +# include: +# - type: git-2.47.1 hg-6.8 macos x86_64 +# runner: macos-13 +# - type: git-2.47.1 hg-6.8 macos arm64 +# runner: macos-14 +# - type: git-2.47.1 hg-6.8 windows x86_64 +# runner: windows-latest +# runs-on: ${{ matrix.runner }} +# needs: [docker, git, git-cinnabar, mercurial, clone, hg-clone] +# steps: +# - uses: actions/checkout@v4 +# - uses: ./.github/actions/task +# with: +# name: test w/ ${{ matrix.type }} +# +# env: +# runs-on: ubuntu-latest +# steps: +# - shell: bash +# run: | +# env +# cat<<'EOF' +# ${{ toJSON(github) }} +# EOF diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index a0cd6f547..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: rust checks -on: [push, pull_request] -jobs: - clippy: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] - toolchain: [1.84.0] - include: - - os: windows-latest - toolchain: 1.84.0-x86_64-pc-windows-gnu - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-${{ runner.arch }}-clippy-${{ matrix.toolchain }}-${{ hashFiles('Cargo.lock', 'Cargo.toml') }} - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.toolchain }} - components: clippy - - uses: msys2/setup-msys2@v2 - with: - release: false - update: false - msystem: MINGW64 - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-make - patch - if: runner.os == 'Windows' - - name: Update PATH - run: | - (Resolve-Path c:/msys64/mingw64/bin).Path >> $Env:GITHUB_PATH - if: runner.os == 'Windows' - - uses: actions-rs-plus/clippy-check@v2 - with: - args: -vv -- -D warnings - - uses: actions-rs-plus/clippy-check@v2 - with: - args: -vv --features self-update -- -D warnings - - uses: actions-rs-plus/clippy-check@v2 - with: - args: -vv --no-default-features -- -D warnings - - uses: actions-rs-plus/clippy-check@v2 - with: - args: -vv --no-default-features --features self-update -- -D warnings - - fmt: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: 1.84.0 - components: rustfmt - - run: cargo fmt -- --check - - test: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - toolchain: [1.84.0, 1.75.0] - include: - - os: windows-latest - toolchain: 1.84.0-x86_64-pc-windows-gnu - - os: windows-latest - toolchain: 1.75.0-x86_64-pc-windows-gnu - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-${{ runner.arch }}-cargo-${{ matrix.toolchain }}-test-${{ hashFiles('Cargo.lock', 'Cargo.toml') }} - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.toolchain }} - - uses: msys2/setup-msys2@v2 - with: - release: false - update: false - msystem: MINGW64 - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-make - patch - if: runner.os == 'Windows' - - name: Update PATH - run: | - (Resolve-Path c:/msys64/mingw64/bin).Path >> $Env:GITHUB_PATH - if: runner.os == 'Windows' - - run: cargo test diff --git a/.github/workflows/taskcluster.yml b/.github/workflows/taskcluster.yml deleted file mode 100644 index 2478a936a..000000000 --- a/.github/workflows/taskcluster.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: taskcluster extra workers -on: [push] -jobs: - osx: - strategy: - fail-fast: false - matrix: - include: - - os: macos-13 - workerType: osx - pyver: '3.9.14' - - os: macos-14 - workerType: macos - pyver: '3.11.7' - runs-on: ${{ matrix.os }} - env: - secret: ${{ secrets.SECRET }} - TC_WORKER_TYPE: ${{ matrix.workerType }} - PATH: /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Frameworks/Python.framework/Versions/2.7/bin - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.pyver }} - - run: python3 -m pip install pip==20.3.4 wheel==0.37.0 --upgrade - - run: CI/start-worker.sh worker-$TC_WORKER_TYPE $TC_WORKER_TYPE diff --git a/.taskcluster.yml b/.taskcluster.yml deleted file mode 100644 index 351e42437..000000000 --- a/.taskcluster.yml +++ /dev/null @@ -1,90 +0,0 @@ -version: 1 -policy: - pullRequests: public -tasks: - - $if: 'tasks_for == "github-push" || (tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"])' - then: - $let: - e: # FIXME after https://github.com/taskcluster/json-e/issues/249 - $if: 'tasks_for == "github-push"' - then: - repo_name: ${event.repository.name} - repo_url: ${event.repository.url} - base_repo_url: ${event.repository.url} - login: ${event.pusher.name} - commit: ${event.after} - branch: - $if: 'event.ref[:11] == "refs/heads/"' - then: ${event.ref[11:]} - base_repo_name: ${event.repository.name} - base_login: ${event.repository.owner.name} - decision_id: - $if: 'event.ref[:10] == "refs/tags/"' - then: ${event.ref[10:]} - else: ${event.after} - else: - repo_name: ${event.pull_request.head.repo.name} - repo_url: ${event.pull_request.head.repo.clone_url} - base_repo_url: ${event.pull_request.base.repo.clone_url} - login: ${event.pull_request.head.user.login} - commit: ${event.pull_request.head.sha} - branch: ${event.pull_request.base.ref} - base_repo_name: ${event.pull_request.base.repo.name} - base_login: ${event.pull_request.base.repo.owner.login} - decision_id: - taskGroupId: {$eval: as_slugid("decision")} - in: - taskId: ${taskGroupId} - taskGroupId: ${taskGroupId} - schedulerId: taskcluster-github - provisionerId: proj-git-cinnabar - workerType: linux - retries: 5 - scopes: - $flatten: - - assume:repo:github.com/${e.base_login}/${e.base_repo_name}:decision-task - - $if: 'tasks_for == "github-push"' - then: - - 'queue:route:index.project.git-cinnabar.*' - - secrets:get:project/git-cinnabar/codecov - routes: - $if: 'e.decision_id' - then: - - 'index.project.git-cinnabar.decision.${e.decision_id}' - payload: - maxRunTime: 3600 - features: - taskclusterProxy: true - env: - TC_PROXY: "1" - TC_DATA: {$json: {$eval: e}} - TC_GROUP_ID: ${taskGroupId} - TC_IS_PUSH: {$if: 'tasks_for == "github-push"', then: "1"} - command: - - - sh - - '-c' - - >- - git clone -n ${e.base_repo_url} repo && - git -C repo remote add head ${e.repo_url} && - git -C repo remote update head && - git -c advice.detachedHead=false -C repo checkout ${e.commit} && - python3 -m venv venv && - venv/bin/pip install pyyaml==6.0.1 requests==2.27.1 setuptools==70.3.0 && - venv/bin/python3 repo/CI/decision.py - artifacts: - $if: 'e.decision_id' - then: - - name: public/actions.json - path: actions.json - type: file - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - expires: - $if: 'tasks_for == "action"' - then: {$fromNow: '6 hours'} - else: {$fromNow: '26 weeks'} - metadata: - name: decision task - description: ${e.repo_name} decision task - owner: ${e.login}@users.noreply.github.com - source: ${e.repo_url} diff --git a/CI/run.py b/CI/run.py new file mode 100644 index 000000000..ce637bf5e --- /dev/null +++ b/CI/run.py @@ -0,0 +1,205 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import errno +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time + +import variables + +variables.NO_INDEX = True +variables.TC_IS_PUSH = True +from decision import action +from tasks import Task + + +def link_or_copy(src, dest): + try: + os.link(src, dest) + except OSError as e: + if e.errno == errno.EXDEV: + shutil.copy2(src, dest) + else: + raise + + +def run_task(task, cwd, out=None, cache=None, recurse=True): + id = task.id + task = task.task + payload = task.get("payload", {}) + with tempfile.TemporaryDirectory(prefix="task", dir=cwd) as task_dir: + for mount in payload.get("mounts", []): + content = mount.get("content", {}) + task_id = content["taskId"] + dep_task = Task.by_id[task_id] + if cache and "routes" in dep_task.task: + assert len(dep_task.task["routes"]) == 1 + artifacts_base_dir = os.path.join(cache, dep_task.task["routes"][0]) + else: + artifacts_base_dir = os.path.join(cwd, task_id) + artifact = os.path.join(artifacts_base_dir, content["artifact"]) + if not os.path.exists(artifact): + if recurse: + run_task(dep_task, cwd, cache=cache) + else: + raise RuntimeError(f"Missing dependency {artifact}") + + if directory := mount.get("directory"): + assert "file" not in mount + directory = os.path.join(task_dir, directory) + assert mount.get("format", "tar.zst") + print(f"Extracting {os.path.basename(artifact)}", file=sys.stderr) + start = time.monotonic() + with subprocess.Popen( + ["zstd", "-cd", artifact], stdout=subprocess.PIPE + ) as proc: + stdout = io.BufferedReader(proc.stdout, 1024 * 1024) + with tarfile.open(fileobj=stdout, mode="r|") as tar: + for tarinfo in tar: + # We want to preserve file mode, but not timestamps. Owner would only + # matter when running as Admin/root, but we're not expecting to be. + tar.extract(tarinfo, path=directory, set_attrs=False) + if tarinfo.type == tarfile.REGTYPE: + os.chmod( + os.path.join(directory, tarinfo.name), tarinfo.mode + ) + end = time.monotonic() + print(f"Took {end - start:.2f}s", file=sys.stderr) + elif file := mount.get("file"): + assert "directory" not in mount + link_or_copy(artifact, os.path.join(task_dir, file)) + else: + assert False + + env = os.environ.copy() + env.update(payload.get("env", {})) + if task.get("workerType") == "windows": + task_cmd = os.path.join(task_dir, "task.cmd") + with open(task_cmd, "w") as fh: + fh.write("\n".join(payload.get("command", []))) + subprocess.check_call([os.path.abspath(task_cmd)], cwd=task_dir, env=env) + else: + for command in payload.get("command", []): + subprocess.check_call(command, cwd=task_dir, env=env) + if cache and "routes" in task: + assert len(task["routes"]) == 1 + artifacts_base_dir = os.path.join(cache, task["routes"][0]) + else: + artifacts_base_dir = os.path.join(cwd, id) + if out: + os.makedirs(out, exist_ok=True) + for artifact in payload.get("artifacts", []): + assert artifact.get("type") == "file" + dest = os.path.join(artifacts_base_dir, artifact["name"]) + os.makedirs(os.path.dirname(dest), exist_ok=True) + assert not artifact["name"].startswith("/") + link_or_copy(os.path.join(task_dir, artifact["path"]), dest) + if out: + link_or_copy(dest, os.path.join(out, os.path.basename(dest))) + + +def print_output(name, value): + value = json.dumps(value, separator=(',', ':')) + print(f"{name}={value}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cache", type=str, metavar="PATH") + parser.add_argument("--out", type=str, metavar="PATH") + parser.add_argument("--artifacts", action="store_true") + parser.add_argument("--mounts", action="store_true") + parser.add_argument("--decision", action="store_true") + parser.add_argument("--no-recurse", action="store_true") + parser.add_argument("task", nargs="?") + args = parser.parse_args() + action.by_name["decision"].func() + + if args.decision: + JOB_NAME = { + "hg": "mercurial", + "graft": "graft-test", + "build": "git-cinnabar", + } + RUNNER = { + "linux": "ubuntu-latest", + "osx": "macos-13", + "macos": "macos-14", + "windows": "windows-latest", + } + matrix = {} + for t in Task.by_id.values(): + name = t.task.get("metadata", {})["name"] + job_name = name.split()[0] + if job_name == "hg" and name.startswith("hg clone"): + job_name = "hg-clone" + if job_name in ("docker", "msys2") and "base" in name: + job_name = f"{job_name}-base" + job_name = JOB_NAME.get(job_name, job_name) + matrix.setdefault(job_name, []).append({ + "task": name, + "runner": RUNNER[t.task["workerType"]], + }) + print_output("data", matrix) + return + + if args.artifacts or args.mounts: + for t in Task.by_id.values(): + if t.task.get("metadata", {}).get("name") == args.task: + if args.artifacts: + task = t.task + payload = task.get("payload", {}) + paths = [ + os.path.basename(artifact["name"]) + for artifact in payload.get("artifacts", []) + ] + if "routes" in task: + print_output("key", task.get("routes")[0]) + print_output("path", paths) + elif args.mounts: + + def info(mount): + content = mount["content"] + return { + "artifact": content["artifact"], + "key": Task.by_id[content["taskId"]].task.get("routes")[0], + } + + data = [ + info(m) for m in t.task.get("payload", {}).get("mounts", []) + ] + if data: + print_output("data", data) + break + else: + print(f"Unknown task: {args.task}", file=sys.stderr) + return 1 + return + + with tempfile.TemporaryDirectory(prefix="run_task") as tmpdir: + for t in Task.by_id.values(): + if t.task.get("metadata", {}).get("name") == args.task: + run_task( + t, + cwd=tmpdir, + out=args.out, + cache=args.cache, + recurse=not args.no_recurse, + ) + break + else: + print(f"Unknown task: {args.task}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/CI/tasks.py b/CI/tasks.py index 3d3be8fe4..55b20017e 100644 --- a/CI/tasks.py +++ b/CI/tasks.py @@ -87,7 +87,7 @@ class Existing(str): def __init__(self): super(Index, self).__init__() - if os.environ.get("NO_INDEX"): + if NO_INDEX: self.session = None else: import requests diff --git a/CI/variables.py b/CI/variables.py index 350e4ccbf..7c752cfae 100644 --- a/CI/variables.py +++ b/CI/variables.py @@ -20,7 +20,7 @@ DEFAULT_DATA = { "repo_name": "git-cinnabar", "login": "glandium", - "commit": "HEAD", + "commit": os.environ.get("GITHUB_SHA", "HEAD"), "branch": "", "decision_id": "", } @@ -50,3 +50,4 @@ def get(k): DEFAULT_REPO = "https://hg.mozilla.org/users/mh_glandium.org/jqplot" REPO = os.environ.get("REPO", DEFAULT_REPO) +NO_INDEX = os.environ.get("NO_INDEX")