From 902aec9d0b85ec1ab625463c3bb2d8a7b1a7245b Mon Sep 17 00:00:00 2001 From: Raunak Bhagat Date: Mon, 16 Dec 2024 14:23:11 -0800 Subject: [PATCH] Add subdir structure to build path --- .github/ci-scripts/upload_wheel_to_s3.py | 31 ++++++++++--------- .github/workflows/build-commit.yaml | 39 ++++++++---------------- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/.github/ci-scripts/upload_wheel_to_s3.py b/.github/ci-scripts/upload_wheel_to_s3.py index fe53c3f659..7988451a00 100644 --- a/.github/ci-scripts/upload_wheel_to_s3.py +++ b/.github/ci-scripts/upload_wheel_to_s3.py @@ -2,43 +2,44 @@ from pathlib import Path import boto3 -import constants import wheellib +BUCKET_NAME = "github-actions-artifacts-bucket" +WHEEL_SUFFIX = ".whl" + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--commit-hash", required=True) - parser.add_argument("--platform-substring", required=True, choices=["x86", "aarch", "arm"]) - parser.add_argument("--path-to-wheel-dir", required=True) + parser.add_argument("--commit-hash", type=str, required=True) + parser.add_argument("--subdir", type=str, required=True) + parser.add_argument("--platform-substring", type=str, required=True, choices=["x86", "aarch", "arm"]) + parser.add_argument("--path-to-wheel-dir", type=Path, required=True) args = parser.parse_args() - commit_hash = args.commit_hash - platform_substring = args.platform_substring - path_to_wheel_dir = Path(args.path_to_wheel_dir) - - assert path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {path_to_wheel_dir}" - wheelpaths = iter(filepath for filepath in path_to_wheel_dir.iterdir() if filepath.suffix == constants.WHEEL_SUFFIX) + assert args.path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {args.path_to_wheel_dir}" + wheelpaths = iter(filepath for filepath in args.path_to_wheel_dir.iterdir() if filepath.suffix == WHEEL_SUFFIX) def f(wheelpath: Path) -> bool: platform_tag = wheellib.get_platform_tag(wheelpath.name) - return platform_substring in platform_tag + return args.platform_substring in platform_tag filtered_wheelpaths: list[Path] = list(filter(f, wheelpaths)) length = len(filtered_wheelpaths) if length == 0: - raise RuntimeError(f"No wheels found that match the given platform substring: {platform_substring}; expected 1") + raise RuntimeError( + f"No wheels found that match the given platform substring: {args.platform_substring}; expected 1" + ) elif length > 1: raise RuntimeError( - f"""Multiple wheels found that match the given platform substring: {platform_substring}; expected just 1 + f"""Multiple wheels found that match the given platform substring: {args.platform_substring}; expected just 1 Wheels available: {wheelpaths}""" ) [wheelpath] = filtered_wheelpaths s3 = boto3.client("s3") - destination = Path("builds") / commit_hash / wheelpath.name + destination = Path("builds") / args.commit_hash / args.subdir / wheelpath.name s3.upload_file( Filename=wheelpath, - Bucket=constants.BUCKET_NAME, + Bucket=BUCKET_NAME, Key=str(destination), ExtraArgs={"ACL": "public-read"}, ) diff --git a/.github/workflows/build-commit.yaml b/.github/workflows/build-commit.yaml index a8a77417ef..0b099c62f1 100644 --- a/.github/workflows/build-commit.yaml +++ b/.github/workflows/build-commit.yaml @@ -34,6 +34,11 @@ jobs: fi echo "platform_substring=$platform_substring" >> $GITHUB_ENV echo "Running on $platform_substring build machines" + - name: Set unique run id + run: | + run_id="${{ github.run_id }}_${{ github.run_attempt }}" + echo "run_id=$run_id" >> $GITHUB_ENV + echo "Unique run id: $run_id" - name: Checkout repo uses: actions/checkout@v4 with: @@ -59,46 +64,26 @@ jobs: uv v source .venv/bin/activate uv pip install boto3 packaging - - name: Check if build already exists in AWS S3 - run: | - source .venv/bin/activate - wheel_name=$(python .github/ci-scripts/get_wheel_name_from_s3.py \ - --commit-hash "${{ github.sha }}" \ - --platform-substring "$platform_substring" \ - ) - if [ "$wheel_name" ]; then - echo "Python wheel for this commit already built and uploaded" - else - echo "No python wheel for this commit found; proceeding with build" - fi - echo "wheel_name=$wheel_name" >> $GITHUB_ENV - name: Build release wheel run: | - if [ "$wheel_name" ]; then - echo "Python wheel for this commit already built and uploaded" - exit 0 - fi - export CARGO_TARGET_DIR=~/target source .venv/bin/activate + export CARGO_TARGET_DIR=~/target uv pip install pip maturin boto3 maturin build --release - name: Upload wheel to AWS S3 run: | - if [ "$wheel_name" ]; then - echo "Python wheel for this commit already built and uploaded" - exit 0 - fi source .venv/bin/activate wheel_name=$(python .github/ci-scripts/upload_wheel_to_s3.py \ - --commit-hash "${{ github.sha }}" \ - --platform-substring "$platform_substring" \ - --path-to-wheel-dir ~/target/wheels \ + --commit-hash="${{ github.sha }}" \ + --subdir="$run_id" \ + --platform-substring="$platform_substring" \ + --path-to-wheel-dir="~/target/wheels" \ ) echo "wheel_name=$wheel_name" >> $GITHUB_ENV - name: Print url of the built wheel to GitHub Actions Summary Page run: | - console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$wheel_name" - download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$wheel_name" + console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$run_id/$wheel_name" + download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$run_id/$wheel_name" echo "View the location of the built wheel in the AWS console here:" >> $GITHUB_STEP_SUMMARY echo "$console_url" >> $GITHUB_STEP_SUMMARY