Skip to content

Commit

Permalink
Add subdir structure to build path
Browse files Browse the repository at this point in the history
  • Loading branch information
raunakab committed Dec 16, 2024
1 parent 6c21917 commit 902aec9
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 42 deletions.
31 changes: 16 additions & 15 deletions .github/ci-scripts/upload_wheel_to_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,44 @@
from pathlib import Path

import boto3
import constants
import wheellib

BUCKET_NAME = "github-actions-artifacts-bucket"
WHEEL_SUFFIX = ".whl"

if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--commit-hash", required=True)
parser.add_argument("--platform-substring", required=True, choices=["x86", "aarch", "arm"])
parser.add_argument("--path-to-wheel-dir", required=True)
parser.add_argument("--commit-hash", type=str, required=True)
parser.add_argument("--subdir", type=str, required=True)
parser.add_argument("--platform-substring", type=str, required=True, choices=["x86", "aarch", "arm"])
parser.add_argument("--path-to-wheel-dir", type=Path, required=True)
args = parser.parse_args()

commit_hash = args.commit_hash
platform_substring = args.platform_substring
path_to_wheel_dir = Path(args.path_to_wheel_dir)

assert path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {path_to_wheel_dir}"
wheelpaths = iter(filepath for filepath in path_to_wheel_dir.iterdir() if filepath.suffix == constants.WHEEL_SUFFIX)
assert args.path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {args.path_to_wheel_dir}"
wheelpaths = iter(filepath for filepath in args.path_to_wheel_dir.iterdir() if filepath.suffix == WHEEL_SUFFIX)

def f(wheelpath: Path) -> bool:
platform_tag = wheellib.get_platform_tag(wheelpath.name)
return platform_substring in platform_tag
return args.platform_substring in platform_tag

filtered_wheelpaths: list[Path] = list(filter(f, wheelpaths))

length = len(filtered_wheelpaths)
if length == 0:
raise RuntimeError(f"No wheels found that match the given platform substring: {platform_substring}; expected 1")
raise RuntimeError(
f"No wheels found that match the given platform substring: {args.platform_substring}; expected 1"
)
elif length > 1:
raise RuntimeError(
f"""Multiple wheels found that match the given platform substring: {platform_substring}; expected just 1
f"""Multiple wheels found that match the given platform substring: {args.platform_substring}; expected just 1
Wheels available: {wheelpaths}"""
)
[wheelpath] = filtered_wheelpaths
s3 = boto3.client("s3")
destination = Path("builds") / commit_hash / wheelpath.name
destination = Path("builds") / args.commit_hash / args.subdir / wheelpath.name
s3.upload_file(
Filename=wheelpath,
Bucket=constants.BUCKET_NAME,
Bucket=BUCKET_NAME,
Key=str(destination),
ExtraArgs={"ACL": "public-read"},
)
Expand Down
39 changes: 12 additions & 27 deletions .github/workflows/build-commit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ jobs:
fi
echo "platform_substring=$platform_substring" >> $GITHUB_ENV
echo "Running on $platform_substring build machines"
- name: Set unique run id
run: |
run_id="${{ github.run_id }}_${{ github.run_attempt }}"
echo "run_id=$run_id" >> $GITHUB_ENV
echo "Unique run id: $run_id"
- name: Checkout repo
uses: actions/checkout@v4
with:
Expand All @@ -59,46 +64,26 @@ jobs:
uv v
source .venv/bin/activate
uv pip install boto3 packaging
- name: Check if build already exists in AWS S3
run: |
source .venv/bin/activate
wheel_name=$(python .github/ci-scripts/get_wheel_name_from_s3.py \
--commit-hash "${{ github.sha }}" \
--platform-substring "$platform_substring" \
)
if [ "$wheel_name" ]; then
echo "Python wheel for this commit already built and uploaded"
else
echo "No python wheel for this commit found; proceeding with build"
fi
echo "wheel_name=$wheel_name" >> $GITHUB_ENV
- name: Build release wheel
run: |
if [ "$wheel_name" ]; then
echo "Python wheel for this commit already built and uploaded"
exit 0
fi
export CARGO_TARGET_DIR=~/target
source .venv/bin/activate
export CARGO_TARGET_DIR=~/target
uv pip install pip maturin boto3
maturin build --release
- name: Upload wheel to AWS S3
run: |
if [ "$wheel_name" ]; then
echo "Python wheel for this commit already built and uploaded"
exit 0
fi
source .venv/bin/activate
wheel_name=$(python .github/ci-scripts/upload_wheel_to_s3.py \
--commit-hash "${{ github.sha }}" \
--platform-substring "$platform_substring" \
--path-to-wheel-dir ~/target/wheels \
--commit-hash="${{ github.sha }}" \
--subdir="$run_id" \
--platform-substring="$platform_substring" \
--path-to-wheel-dir="~/target/wheels" \
)
echo "wheel_name=$wheel_name" >> $GITHUB_ENV
- name: Print url of the built wheel to GitHub Actions Summary Page
run: |
console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$wheel_name"
download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$wheel_name"
console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$run_id/$wheel_name"
download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$run_id/$wheel_name"
echo "View the location of the built wheel in the AWS console here:" >> $GITHUB_STEP_SUMMARY
echo "$console_url" >> $GITHUB_STEP_SUMMARY
Expand Down

0 comments on commit 902aec9

Please sign in to comment.