From 9514f6874c068da68a3e4a0c7736fb7c6c8b2ca1 Mon Sep 17 00:00:00 2001 From: Oliver Koenig Date: Thu, 18 Apr 2024 21:07:36 +0200 Subject: [PATCH] test with sandbox --- .github/container/manifest.yaml | 3 + .github/workflows/_sandbox.yaml | 177 ++++++++++++++++++++++++++------ 2 files changed, 146 insertions(+), 34 deletions(-) diff --git a/.github/container/manifest.yaml b/.github/container/manifest.yaml index 712b13b55..0aede0826 100644 --- a/.github/container/manifest.yaml +++ b/.github/container/manifest.yaml @@ -113,9 +113,12 @@ jax-triton: mode: git-clone maxtext: url: https://github.com/google/maxtext.git + mirror_url: https://github.com/nvjax-svc-0/maxtext.git tracking_ref: main latest_verified_commit: 78daad198544def8274dbd656d122fbe6a0e1129 mode: git-clone + patches: + mirror/patch/test_rosetta_maxtext: file://patches/maxtext/mirror-patch-rosetta-maxtext.patch levanter: url: https://github.com/stanford-crfm/levanter.git tracking_ref: main diff --git a/.github/workflows/_sandbox.yaml b/.github/workflows/_sandbox.yaml index 7b90b72ca..af3918e32 100644 --- a/.github/workflows/_sandbox.yaml +++ b/.github/workflows/_sandbox.yaml @@ -1,41 +1,150 @@ -name: "~Sandbox" - +name: Sandbox +run-name: CI-amd64 on: - workflow_dispatch: + pull_request: + types: + - opened + - reopened + - ready_for_review + - synchronize + paths-ignore: + - "**.md" + +env: + DEFAULT_MANIFEST_ARTIFACT_NAME: bumped-manifest + +permissions: + contents: read # to fetch code + actions: write # to cancel previous workflows + packages: write # to upload container jobs: - sandbox: + metadata: runs-on: ubuntu-22.04 + outputs: + PUBLISH: ${{ steps.if-publish.outputs.PUBLISH }} + BUMP_MANIFEST: ${{ steps.manifest-branch.outputs.BUMP_MANIFEST }} + MANIFEST_ARTIFACT_NAME: ${{ steps.manifest-branch.outputs.MANIFEST_ARTIFACT_NAME }} + MANIFEST_BRANCH: ${{ steps.manifest-branch.outputs.MANIFEST_BRANCH }} + MERGE_BUMPED_MANIFEST: ${{ steps.manifest-branch.outputs.MERGE_BUMBED_MANIFEST }} steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Set build date + id: date + shell: bash -x -e {0} + run: | + BUILD_DATE=$(TZ='US/Los_Angeles' date '+%Y-%m-%d') + echo "BUILD_DATE=${BUILD_DATE}" >> $GITHUB_OUTPUT + + - name: Determine whether results will be 'published' + id: if-publish + shell: bash -x -e {0} + run: | + echo "PUBLISH=${{ github.event_name == 'schedule' || inputs.PUBLISH }}" >> $GITHUB_OUTPUT + + - name: Set manifest branch name + id: manifest-branch + shell: bash -x -e {0} + run: | + BUMP_MANIFEST=${{ 'true' }} + MERGE_BUMPED_MANIFEST=${{ github.event_name == 'schedule' || inputs.MERGE_BUMPED_MANIFEST || 'false' }} + # Prepend nightly manifest branch with "z" to make it appear at the end + if [[ "$BUMP_MANIFEST" == "true" ]]; then + # This branch is for scheduled nightlies or manually run nightlies + MANIFEST_BRANCH=znightly-${{ steps.date.outputs.BUILD_DATE }}-${{ github.run_id }} + MANIFEST_ARTIFACT_NAME=${{ env.DEFAULT_MANIFEST_ARTIFACT_NAME }} + else + # This branch is for presubmits (no bumping needed) + MANIFEST_BRANCH=${{ github.sha }} + # Empty artifact name means to use the one in version control + MANIFEST_ARTIFACT_NAME="" + fi + echo "MANIFEST_BRANCH=$MANIFEST_BRANCH" | tee -a $GITHUB_OUTPUT + echo "MANIFEST_ARTIFACT_NAME=$MANIFEST_ARTIFACT_NAME" | tee -a $GITHUB_OUTPUT + echo "BUMP_MANIFEST=$BUMP_MANIFEST" | tee -a $GITHUB_OUTPUT + echo "MERGE_BUMBED_MANIFEST=$MERGE_BUMPED_MANIFEST" | tee -a $GITHUB_OUTPUT + if [[ "$BUMP_MANIFEST" == "false" && "$MERGE_BUMPED_MANIFEST" == "true" ]]; then + echo "Error: If BUMP_MANIFEST=false, MERGE_BUMPED_MANIFEST cannot be true" >&2 + exit 1 + fi + + bump-manifest: + needs: metadata + runs-on: ubuntu-22.04 + steps: + - name: Check out the repository under ${GITHUB_WORKSPACE} + uses: actions/checkout@v4 - - name: Print usage + - name: Test if manifest bump is functional, and save result to a new file + working-directory: .github/container + shell: bash -x -e {0} run: | - cat << EOF - This is an empty workflow file located in the main branch of your - repository. It serves as a testing ground for new GitHub Actions on - development branches before merging them to the main branch. By - defining and overloading this workflow on your development branch, - you can test new actions without affecting your main branch, ensuring - a smooth integration process once the changes are ready to be merged. - - Usage: - - 1. In your development branch, modify the sandbox.yml workflow file - to include the new actions you want to test. Make sure to commit - the changes to the development branch. - 2. Navigate to the 'Actions' tab in your repository, select the - '~Sandbox' workflow, and choose your development branch from the - branch dropdown menu. Click on 'Run workflow' to trigger the - workflow on your development branch. - 3. Once you have tested and verified the new actions in the Sandbox - workflow, you can incorporate them into your main workflow(s) and - merge the development branch into the main branch. Remember to - revert the changes to the sandbox.yml file in the main branch to - keep it empty for future testing. - EOF + bash bump.sh --input-manifest manifest.yaml --output-manifest manifest.yaml.new --base-patch-dir ./patches-new + + - name: Maybe replace current manifest/patches with the new one and show diff + working-directory: .github/container + shell: bash -x -e {0} + run: | + if [[ "${{ needs.metadata.outputs.BUMP_MANIFEST }}" == "true" ]]; then + mv manifest.yaml.new manifest.yaml + rm -rf patches + mv patches-new patches + else + rm -rf patches-new manifest.yaml.new + fi + sed -i 's|file://patches-new/|file://patches/|g' manifest.yaml + git diff + + - name: Upload bumped manifest/patches to be used in build-base + if: needs.metadata.outputs.MANIFEST_ARTIFACT_NAME != '' + uses: actions/upload-artifact@v4 + with: + name: ${{ needs.metadata.outputs.MANIFEST_ARTIFACT_NAME }} + path: | + .github/container/manifest.yaml + .github/container/patches + + build-base: + uses: ./.github/workflows/_build_base.yaml + needs: [metadata, bump-manifest] + with: + ARCHITECTURE: amd64 + BUILD_DATE: 20240418 + MANIFEST_ARTIFACT_NAME: ${{ needs.metadata.outputs.MANIFEST_ARTIFACT_NAME }} + secrets: inherit + + build-jax: + needs: build-base + uses: ./.github/workflows/_build.yaml + with: + ARCHITECTURE: amd64 + ARTIFACT_NAME: artifact-jax-build + BADGE_FILENAME: badge-jax-build + BUILD_DATE: 20240418 + BASE_IMAGE: ${{ needs.build-base.outputs.DOCKER_TAG }} + CONTAINER_NAME: jax + DOCKERFILE: .github/container/Dockerfile.jax + RUNNER_SIZE: large + secrets: inherit + + build-upstream-maxtext: + needs: build-jax + uses: ./.github/workflows/_build.yaml + with: + ARCHITECTURE: amd64 + ARTIFACT_NAME: artifact-maxtext-build + BADGE_FILENAME: badge-maxtext-build + BUILD_DATE: 20240418 + BASE_IMAGE: ${{ needs.build-jax.outputs.DOCKER_TAG_MEALKIT }} + CONTAINER_NAME: maxtext + DOCKERFILE: .github/container/Dockerfile.maxtext.amd64 + secrets: inherit + + build-rosetta-maxtext: + needs: build-upstream-maxtext + uses: ./.github/workflows/_build_rosetta.yaml + with: + ARCHITECTURE: amd64 + BUILD_DATE: 20240418 + BASE_IMAGE: ${{ needs.build-upstream-maxtext.outputs.DOCKER_TAG_MEALKIT }} + BASE_LIBRARY: maxtext + secrets: inherit