Extended GHA CI to test HTTP-based push ingest on the second pass of … #3375
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Qserv CI workflow | |
| --- | |
| name: "CI" | |
| on: | |
| push: | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| jobs: | |
| image-names: | |
| # This job works around an issue with actions/checkout @v2 and @v3 where if | |
| # a workflow is triggered by an annotated tag push, the tag may not be | |
| # visible to git in the cloned repository. The issue for that problem is at | |
| # https://github.com/actions/checkout/issues/290 | |
| # The workaround implemented here does two things: | |
| # 1. After checking out the repo, there is a step in this job that force | |
| # fetches tags. This ensures that all tags are available locally. | |
| # 2. Image names are determined and saved. These image names are derived | |
| # from the certain files and tags in the repo. Dependant jobs use the | |
| # images names stored in this job. | |
| # Image names must be cached after force fetching tags in a single job | |
| # instead of force fetching tags in each job because if a new release tag is | |
| # pushed while a workflow is running it's possible the derived image names | |
| # could change from job to job, breaking the running workflow in a difficult | |
| # to understand way. | |
| name: Save image names | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| build-image: ${{ steps.image-names.outputs.build-image }} | |
| user-build-image: ${{ steps.image-names.outputs.user-build-image }} | |
| run-base-image: ${{ steps.image-names.outputs.run-base-image }} | |
| mariadb-image: ${{ steps.image-names.outputs.mariadb-image }} | |
| qserv-image: ${{ steps.image-names.outputs.qserv-image }} | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name: force update tags | |
| run: git fetch --force --tags | |
| - name: Save image names | |
| id: image-names | |
| run: | | |
| echo "build-image=$(./admin/local/cli/qserv env --build-image)" >> "$GITHUB_OUTPUT" | |
| echo "user-build-image=$(./admin/local/cli/qserv env --user-build-image)" >> "$GITHUB_OUTPUT" | |
| echo "run-base-image=$(./admin/local/cli/qserv env --run-base-image)" >> "$GITHUB_OUTPUT" | |
| echo "mariadb-image=$(./admin/local/cli/qserv env --mariadb-image)" >> "$GITHUB_OUTPUT" | |
| echo "qserv-image=$(./admin/local/cli/qserv env --qserv-image)" >> "$GITHUB_OUTPUT" | |
| update-base-images: | |
| name: Update base images | |
| runs-on: ubuntu-20.04 | |
| needs: image-names | |
| outputs: | |
| build-rebuilt: ${{ steps.rebuild.outputs.build-rebuilt }} | |
| run-base-rebuilt: ${{ steps.rebuild.outputs.run-base-rebuilt }} | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| password: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| - name: Rebuild and push build image if needed | |
| id: rebuild | |
| run: | | |
| if [[ $(./admin/local/cli/qserv --log-level DEBUG dh-image-exists ${{ needs.image-names.outputs.build-image }}) == "True" ]]; then | |
| echo "Build image already on docker hub; skipping..." | |
| echo "build-rebuilt=False" >> "$GITHUB_OUTPUT" | |
| else | |
| ./admin/local/cli/qserv --log-level DEBUG build-build-image --push-image \ | |
| --build-image ${{ needs.image-names.outputs.build-image }} | |
| echo "build-rebuilt=True" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| - name: Rebuild and push run base image if needed | |
| run: | | |
| if [[ $(./admin/local/cli/qserv --log-level DEBUG dh-image-exists ${{ needs.image-names.outputs.run-base-image }}) == "True" ]]; then | |
| echo "Run base image already on docker hub; skipping..." | |
| echo "run-base-rebuilt=False" >> "$GITHUB_OUTPUT" | |
| else | |
| ./admin/local/cli/qserv --log-level DEBUG build-run-base-image --push-image \ | |
| --run-base-image ${{ needs.image-names.outputs.run-base-image }} | |
| echo "run-base-rebuilt=True" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| update-mariadb-image: | |
| name: Update MariaDB image | |
| runs-on: ubuntu-20.04 | |
| needs: image-names | |
| outputs: | |
| rebuilt: ${{ steps.rebuild.outputs.rebuilt }} | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| password: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| - name: Rebuild and push if needed | |
| id: rebuild | |
| run: | | |
| if [[ $(./admin/local/cli/qserv --log-level DEBUG dh-image-exists ${{ needs.image-names.outputs.mariadb-image }}) == "True" ]]; then | |
| echo "MariaDB image already on docker hub; skipping..." | |
| echo "rebuilt=False" >> "$GITHUB_OUTPUT" | |
| else | |
| ./admin/local/cli/qserv --log-level DEBUG build-mariadb-image --push-image --mariadb-image ${{ needs.image-names.outputs.mariadb-image }} | |
| echo "rebuilt=True" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| documentation: | |
| name: Documentation | |
| runs-on: ubuntu-20.04 | |
| needs: [image-names, update-base-images] | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name: Check for changes | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| docs: | |
| - 'doc/**' | |
| - name: Prepare user build image | |
| if: needs.update-base-images.outputs.build-rebuilt == 'True' || steps.filter.outputs.docs == 'true' | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG build-user-build-image \ | |
| --group docker_outer \ | |
| --build-image ${{ needs.image-names.outputs.build-image }} \ | |
| --user-build-image ${{ needs.image-names.outputs.user-build-image }} | |
| - name: Build docs | |
| if: needs.update-base-images.outputs.build-rebuilt == 'True' || steps.filter.outputs.docs == 'true' | |
| run: | | |
| ./admin/local/cli/qserv build-docs --cmake --linkcheck \ | |
| --user-build-image ${{ needs.image-names.outputs.user-build-image }} | |
| env: | |
| QSERV_GH_EVENT_NAME: ${{ github.event_name }} | |
| QSERV_GH_HEAD_REF: ${{ github.head_ref }} | |
| QSERV_GH_REF: ${{ github.ref }} | |
| - name: Upload to LSST the Docs | |
| if: needs.update-base-images.outputs.build-rebuilt == 'True' || steps.filter.outputs.docs == 'true' | |
| uses: lsst-sqre/ltd-upload@v1 | |
| with: | |
| project: "Qserv" | |
| dir: "build/doc/html" | |
| username: ${{ secrets.LTD_USERNAME }} | |
| password: ${{ secrets.LTD_PASSWORD }} | |
| update-run-image: | |
| name: Update Qserv image | |
| runs-on: ubuntu-20.04 | |
| needs: [image-names, update-base-images] | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name : Update submodules | |
| run: | | |
| git submodule update --init | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| password: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| - name: Prepare user build image | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG build-user-build-image \ | |
| --group docker_outer \ | |
| --build-image ${{ needs.image-names.outputs.build-image }} \ | |
| --user-build-image ${{ needs.image-names.outputs.user-build-image }} | |
| - name: Build lite-qserv image | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG build \ | |
| --qserv-image ${{ needs.image-names.outputs.qserv-image }} \ | |
| --run-base-image ${{ needs.image-names.outputs.run-base-image }} \ | |
| --user-build-image ${{ needs.image-names.outputs.user-build-image }} \ | |
| --pull-image \ | |
| --push-image \ | |
| --clang-format CHECK \ | |
| -j2 | |
| env: | |
| QSERV_DH_USER: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| QSERV_DH_TOKEN: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| compose-integration-tests: | |
| name: Integration tests (compose) | |
| runs-on: ubuntu-20.04 | |
| needs: [image-names, update-mariadb-image, update-run-image] | |
| steps: | |
| - name: Install python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install docker-compose | |
| uses: KengoTODA/actions-setup-docker-compose@v1 | |
| with: | |
| version: '1.29.2' | |
| - name: Install click | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install click pyyaml requests | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # 0 is "all history and branch tags" | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.FM_DOCKERHUB_TOKEN_USER }} | |
| password: ${{ secrets.FM_DOCKERHUB_TOKEN }} | |
| - name: Launch qserv | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG up \ | |
| --qserv-image ${{ needs.image-names.outputs.qserv-image }} \ | |
| --mariadb-image ${{ needs.image-names.outputs.mariadb-image }} | |
| - name: Check Qserv containers | |
| run: sleep 180 && docker ps -a | |
| - name: Run integration tests | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG itest \ | |
| --qserv-image ${{ needs.image-names.outputs.qserv-image }} \ | |
| --mariadb-image ${{ needs.image-names.outputs.mariadb-image }} | |
| - name: Run integration tests of the HTTP frontend | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG itest-http --reload --load-http \ | |
| --qserv-image ${{ needs.image-names.outputs.qserv-image }} \ | |
| --mariadb-image ${{ needs.image-names.outputs.mariadb-image }} | |
| - name: Check Qserv containers | |
| if: always() | |
| run: docker ps -a | |
| - name: Replication Controller Log | |
| if: always() | |
| run: docker logs ${USER}_repl_controller_1 | |
| - name: Replication Registry Log | |
| if: always() | |
| run: docker logs ${USER}_repl_registry_1 | |
| - name: Replication Database Log | |
| if: always() | |
| run: docker logs ${USER}_repl_mariadb_1 | |
| - name: Replication Worker 0 Log | |
| if: always() | |
| run: docker logs ${USER}_repl_worker_0_1 | |
| - name: Replication Worker 1 Log | |
| if: always() | |
| run: docker logs ${USER}_repl_worker_1_1 | |
| - name: Czar MySQL Proxy Log | |
| if: always() | |
| run: docker logs ${USER}_czar_proxy_1 | |
| - name: Czar CMSD Log | |
| if: always() | |
| run: docker logs ${USER}_czar_cmsd_1 | |
| - name: Czar XROOTD Log | |
| if: always() | |
| run: docker logs ${USER}_czar_xrootd_1 | |
| - name: Czar MariaDB Log | |
| if: always() | |
| run: docker logs ${USER}_czar_mariadb_1 | |
| - name: Qzerv Worker 0 CMSD Log | |
| if: always() | |
| run: docker logs ${USER}_worker_cmsd_0_1 | |
| - name: Qzerv Worker 0 XROOTD Log | |
| if: always() | |
| run: docker logs ${USER}_worker_xrootd_0_1 | |
| - name: Qzerv Worker 0 MariaDB Log | |
| if: always() | |
| run: docker logs ${USER}_worker_mariadb_0_1 | |
| - name: Qzerv Worker 1 CMSD Log | |
| if: always() | |
| run: docker logs ${USER}_worker_cmsd_1_1 | |
| - name: Qzerv Worker 1 XROOTD Log | |
| if: always() | |
| run: docker logs ${USER}_worker_xrootd_1_1 | |
| - name: Qzerv Worker 1 MariaDB Log | |
| if: always() | |
| run: docker logs ${USER}_worker_mariadb_1_1 | |
| - name: Remove integration test volumes | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG itest-rm | |
| - name: Shut down qserv | |
| run: | | |
| ./admin/local/cli/qserv --log-level DEBUG down \ | |
| -v | |
| notify-on-fail: | |
| name: Notify Slack if fail on main | |
| runs-on: ubuntu-20.04 | |
| needs: [documentation, compose-integration-tests] | |
| if: github.ref == 'refs/heads/main' && failure() | |
| steps: | |
| - name: Notify | |
| uses: voxmedia/github-action-slack-notify-build@v1 | |
| with: | |
| channel_id: G2JPZ3GC8 # this is the channel id of the dm_db_team room | |
| status: FAILED | |
| color: danger | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.QSERV_GHA_BUILD_NOTIFICATIONS }} |