feat(clp-s): Add support for ingesting logs from S3. #2280
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "clp-core-build" | |
| on: | |
| pull_request: | |
| paths: | |
| - ".github/actions/clp-core-build/action.yaml" | |
| - ".github/actions/clp-core-build-containers/action.yaml" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - ".gitmodules" | |
| - "components/core/**" | |
| - "deps-tasks.yml" | |
| - "Taskfile.yml" | |
| - "tools/scripts/deps-download/**" | |
| - "!components/core/tools/scripts/lib_install/macos/**" | |
| push: | |
| paths: | |
| - ".github/actions/clp-core-build/action.yaml" | |
| - ".github/actions/clp-core-build-containers/action.yaml" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - ".gitmodules" | |
| - "components/core/**" | |
| - "deps-tasks.yml" | |
| - "Taskfile.yml" | |
| - "tools/scripts/deps-download/**" | |
| - "!components/core/tools/scripts/lib_install/macos/**" | |
| schedule: | |
| # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) | |
| - cron: "15 0 * * *" | |
| workflow_dispatch: | |
| env: | |
| BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" | |
| DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" | |
| # Currency group to prevent multiple workflow instances from trying to publish container images | |
| concurrency: "${{github.workflow}}-${{github.ref}}" | |
| jobs: | |
| filter-relevant-changes: | |
| runs-on: "ubuntu-latest" | |
| outputs: | |
| centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" | |
| ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}" | |
| ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" | |
| clp_changed: "${{steps.filter.outputs.clp}}" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - name: "Filter relevant changes" | |
| uses: "dorny/paths-filter@v3" | |
| id: "filter" | |
| with: | |
| # Consider changes between the current commit and `main` | |
| # NOTE: If a pull request changes one of the images, then we need to (1) build the image | |
| # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull | |
| # request doesn't change an image, then we don't need to rebuild the image; instead we can | |
| # use the published image which is based on `main`. So when determining what files have | |
| # changed, we need to consider the delta between the current commit and `main` (rather | |
| # than the current and previous commits) in order to detect if we need to rebuild the | |
| # image (since it would be different from the published image). | |
| base: "main" | |
| filters: | | |
| centos_stream_9_image: | |
| - ".github/actions/**" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - "components/core/tools/scripts/lib_install/*.sh" | |
| - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" | |
| - "components/core/tools/scripts/lib_install/centos-stream-9/**" | |
| ubuntu_focal_image: | |
| - ".github/actions/**" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - "components/core/tools/scripts/lib_install/*.sh" | |
| - "components/core/tools/docker-images/clp-env-base-ubuntu-focal/**" | |
| - "components/core/tools/scripts/lib_install/ubuntu-focal/**" | |
| ubuntu_jammy_image: | |
| - ".github/actions/**" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - "components/core/tools/scripts/lib_install/*.sh" | |
| - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" | |
| - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" | |
| clp: | |
| - ".github/actions/**" | |
| - ".github/workflows/clp-core-build.yaml" | |
| - ".gitmodules" | |
| - "components/core/cmake/**" | |
| - "components/core/CMakeLists.txt" | |
| - "components/core/src/**" | |
| - "components/core/tests/**" | |
| - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" | |
| - "deps-tasks.yml" | |
| - "Taskfile.yml" | |
| - "tools/scripts/deps-download/**" | |
| centos-stream-9-deps-image: | |
| if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" | |
| needs: "filter-relevant-changes" | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build-containers" | |
| env: | |
| OS_NAME: "centos-stream-9" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| docker_context: "components/core" | |
| docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ | |
| /Dockerfile" | |
| push_deps_image: >- | |
| ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} | |
| token: "${{secrets.GITHUB_TOKEN}}" | |
| ubuntu-focal-deps-image: | |
| if: "needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'true'" | |
| needs: "filter-relevant-changes" | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build-containers" | |
| env: | |
| OS_NAME: "ubuntu-focal" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| docker_context: "components/core" | |
| docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ | |
| /Dockerfile" | |
| push_deps_image: >- | |
| ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} | |
| token: "${{secrets.GITHUB_TOKEN}}" | |
| ubuntu-jammy-deps-image: | |
| if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" | |
| needs: "filter-relevant-changes" | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build-containers" | |
| env: | |
| OS_NAME: "ubuntu-jammy" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| docker_context: "components/core" | |
| docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ | |
| /Dockerfile" | |
| push_deps_image: >- | |
| ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} | |
| token: "${{secrets.GITHUB_TOKEN}}" | |
| centos-stream-9-binaries: | |
| # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. | |
| if: >- | |
| success() | |
| || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') | |
| needs: | |
| - "centos-stream-9-deps-image" | |
| - "filter-relevant-changes" | |
| strategy: | |
| matrix: | |
| use_shared_libs: [true, false] | |
| name: "centos-stream-9-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins" | |
| continue-on-error: true | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build" | |
| env: | |
| OS_NAME: "centos-stream-9" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| use_shared_libs: "${{matrix.use_shared_libs}}" | |
| use_published_image: >- | |
| ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' | |
| || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} | |
| upload_binaries: "false" | |
| ubuntu-focal-binaries: | |
| # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. | |
| if: >- | |
| success() | |
| || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') | |
| needs: | |
| - "filter-relevant-changes" | |
| - "ubuntu-focal-deps-image" | |
| strategy: | |
| matrix: | |
| include: | |
| - use_shared_libs: true | |
| upload_binaries: false | |
| - use_shared_libs: false | |
| upload_binaries: true | |
| name: "ubuntu-focal-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins" | |
| continue-on-error: true | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build" | |
| env: | |
| OS_NAME: "ubuntu-focal" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| use_shared_libs: "${{matrix.use_shared_libs}}" | |
| use_published_image: >- | |
| ${{needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'false' | |
| || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} | |
| upload_binaries: "${{matrix.upload_binaries}}" | |
| binaries_artifact_name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" | |
| ubuntu-jammy-binaries: | |
| # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. | |
| if: >- | |
| success() | |
| || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') | |
| needs: | |
| - "filter-relevant-changes" | |
| - "ubuntu-jammy-deps-image" | |
| strategy: | |
| matrix: | |
| use_shared_libs: [true, false] | |
| name: "ubuntu-jammy-${{matrix.use_shared_libs && 'dynamic' || 'static'}}-linked-bins" | |
| continue-on-error: true | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "./.github/actions/clp-core-build" | |
| env: | |
| OS_NAME: "ubuntu-jammy" | |
| with: | |
| image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" | |
| use_shared_libs: "${{matrix.use_shared_libs}}" | |
| use_published_image: >- | |
| ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' | |
| || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} | |
| upload_binaries: "false" | |
| ubuntu-focal-binaries-image: | |
| # Run if the ancestor jobs were successful/skipped, building clp was successful, and this is a | |
| # push to `main`. | |
| if: >- | |
| !cancelled() | |
| && !failure() | |
| && needs.ubuntu-focal-binaries.result == 'success' | |
| && github.event_name == 'push' | |
| && github.ref == 'refs/heads/main' | |
| needs: "ubuntu-focal-binaries" | |
| runs-on: "ubuntu-latest" | |
| env: | |
| OS_NAME: "ubuntu-focal" | |
| TMP_OUTPUT_DIR: "/tmp" | |
| steps: | |
| - uses: "actions/checkout@v4" | |
| with: | |
| submodules: "recursive" | |
| - name: "Work around actions/runner-images/issues/6775" | |
| run: "chown $(id -u):$(id -g) -R ." | |
| shell: "bash" | |
| - uses: "actions/download-artifact@v4" | |
| with: | |
| name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" | |
| path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" | |
| - name: "Untar binaries" | |
| working-directory: >- | |
| ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} | |
| run: |- | |
| tar xf clp.tar | |
| rm clp.tar | |
| - uses: "docker/login-action@v3" | |
| with: | |
| registry: "ghcr.io" | |
| username: "${{github.actor}}" | |
| password: "${{secrets.GITHUB_TOKEN}}" | |
| - name: "Sanitize the repo's name" | |
| id: "sanitize_repo_name" | |
| run: |- | |
| # Docker doesn't support repository names with uppercase characters, so we convert to | |
| # lowercase here. | |
| lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') | |
| echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" | |
| shell: "bash" | |
| - id: "core_image_meta" | |
| uses: "docker/metadata-action@v5" | |
| with: | |
| images: >- | |
| ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} | |
| tags: "type=raw,value=${{github.ref_name}}" | |
| - uses: "docker/build-push-action@v5" | |
| with: | |
| context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" | |
| file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" | |
| push: true | |
| tags: "${{steps.core_image_meta.outputs.tags}}" | |
| labels: "${{steps.core_image_meta.outputs.labels}}" |