From f7ae499e4cb0baaff264ec222bee9356179f59b3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 15:34:32 -0500 Subject: [PATCH 01/22] Refactor Docker image workflows and add `ubuntu-jammy` Docker image building workflow. --- .../build.yaml} | 28 +++++++++--------- .../clp-excution-image/ubuntu-focal.yml | 29 +++++++++++++++++++ .../clp-excution-image/ubuntu-jammy.yml | 29 +++++++++++++++++++ docs/gh-workflows.md | 12 ++++++-- .../clp-execution-base-focal/Dockerfile | 16 ---------- .../clp-execution-base-jammy/Dockerfile | 16 ---------- .../Dockerfile | 16 ++++++++++ .../build.sh | 0 .../install-prebuilt-packages.sh | 0 .../Dockerfile | 16 ++++++++++ .../build.sh | 0 .../install-prebuilt-packages.sh | 0 12 files changed, 113 insertions(+), 49 deletions(-) rename .github/workflows/{clp-execution-image-build.yaml => clp-excution-image/build.yaml} (69%) create mode 100644 .github/workflows/clp-excution-image/ubuntu-focal.yml create mode 100644 .github/workflows/clp-excution-image/ubuntu-jammy.yml delete mode 100644 tools/docker-images/clp-execution-base-focal/Dockerfile delete mode 100644 tools/docker-images/clp-execution-base-jammy/Dockerfile create mode 100644 tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile rename tools/docker-images/{clp-execution-base-focal => clp-execution-base-ubuntu-focal}/build.sh (100%) rename tools/docker-images/{clp-execution-base-focal => clp-execution-base-ubuntu-focal}/setup-scripts/install-prebuilt-packages.sh (100%) create mode 100644 tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile rename tools/docker-images/{clp-execution-base-jammy => clp-execution-base-ubuntu-jammy}/build.sh (100%) rename tools/docker-images/{clp-execution-base-jammy => clp-execution-base-ubuntu-jammy}/setup-scripts/install-prebuilt-packages.sh (100%) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-excution-image/build.yaml similarity index 69% rename from .github/workflows/clp-execution-image-build.yaml rename to .github/workflows/clp-excution-image/build.yaml index 49dece398..6f70e6325 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-excution-image/build.yaml @@ -1,17 +1,17 @@ -name: "clp-execution-image-build" +name: "clp-execution-image/build" on: - pull_request: - paths: - - ".github/workflows/clp-execution-image-build.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-focal/**" - push: - paths: - - ".github/workflows/clp-execution-image-build.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-focal/**" - workflow_dispatch: + workflow_call: + inputs: + dockerfilePath: + required: true + type: string + platformId: + required: true + type: string + platformVersionId: + required: true + type: string env: CONTAINER_IMAGE_REGISTRY: "ghcr.io" @@ -45,13 +45,13 @@ jobs: uses: "docker/metadata-action@v5" with: images: "${{env.CONTAINER_IMAGE_REGISTRY}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-ubuntu-focal" + /clp-execution-x86-${{inputs.platformId}}-${{inputs.platformVersionId}}" - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@v5" with: context: "./" - file: "./tools/docker-images/clp-execution-base-focal/Dockerfile" + file: "./tools/docker-images/clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-excution-image/ubuntu-focal.yml b/.github/workflows/clp-excution-image/ubuntu-focal.yml new file mode 100644 index 000000000..6de8895a1 --- /dev/null +++ b/.github/workflows/clp-excution-image/ubuntu-focal.yml @@ -0,0 +1,29 @@ +name: "clp-execution-image/ubuntu-focal" + +on: + pull_request: + paths: + - ".github/workflows/clp-execution-image/build.yaml" + - ".github/workflows/clp-execution-image/ubuntu-focal.yaml" + - "components/core/tools/scripts/lib_install/*" + - "tools/docker-images/clp-execution-base-ubuntu-focal/**" + push: + paths: + - ".github/workflows/build.yaml" + - ".github/workflows/clp-execution-image/ubuntu-focal.yaml" + - "components/core/tools/scripts/lib_install/*" + - "tools/docker-images/clp-execution-base-ubuntu-focal/**" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + uses: "./.github/workflows/clp-execution-image/build.yml" + with: + dockerfilePath: './tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile' + platformId: 'ubuntu' + platformVersionId: 'focal' \ No newline at end of file diff --git a/.github/workflows/clp-excution-image/ubuntu-jammy.yml b/.github/workflows/clp-excution-image/ubuntu-jammy.yml new file mode 100644 index 000000000..dd5f394bc --- /dev/null +++ b/.github/workflows/clp-excution-image/ubuntu-jammy.yml @@ -0,0 +1,29 @@ +name: "clp-execution-image/ubuntu-jammy" + +on: + pull_request: + paths: + - ".github/workflows/clp-execution-image/build.yaml" + - ".github/workflows/clp-execution-image/ubuntu-jammy.yaml" + - "components/core/tools/scripts/lib_install/*" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" + push: + paths: + - ".github/workflows/build.yaml" + - ".github/workflows/clp-execution-image/ubuntu-jammy.yaml" + - "components/core/tools/scripts/lib_install/*" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + uses: "./.github/workflows/clp-execution-image/build.yml" + with: + dockerfilePath: './tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile' + platformId: 'ubuntu' + platformVersionId: 'jammy' \ No newline at end of file diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index e70e4bf15..6bc3741b7 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -65,10 +65,16 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) +## [clp-execution-image/build](../.github/workflows/clp-excution-image/build.yaml) -This workflow builds a container image that contains the dependencies necessary to run the CLP -package. +This reusable workflow builds a container image that contains the dependencies necessary to run the +CLP package on a specific platform. + +## [clp-execution-image/ubuntu-jammy](../.github/workflows/clp-excution-image/ubuntu-jammy.yaml) + +This workflow calls [clp-execution-image/build](../.github/workflows/clp-excution-image/build.yaml) +to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu +Jammy. ## [clp-lint](../.github/workflows/clp-lint.yaml) diff --git a/tools/docker-images/clp-execution-base-focal/Dockerfile b/tools/docker-images/clp-execution-base-focal/Dockerfile deleted file mode 100644 index 88c6e9a63..000000000 --- a/tools/docker-images/clp-execution-base-focal/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:focal AS BASE - -WORKDIR /root - -RUN mkdir -p ./tools/docker-images/clp-execution-base-focal -ADD ./tools/docker-images/clp-execution-base-focal/setup-scripts ./tools/docker-images/clp-execution-base-focal/setup-scripts - -RUN ./tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh - -# Remove cached files -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Flatten the image -FROM scratch -COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-jammy/Dockerfile b/tools/docker-images/clp-execution-base-jammy/Dockerfile deleted file mode 100644 index 95be22448..000000000 --- a/tools/docker-images/clp-execution-base-jammy/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:jammy AS BASE - -WORKDIR /root - -RUN mkdir -p ./tools/docker-images/clp-execution-base-jammy -ADD ./tools/docker-images/clp-execution-base-jammy/setup-scripts ./tools/docker-images/clp-execution-base-jammy/setup-scripts - -RUN ./tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh - -# Remove cached files -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Flatten the image -FROM scratch -COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile new file mode 100644 index 000000000..6a7cc4318 --- /dev/null +++ b/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:focal AS BASE + +WORKDIR /root + +RUN mkdir -p ./tools/docker-images/clp-execution-base-ubuntu-focal +ADD ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts + +RUN ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Flatten the image +FROM scratch +COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-focal/build.sh b/tools/docker-images/clp-execution-base-ubuntu-focal/build.sh similarity index 100% rename from tools/docker-images/clp-execution-base-focal/build.sh rename to tools/docker-images/clp-execution-base-ubuntu-focal/build.sh diff --git a/tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh similarity index 100% rename from tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh rename to tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile new file mode 100644 index 000000000..8b1a3885f --- /dev/null +++ b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:jammy AS BASE + +WORKDIR /root + +RUN mkdir -p ./tools/docker-images/clp-execution-base-ubuntu-jammy +ADD ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts + +RUN ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Flatten the image +FROM scratch +COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-jammy/build.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh similarity index 100% rename from tools/docker-images/clp-execution-base-jammy/build.sh rename to tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh diff --git a/tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh similarity index 100% rename from tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh rename to tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh From eef4cbf118370a32878ce520c3cc2e34df070109 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 15:40:48 -0500 Subject: [PATCH 02/22] Reformat Docker image build workflow YAML files. --- .github/workflows/clp-excution-image/build.yaml | 9 +++++---- .github/workflows/clp-excution-image/ubuntu-focal.yml | 6 +++--- .github/workflows/clp-excution-image/ubuntu-jammy.yml | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/clp-excution-image/build.yaml b/.github/workflows/clp-excution-image/build.yaml index 6f70e6325..c79462567 100644 --- a/.github/workflows/clp-excution-image/build.yaml +++ b/.github/workflows/clp-excution-image/build.yaml @@ -5,13 +5,13 @@ on: inputs: dockerfilePath: required: true - type: string + type: "string" platformId: required: true - type: string + type: "string" platformVersionId: required: true - type: string + type: "string" env: CONTAINER_IMAGE_REGISTRY: "ghcr.io" @@ -51,7 +51,8 @@ jobs: uses: "docker/build-push-action@v5" with: context: "./" - file: "./tools/docker-images/clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" + file: "./tools/docker-images/\ + clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-excution-image/ubuntu-focal.yml b/.github/workflows/clp-excution-image/ubuntu-focal.yml index 6de8895a1..3f138219a 100644 --- a/.github/workflows/clp-excution-image/ubuntu-focal.yml +++ b/.github/workflows/clp-excution-image/ubuntu-focal.yml @@ -24,6 +24,6 @@ jobs: build: uses: "./.github/workflows/clp-execution-image/build.yml" with: - dockerfilePath: './tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile' - platformId: 'ubuntu' - platformVersionId: 'focal' \ No newline at end of file + dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" + platformId: "ubuntu" + platformVersionId: "focal" diff --git a/.github/workflows/clp-excution-image/ubuntu-jammy.yml b/.github/workflows/clp-excution-image/ubuntu-jammy.yml index dd5f394bc..ffa6fb49a 100644 --- a/.github/workflows/clp-excution-image/ubuntu-jammy.yml +++ b/.github/workflows/clp-excution-image/ubuntu-jammy.yml @@ -24,6 +24,6 @@ jobs: build: uses: "./.github/workflows/clp-execution-image/build.yml" with: - dockerfilePath: './tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile' - platformId: 'ubuntu' - platformVersionId: 'jammy' \ No newline at end of file + dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" + platformId: "ubuntu" + platformVersionId: "jammy" From 83a29d7be65571ecfbf8384ed9c9ca4c1c614906 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 15:51:37 -0500 Subject: [PATCH 03/22] Move YAML files out from `.github/workflow` subdirectories to the directory base due to missing support. --- .github/workflows/clp-core-build.yaml | 4 ++-- .../build.yaml => clp-execution-image-build.yaml} | 0 ...l.yml => clp-execution-image-ubuntu-focal.yml} | 10 +++++----- ...y.yml => clp-execution-image-ubuntu-jammy.yml} | 10 +++++----- docs/gh-workflows.md | 15 +++++++++++---- 5 files changed, 23 insertions(+), 16 deletions(-) rename .github/workflows/{clp-excution-image/build.yaml => clp-execution-image-build.yaml} (100%) rename .github/workflows/{clp-excution-image/ubuntu-focal.yml => clp-execution-image-ubuntu-focal.yml} (68%) rename .github/workflows/{clp-excution-image/ubuntu-jammy.yml => clp-execution-image-ubuntu-jammy.yml} (68%) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 0dc0e532c..0d7d9b1e3 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -4,14 +4,14 @@ on: pull_request: paths: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-clp-execution-image-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-clp-execution-image-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" diff --git a/.github/workflows/clp-excution-image/build.yaml b/.github/workflows/clp-execution-image-build.yaml similarity index 100% rename from .github/workflows/clp-excution-image/build.yaml rename to .github/workflows/clp-execution-image-build.yaml diff --git a/.github/workflows/clp-excution-image/ubuntu-focal.yml b/.github/workflows/clp-execution-image-ubuntu-focal.yml similarity index 68% rename from .github/workflows/clp-excution-image/ubuntu-focal.yml rename to .github/workflows/clp-execution-image-ubuntu-focal.yml index 3f138219a..050a98a19 100644 --- a/.github/workflows/clp-excution-image/ubuntu-focal.yml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yml @@ -3,14 +3,14 @@ name: "clp-execution-image/ubuntu-focal" on: pull_request: paths: - - ".github/workflows/clp-execution-image/build.yaml" - - ".github/workflows/clp-execution-image/ubuntu-focal.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" push: paths: - - ".github/workflows/build.yaml" - - ".github/workflows/clp-execution-image/ubuntu-focal.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" workflow_dispatch: @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image/build.yml" + uses: "./.github/workflows/clp-execution-image-build.yml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" platformId: "ubuntu" diff --git a/.github/workflows/clp-excution-image/ubuntu-jammy.yml b/.github/workflows/clp-execution-image-ubuntu-jammy.yml similarity index 68% rename from .github/workflows/clp-excution-image/ubuntu-jammy.yml rename to .github/workflows/clp-execution-image-ubuntu-jammy.yml index ffa6fb49a..cb5f1108e 100644 --- a/.github/workflows/clp-excution-image/ubuntu-jammy.yml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yml @@ -3,14 +3,14 @@ name: "clp-execution-image/ubuntu-jammy" on: pull_request: paths: - - ".github/workflows/clp-execution-image/build.yaml" - - ".github/workflows/clp-execution-image/ubuntu-jammy.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" push: paths: - - ".github/workflows/build.yaml" - - ".github/workflows/clp-execution-image/ubuntu-jammy.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" workflow_dispatch: @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image/build.yml" + uses: "./.github/workflows/clp-execution-image-build.yml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" platformId: "ubuntu" diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index 6bc3741b7..6a04e3e98 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -65,16 +65,23 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image/build](../.github/workflows/clp-excution-image/build.yaml) +## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) This reusable workflow builds a container image that contains the dependencies necessary to run the CLP package on a specific platform. -## [clp-execution-image/ubuntu-jammy](../.github/workflows/clp-excution-image/ubuntu-jammy.yaml) +## [clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml) -This workflow calls [clp-execution-image/build](../.github/workflows/clp-excution-image/build.yaml) +This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yaml) to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu -Jammy. +22.04 (Jammy). + +## [clp-execution-image-ubuntu-focal](../.github/workflows/clp-excution-image-ubuntu-focal.yaml) + +Similar to +[clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml), this +workflow builds a container image that contains the dependencies necessary to run the CLP package on Ubuntu +20.04 (Focal). ## [clp-lint](../.github/workflows/clp-lint.yaml) From ba3fefd7ddd722f51fd69a01915ccc134deda22d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 15:55:51 -0500 Subject: [PATCH 04/22] Rename YAML files extension to `.yml` for consistency. --- .../{clp-core-build.yaml => clp-core-build.yml} | 12 ++++++------ ...mage-build.yaml => clp-execution-image-build.yml} | 0 .../workflows/clp-execution-image-ubuntu-focal.yml | 10 +++++----- .../workflows/clp-execution-image-ubuntu-jammy.yml | 10 +++++----- .../tools/scripts/lib_install/macos-12/README.md | 2 +- docs/gh-workflows.md | 6 +++--- 6 files changed, 20 insertions(+), 20 deletions(-) rename .github/workflows/{clp-core-build.yaml => clp-core-build.yml} (96%) rename .github/workflows/{clp-execution-image-build.yaml => clp-execution-image-build.yml} (100%) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yml similarity index 96% rename from .github/workflows/clp-core-build.yaml rename to .github/workflows/clp-core-build.yml index 0d7d9b1e3..f90dd08b7 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yml @@ -4,14 +4,14 @@ on: pull_request: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yaml" + - ".github/workflows/clp-core-clp-execution-image-build.yml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yaml" + - ".github/workflows/clp-core-clp-execution-image-build.yml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" @@ -57,25 +57,25 @@ jobs: filters: | centos74_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-build.yml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos7.4/**" - "components/core/tools/scripts/lib_install/centos7.4/**" ubuntu_focal_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-build.yml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-focal/**" - "components/core/tools/scripts/lib_install/ubuntu-focal/**" ubuntu_jammy_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-build.yml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" clp: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-core-build.yml" - ".gitmodules" - "components/core/cmake/**" - "components/core/CMakeLists.txt" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yml similarity index 100% rename from .github/workflows/clp-execution-image-build.yaml rename to .github/workflows/clp-execution-image-build.yml diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yml b/.github/workflows/clp-execution-image-ubuntu-focal.yml index 050a98a19..ce26e4b3e 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yml @@ -1,16 +1,16 @@ -name: "clp-execution-image/ubuntu-focal" +name: "clp-execution-image-ubuntu-focal" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yaml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" + - ".github/workflows/clp-execution-image-build.yml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" push: paths: - - ".github/workflows/clp-execution-image-build.yaml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" + - ".github/workflows/clp-execution-image-build.yml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" workflow_dispatch: diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yml b/.github/workflows/clp-execution-image-ubuntu-jammy.yml index cb5f1108e..6b0d7eb3c 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yml @@ -1,16 +1,16 @@ -name: "clp-execution-image/ubuntu-jammy" +name: "clp-execution-image-ubuntu-jammy" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yaml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" + - ".github/workflows/clp-execution-image-build.yml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" push: paths: - - ".github/workflows/clp-execution-image-build.yaml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" + - ".github/workflows/clp-execution-image-build.yml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" workflow_dispatch: diff --git a/components/core/tools/scripts/lib_install/macos-12/README.md b/components/core/tools/scripts/lib_install/macos-12/README.md index e462b04a4..f696ef83b 100644 --- a/components/core/tools/scripts/lib_install/macos-12/README.md +++ b/components/core/tools/scripts/lib_install/macos-12/README.md @@ -1,6 +1,6 @@ To install the dependencies required to build clp-core, follow the steps below. These same steps are used by our -[GitHub workflow](../../../../../../.github/workflows/clp-core-build.yaml). +[GitHub workflow](../../../../../../.github/workflows/clp-core-build.yml). # Installing dependencies diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index 6a04e3e98..d56de996d 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -3,7 +3,7 @@ The CLP repo includes several GitHub workflows for automating container image builds, artifact builds, testing, and linting. We briefly describe each workflow below. -## [clp-core-build](../.github/workflows/clp-core-build.yaml) +## [clp-core-build](../.github/workflows/clp-core-build.yml) This workflow is responsible for: @@ -65,14 +65,14 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) +## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yml) This reusable workflow builds a container image that contains the dependencies necessary to run the CLP package on a specific platform. ## [clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml) -This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yaml) +This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yml) to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu 22.04 (Jammy). From ce1a4f52ebebe4613607c9abc21cd7acb9df329d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:02:10 -0500 Subject: [PATCH 05/22] Rename GitHub workflow YAML files extension to `.yaml` for consistency. --- .../{clp-core-build.yml => clp-core-build.yaml} | 4 ++-- ...-image-build.yml => clp-execution-image-build.yaml} | 0 ...focal.yml => clp-execution-image-ubuntu-focal.yaml} | 10 +++++----- ...jammy.yml => clp-execution-image-ubuntu-jammy.yaml} | 10 +++++----- .../core/tools/scripts/lib_install/macos-12/README.md | 2 +- docs/gh-workflows.md | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) rename .github/workflows/{clp-core-build.yml => clp-core-build.yaml} (98%) rename .github/workflows/{clp-execution-image-build.yml => clp-execution-image-build.yaml} (100%) rename .github/workflows/{clp-execution-image-ubuntu-focal.yml => clp-execution-image-ubuntu-focal.yaml} (67%) rename .github/workflows/{clp-execution-image-ubuntu-jammy.yml => clp-execution-image-ubuntu-jammy.yaml} (67%) diff --git a/.github/workflows/clp-core-build.yml b/.github/workflows/clp-core-build.yaml similarity index 98% rename from .github/workflows/clp-core-build.yml rename to .github/workflows/clp-core-build.yaml index f90dd08b7..96dcac7e8 100644 --- a/.github/workflows/clp-core-build.yml +++ b/.github/workflows/clp-core-build.yaml @@ -4,14 +4,14 @@ on: pull_request: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yml" + - ".github/workflows/clp-core-clp-execution-image-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yml" + - ".github/workflows/clp-core-clp-execution-image-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" diff --git a/.github/workflows/clp-execution-image-build.yml b/.github/workflows/clp-execution-image-build.yaml similarity index 100% rename from .github/workflows/clp-execution-image-build.yml rename to .github/workflows/clp-execution-image-build.yaml diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml similarity index 67% rename from .github/workflows/clp-execution-image-ubuntu-focal.yml rename to .github/workflows/clp-execution-image-ubuntu-focal.yaml index ce26e4b3e..bb0fa9c8a 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -3,14 +3,14 @@ name: "clp-execution-image-ubuntu-focal" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" push: paths: - - ".github/workflows/clp-execution-image-build.yml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" workflow_dispatch: @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image-build.yml" + uses: "./.github/workflows/clp-execution-image-build.yaml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" platformId: "ubuntu" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml similarity index 67% rename from .github/workflows/clp-execution-image-ubuntu-jammy.yml rename to .github/workflows/clp-execution-image-ubuntu-jammy.yaml index 6b0d7eb3c..8048515e7 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -3,14 +3,14 @@ name: "clp-execution-image-ubuntu-jammy" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" push: paths: - - ".github/workflows/clp-execution-image-build.yml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yml" + - ".github/workflows/clp-execution-image-build.yaml" + - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" workflow_dispatch: @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image-build.yml" + uses: "./.github/workflows/clp-execution-image-build.yaml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" platformId: "ubuntu" diff --git a/components/core/tools/scripts/lib_install/macos-12/README.md b/components/core/tools/scripts/lib_install/macos-12/README.md index f696ef83b..e462b04a4 100644 --- a/components/core/tools/scripts/lib_install/macos-12/README.md +++ b/components/core/tools/scripts/lib_install/macos-12/README.md @@ -1,6 +1,6 @@ To install the dependencies required to build clp-core, follow the steps below. These same steps are used by our -[GitHub workflow](../../../../../../.github/workflows/clp-core-build.yml). +[GitHub workflow](../../../../../../.github/workflows/clp-core-build.yaml). # Installing dependencies diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index d56de996d..6a04e3e98 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -3,7 +3,7 @@ The CLP repo includes several GitHub workflows for automating container image builds, artifact builds, testing, and linting. We briefly describe each workflow below. -## [clp-core-build](../.github/workflows/clp-core-build.yml) +## [clp-core-build](../.github/workflows/clp-core-build.yaml) This workflow is responsible for: @@ -65,14 +65,14 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yml) +## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) This reusable workflow builds a container image that contains the dependencies necessary to run the CLP package on a specific platform. ## [clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml) -This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yml) +This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yaml) to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu 22.04 (Jammy). From 27a3c71f8ba0235fc827316c46c0370601208269 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:27:48 -0500 Subject: [PATCH 06/22] Refactor reusable CLP image build workflow to a reusable GitHub Action. --- .../clp-execution-image-build/action.yaml | 59 +++++++++++++++++++ .../workflows/clp-execution-image-build.yaml | 58 ------------------ .../clp-execution-image-ubuntu-focal.yaml | 6 +- .../clp-execution-image-ubuntu-jammy.yaml | 6 +- docs/gh-workflows.md | 8 +-- 5 files changed, 67 insertions(+), 70 deletions(-) create mode 100644 .github/actions/clp-execution-image-build/action.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml new file mode 100644 index 000000000..9294aaf42 --- /dev/null +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -0,0 +1,59 @@ +name: "clp-execution-image-build" +description: "Builds a container image that contains the dependencies necessary +to run the CLP package." + +inputs: + dockerfilePath: + description: "Path to the Dockerfile." + required: true + platformId: + description: "Platform ID of the container. e.g., ubuntu." + required: true + platformVersionId: + description: "Platform VERSION_ID / VERSION_CODENAME of the container. + e.g., jammy, focal." + required: true + +env: + CONTAINER_IMAGE_REGISTRY: "ghcr.io" + +runs: + using: "composite" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "docker/login-action@v3" + with: + registry: "${{env.CONTAINER_IMAGE_REGISTRY}}" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - id: "sanitization" + shell: "bash" + run: | + # Docker doesn't support repository names with uppercase characters, so we convert the + # name to lowercase here. + echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ + >> "$GITHUB_OUTPUT" + + - id: "meta" + uses: "docker/metadata-action@v5" + with: + images: "${{env.CONTAINER_IMAGE_REGISTRY}}/${{steps.sanitization.outputs.REPOSITORY}}\ + /clp-execution-x86-${{inputs.platformId}}-${{inputs.platformVersionId}}" + + - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@v5" + with: + context: "./" + file: "./tools/docker-images/\ + clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" + push: true + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index c79462567..000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: "clp-execution-image/build" - -on: - workflow_call: - inputs: - dockerfilePath: - required: true - type: "string" - platformId: - required: true - type: "string" - platformVersionId: - required: true - type: "string" - -env: - CONTAINER_IMAGE_REGISTRY: "ghcr.io" - -jobs: - build: - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@v3" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "docker/login-action@v3" - with: - registry: "${{env.CONTAINER_IMAGE_REGISTRY}}" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - id: "sanitization" - shell: "bash" - run: | - # Docker doesn't support repository names with uppercase characters, so we convert the - # name to lowercase here. - echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ - >> "$GITHUB_OUTPUT" - - - id: "meta" - uses: "docker/metadata-action@v5" - with: - images: "${{env.CONTAINER_IMAGE_REGISTRY}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-${{inputs.platformId}}-${{inputs.platformVersionId}}" - - - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@v5" - with: - context: "./" - file: "./tools/docker-images/\ - clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" - push: true - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml index bb0fa9c8a..453652b83 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -3,13 +3,13 @@ name: "clp-execution-image-ubuntu-focal" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yaml" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" push: paths: - - ".github/workflows/clp-execution-image-build.yaml" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image-build.yaml" + uses: ".github/actions/clp-execution-image-build/action.yaml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" platformId: "ubuntu" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml index 8048515e7..0683c6ae4 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -3,13 +3,13 @@ name: "clp-execution-image-ubuntu-jammy" on: pull_request: paths: - - ".github/workflows/clp-execution-image-build.yaml" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" push: paths: - - ".github/workflows/clp-execution-image-build.yaml" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - "components/core/tools/scripts/lib_install/*" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: "./.github/workflows/clp-execution-image-build.yaml" + uses: ".github/actions/clp-execution-image-build/action.yaml" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" platformId: "ubuntu" diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index 6a04e3e98..6ab95765e 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -65,14 +65,10 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) - -This reusable workflow builds a container image that contains the dependencies necessary to run the -CLP package on a specific platform. - ## [clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml) -This workflow calls [clp-execution-image/build](../.github/workflows/clp-execution-image-build.yaml) +This workflow calls +[clp-execution-image-build](../.github/workflows/actions/clp-execution-image-build/action.yaml) to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu 22.04 (Jammy). From 13dd15392c16839a5bdd30fbcc8ed5b8df054193 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:37:15 -0500 Subject: [PATCH 07/22] Revert accidental changes in `clp-core-build.yaml`. --- .github/workflows/clp-core-build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 96dcac7e8..0dc0e532c 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -4,14 +4,14 @@ on: pull_request: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yaml" + - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - ".github/actions/**" - - ".github/workflows/clp-core-clp-execution-image-build.yaml" + - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" @@ -57,25 +57,25 @@ jobs: filters: | centos74_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yml" + - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos7.4/**" - "components/core/tools/scripts/lib_install/centos7.4/**" ubuntu_focal_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yml" + - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-focal/**" - "components/core/tools/scripts/lib_install/ubuntu-focal/**" ubuntu_jammy_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yml" + - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" clp: - ".github/actions/**" - - ".github/workflows/clp-core-build.yml" + - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/cmake/**" - "components/core/CMakeLists.txt" From b1109b2e69fa642807a00975805f081e2bf12cd5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:38:17 -0500 Subject: [PATCH 08/22] Move env var CONTAINER_IMAGE_REGISTRY to an optional input since GitHub Actions files do not support env vars. --- .../actions/clp-execution-image-build/action.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index 9294aaf42..b4021ee6b 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -3,6 +3,10 @@ description: "Builds a container image that contains the dependencies necessary to run the CLP package." inputs: + containerImageRegistry: + description: "Container Image Registry" + required: false + default: "ghcr.io" dockerfilePath: description: "Path to the Dockerfile." required: true @@ -10,13 +14,10 @@ inputs: description: "Platform ID of the container. e.g., ubuntu." required: true platformVersionId: - description: "Platform VERSION_ID / VERSION_CODENAME of the container. + description: "Platform VERSION_ID / VERSION_CODENAME of the container. e.g., jammy, focal." required: true -env: - CONTAINER_IMAGE_REGISTRY: "ghcr.io" - runs: using: "composite" steps: @@ -30,7 +31,7 @@ runs: - uses: "docker/login-action@v3" with: - registry: "${{env.CONTAINER_IMAGE_REGISTRY}}" + registry: "${{inputs.containerImageRegistry}}" username: "${{github.actor}}" password: "${{secrets.GITHUB_TOKEN}}" @@ -45,7 +46,7 @@ runs: - id: "meta" uses: "docker/metadata-action@v5" with: - images: "${{env.CONTAINER_IMAGE_REGISTRY}}/${{steps.sanitization.outputs.REPOSITORY}}\ + images: "${{inputs.containerImageRegistry}}/${{steps.sanitization.outputs.REPOSITORY}}\ /clp-execution-x86-${{inputs.platformId}}-${{inputs.platformVersionId}}" - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" From 10fea7c6b968915de3120184f304191bbd050723 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:43:19 -0500 Subject: [PATCH 09/22] Update GitHub actions paths. --- .github/workflows/clp-execution-image-ubuntu-focal.yaml | 2 +- .github/workflows/clp-execution-image-ubuntu-jammy.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml index 453652b83..6fe3fa95f 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: ".github/actions/clp-execution-image-build/action.yaml" + uses: ".github/actions/clp-execution-image-build" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" platformId: "ubuntu" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml index 0683c6ae4..33f3acfb6 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -22,7 +22,7 @@ concurrency: jobs: build: - uses: ".github/actions/clp-execution-image-build/action.yaml" + uses: ".github/actions/clp-execution-image-build" with: dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" platformId: "ubuntu" From 8560e377181bd0ed82314943a260a74b5d32ccaf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:44:06 -0500 Subject: [PATCH 10/22] Refine GitHub workflow clp-core-build.yaml trigger paths. --- .github/workflows/clp-core-build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 0dc0e532c..cdc47591d 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -3,14 +3,16 @@ name: "clp-core-build" on: pull_request: paths: - - ".github/actions/**" + - ".github/actions/clp-core-build/action.yaml" + - ".github/actions/clp-core-build-containers/action.yaml" - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - - ".github/actions/**" + - ".github/actions/clp-core-build/action.yaml" + - ".github/actions/clp-core-build-containers/action.yaml" - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" From 4fc87f06de80f2bc7f397aab14582f628e760528 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:51:29 -0500 Subject: [PATCH 11/22] Refactor GitHub actions for image build process. --- .../clp-execution-image-build/action.yaml | 11 ----------- .../clp-execution-image-ubuntu-focal.yaml | 19 ++++++++++++++----- .../clp-execution-image-ubuntu-jammy.yaml | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index b4021ee6b..e1f2064ca 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -7,9 +7,6 @@ inputs: description: "Container Image Registry" required: false default: "ghcr.io" - dockerfilePath: - description: "Path to the Dockerfile." - required: true platformId: description: "Platform ID of the container. e.g., ubuntu." required: true @@ -21,14 +18,6 @@ inputs: runs: using: "composite" steps: - - uses: "actions/checkout@v3" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - uses: "docker/login-action@v3" with: registry: "${{inputs.containerImageRegistry}}" diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml index 6fe3fa95f..bf0731b09 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -22,8 +22,17 @@ concurrency: jobs: build: - uses: ".github/actions/clp-execution-image-build" - with: - dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile" - platformId: "ubuntu" - platformVersionId: "focal" + runs-on: ubuntu-latest + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: ".github/actions/clp-execution-image-build" + with: + platformId: "ubuntu" + platformVersionId: "focal" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml index 33f3acfb6..5006dd938 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -22,8 +22,17 @@ concurrency: jobs: build: - uses: ".github/actions/clp-execution-image-build" - with: - dockerfilePath: "./tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile" - platformId: "ubuntu" - platformVersionId: "jammy" + runs-on: ubuntu-latest + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: ".github/actions/clp-execution-image-build" + with: + platformId: "ubuntu" + platformVersionId: "jammy" From 44b2d9539725e231171f7348e794077b85f96994 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 16:53:40 -0500 Subject: [PATCH 12/22] Update github workflow file paths. --- .github/workflows/clp-execution-image-ubuntu-focal.yaml | 4 ++-- .github/workflows/clp-execution-image-ubuntu-jammy.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml index bf0731b09..133eaeadb 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -22,7 +22,7 @@ concurrency: jobs: build: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" with: @@ -32,7 +32,7 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - uses: ".github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-execution-image-build" with: platformId: "ubuntu" platformVersionId: "focal" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml index 5006dd938..9b2bddb45 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -22,7 +22,7 @@ concurrency: jobs: build: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" with: @@ -32,7 +32,7 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - uses: ".github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-execution-image-build" with: platformId: "ubuntu" platformVersionId: "jammy" From 0eef743dbe414e9cdf3c192b6f853b009d99e114 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 9 Mar 2024 17:08:44 -0500 Subject: [PATCH 13/22] Add configurable container registry credentials. --- .../actions/clp-execution-image-build/action.yaml | 14 +++++++++++--- .../clp-execution-image-ubuntu-focal.yaml | 3 +++ .../clp-execution-image-ubuntu-jammy.yaml | 5 ++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index e1f2064ca..0fac99ef4 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -4,9 +4,17 @@ to run the CLP package." inputs: containerImageRegistry: + default: "ghcr.io" description: "Container Image Registry" required: false - default: "ghcr.io" + containerImageRegistryUsername: + default: "${{github.actor}}" + description: "Container Image Registry Username" + required: false + containerImageRegistryPassword: + default: "" + description: "Container Image Registry Password" + required: false platformId: description: "Platform ID of the container. e.g., ubuntu." required: true @@ -21,8 +29,8 @@ runs: - uses: "docker/login-action@v3" with: registry: "${{inputs.containerImageRegistry}}" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" + username: "${{inputs.containerImageRegistryUsername}}" + password: "${{inputs.containerImageRegistryPassword}}" - id: "sanitization" shell: "bash" diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml index 133eaeadb..865309592 100644 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-focal.yaml @@ -34,5 +34,8 @@ jobs: - uses: "./.github/actions/clp-execution-image-build" with: + containerImageRegistry: "ghcr.io" + containerImageRegistryUsername: "${{github.actor}}" + containerImageRegistryPassword: "${{secrets.GITHUB_TOKEN}}" platformId: "ubuntu" platformVersionId: "focal" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml index 9b2bddb45..2a2cea05d 100644 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml @@ -34,5 +34,8 @@ jobs: - uses: "./.github/actions/clp-execution-image-build" with: + containerImageRegistry: "ghcr.io" + containerImageRegistryUsername: "${{github.actor}}" + containerImageRegistryPassword: "${{secrets.GITHUB_TOKEN}}" platformId: "ubuntu" - platformVersionId: "jammy" + platformVersionId: "focal" From a61d5e95c3168b96d6b4c7ce4422c2558f617e62 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 00:22:02 -0500 Subject: [PATCH 14/22] Update README.md to add the new Ubuntu Jammy execution environment. --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 234c44d8c..a25e74260 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ YScope's Compressed Log Processor (CLP) compresses your logs, and allows you to compressed logs without decompression. CLP supports both JSON logs and unstructured (i.e., free text) logs. It also supports real-time log compression within several logging libraries. CLP also includes purpose-built web interfaces for searching and viewing the compressed logs. To learn more -about it, you can read our [paper][8]. +about it, you can read our [paper][9]. # Benchmarks @@ -27,7 +27,7 @@ index-less design, so for a fair comparison, we disabled MongoDB and PostgreSQL' left them enabled, MongoDB and PostgreSQL's compression ratio would be worse. We didn't disable indexing for Elasticsearch or Splunk since these tools are fundamentally index-based (i.e., logs cannot be searched without indexes). More details about our experimental methodology can be found in -the [CLP paper][8]. +the [CLP paper][9]. # System Overview @@ -42,25 +42,25 @@ features: (intermediate representation) produced by CLP's logging libraries. - **Real-time Compression with CLP Logging Libraries**: CLP provides logging libraries for - [Python][9] and Java ([Log4j][10] and [Logback][11]). The logging libraries compress logs in + [Python][10] and Java ([Log4j][11] and [Logback][12]). The logging libraries compress logs in real-time, so only compressed logs are written to disk or transmitted over the network. The compressed logs use CLP's intermediate representation (IR) format which achieves a higher compression ratio than general purpose compressors like Zstandard. Compressing IR into archives can further double the compression ratio and enable global search, but this requires more memory usage as it needs to buffer enough logs. More details on IR versus archives can be found in this - [Uber Engineering Blog][12]. + [Uber Engineering Blog][13]. -- **[Log Viewer][13]**: the compressed IR can be viewed in a web-based log viewer. Compared to +- **[Log Viewer][14]**: the compressed IR can be viewed in a web-based log viewer. Compared to viewing the logs in an editor, CLP's log viewer supports advanced features like filtering logs based on log level verbosity (e.g., only displaying logs with log level equal or higher than ERROR). These features are possible because CLP's logging libraries parse the logs before compressing them into IR. -- **IR Analytics Libraries**: we also provide a [Python library][14] and a [Go library][15] that can +- **IR Analytics Libraries**: we also provide a [Python library][15] and a [Go library][16] that can analyze compressed IR. -- **[Log parser][16]**: CLP also includes a custom pushdown-automata-based log parser that is 3x - faster than state-of-the-art regular expression engines like [RE2][17]. The log parser is +- **[Log parser][17]**: CLP also includes a custom pushdown-automata-based log parser that is 3x + faster than state-of-the-art regular expression engines like [RE2][18]. The log parser is available as a library that can be used by other applications. # Getting Started @@ -110,7 +110,8 @@ useful for building and running CLP: | `ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-focal:main` | The dependencies necessary to build CLP core in an Ubuntu Focal x86 environment. | [↗][3] | | `ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main` | The dependencies necessary to build CLP core in an Ubuntu Jammy x86 environment. | [↗][4] | | `ghcr.io/y-scope/clp/clp-core-x86-ubuntu-focal:main` | The CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, etc.) built in an Ubuntu Focal x86 environment. | [↗][5] | -| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main` | The dependencies necessary to run the CLP package in an x86 environment. | [↗][6] | +| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main` | The dependencies necessary to run the CLP package in an Ubuntu Focal x86 environment. | [↗][6] | +| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main` | The dependencies necessary to run the CLP package in an Ubuntu Jammy x86 environment. | [↗][7] | # Next Steps @@ -124,7 +125,7 @@ Have an issue you want to fix or a feature you'd like to implement? We'd love to ## Linting Before submitting a PR, ensure you've run our linting tools and either fixed any violations or -suppressed the warning. To run our linting workflows locally, you'll need [Task][7]. Alternatively, +suppressed the warning. To run our linting workflows locally, you'll need [Task][8]. Alternatively, you can run the [clp-lint](.github/workflows/clp-lint.yaml) workflow in your fork. To perform the linting checks: @@ -145,14 +146,15 @@ task lint:fix [4]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [5]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-focal [6]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-focal -[7]: https://taskfile.dev/ -[8]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf -[9]: https://github.com/y-scope/clp-loglib-py -[10]: https://github.com/y-scope/log4j1-appenders -[11]: https://github.com/y-scope/logback-appenders -[12]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp -[13]: https://github.com/y-scope/yscope-log-viewer -[14]: https://github.com/y-scope/clp-ffi-py -[15]: https://github.com/y-scope/clp-ffi-go -[16]: https://github.com/y-scope/log-surgeon -[17]: https://github.com/google/re2 +[7]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy +[8]: https://taskfile.dev/ +[9]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf +[10]: https://github.com/y-scope/clp-loglib-py +[11]: https://github.com/y-scope/log4j1-appenders +[12]: https://github.com/y-scope/logback-appenders +[13]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp +[14]: https://github.com/y-scope/yscope-log-viewer +[15]: https://github.com/y-scope/clp-ffi-py +[16]: https://github.com/y-scope/clp-ffi-go +[17]: https://github.com/y-scope/log-surgeon +[18]: https://github.com/google/re2 From cd431d0e1776f03e6903b356412e7025b085795c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 20:24:58 -0400 Subject: [PATCH 15/22] Refactor execution image workflow and action inputs naming. --- .../clp-execution-image-build/action.yaml | 34 ++++---- .../workflows/clp-execution-image-build.yaml | 78 +++++++++++++++++++ .../clp-execution-image-ubuntu-focal.yaml | 41 ---------- .../clp-execution-image-ubuntu-jammy.yaml | 41 ---------- docs/gh-workflows.md | 15 +--- 5 files changed, 98 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-execution-image-ubuntu-focal.yaml delete mode 100644 .github/workflows/clp-execution-image-ubuntu-jammy.yaml diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index 0fac99ef4..f8ff24823 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -3,24 +3,24 @@ description: "Builds a container image that contains the dependencies necessary to run the CLP package." inputs: - containerImageRegistry: + image_registry: default: "ghcr.io" - description: "Container Image Registry" + description: "Container image registry." required: false - containerImageRegistryUsername: + image_registry_username: default: "${{github.actor}}" - description: "Container Image Registry Username" + description: "Container image registry username." required: false - containerImageRegistryPassword: + image_registry_password: default: "" - description: "Container Image Registry Password" + description: "Container image registry password." required: false - platformId: - description: "Platform ID of the container. e.g., ubuntu." + platform_id: + description: "Platform ID of the container (e.g. ubuntu)" required: true - platformVersionId: - description: "Platform VERSION_ID / VERSION_CODENAME of the container. - e.g., jammy, focal." + platform_version_id: + description: "Platform VERSION_ID / VERSION_CODENAME of the container + (e.g. jammy, focal, etc.)" required: true runs: @@ -28,9 +28,9 @@ runs: steps: - uses: "docker/login-action@v3" with: - registry: "${{inputs.containerImageRegistry}}" - username: "${{inputs.containerImageRegistryUsername}}" - password: "${{inputs.containerImageRegistryPassword}}" + registry: "${{inputs.image_registry}}" + username: "${{inputs.image_registry_username}}" + password: "${{inputs.image_registry_password}}" - id: "sanitization" shell: "bash" @@ -43,15 +43,15 @@ runs: - id: "meta" uses: "docker/metadata-action@v5" with: - images: "${{inputs.containerImageRegistry}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-${{inputs.platformId}}-${{inputs.platformVersionId}}" + images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ + /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@v5" with: context: "./" file: "./tools/docker-images/\ - clp-execution-base-${{inputs.platformId}}-${{inputs.platformVersionId}}/Dockerfile" + clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile" push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 000000000..1e5cd7034 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,78 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + runs-on: "ubuntu-latest" + outputs: + ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@v2" + id: "filter" + with: + base: "main" + filters: | + ubuntu_focal_image: + - ".github/actions/**" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-focal/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" + + build-image: + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + strategy: + matrix: + include: + - platform_version_id: "focal" + condition: "needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" + - platform_version_id: "jammy" + condition: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Build and Push Docker Image" + if: "'true' == matrix.condition" + uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "${{matrix.platform_version_id}}" diff --git a/.github/workflows/clp-execution-image-ubuntu-focal.yaml b/.github/workflows/clp-execution-image-ubuntu-focal.yaml deleted file mode 100644 index 865309592..000000000 --- a/.github/workflows/clp-execution-image-ubuntu-focal.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: "clp-execution-image-ubuntu-focal" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-ubuntu-focal/**" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-ubuntu-focal.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-ubuntu-focal/**" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@v3" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - containerImageRegistry: "ghcr.io" - containerImageRegistryUsername: "${{github.actor}}" - containerImageRegistryPassword: "${{secrets.GITHUB_TOKEN}}" - platformId: "ubuntu" - platformVersionId: "focal" diff --git a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml b/.github/workflows/clp-execution-image-ubuntu-jammy.yaml deleted file mode 100644 index 2a2cea05d..000000000 --- a/.github/workflows/clp-execution-image-ubuntu-jammy.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: "clp-execution-image-ubuntu-jammy" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-ubuntu-jammy.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@v3" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - containerImageRegistry: "ghcr.io" - containerImageRegistryUsername: "${{github.actor}}" - containerImageRegistryPassword: "${{secrets.GITHUB_TOKEN}}" - platformId: "ubuntu" - platformVersionId: "focal" diff --git a/docs/gh-workflows.md b/docs/gh-workflows.md index 6ab95765e..e70e4bf15 100644 --- a/docs/gh-workflows.md +++ b/docs/gh-workflows.md @@ -65,19 +65,10 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## [clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml) +## [clp-execution-image-build](../.github/workflows/clp-execution-image-build.yaml) -This workflow calls -[clp-execution-image-build](../.github/workflows/actions/clp-execution-image-build/action.yaml) -to build a container image that contains the dependencies necessary to run the CLP package on Ubuntu -22.04 (Jammy). - -## [clp-execution-image-ubuntu-focal](../.github/workflows/clp-excution-image-ubuntu-focal.yaml) - -Similar to -[clp-execution-image-ubuntu-jammy](../.github/workflows/clp-excution-image-ubuntu-jammy.yaml), this -workflow builds a container image that contains the dependencies necessary to run the CLP package on Ubuntu -20.04 (Focal). +This workflow builds a container image that contains the dependencies necessary to run the CLP +package. ## [clp-lint](../.github/workflows/clp-lint.yaml) From 16b8e0b89d34df39aa779139c9707d66748533fa Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 20:30:13 -0400 Subject: [PATCH 16/22] Refine paths in GitHub workflow filters. --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 1e5cd7034..f84c2cbcd 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -40,11 +40,11 @@ jobs: base: "main" filters: | ubuntu_focal_image: - - ".github/actions/**" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/clp-execution-base-ubuntu-focal/**" ubuntu_jammy_image: - - ".github/actions/**" + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" From 1a6910e4e63720ce0ef71fa750a7626dd19f1beb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 20:42:07 -0400 Subject: [PATCH 17/22] Separated ubuntu_focal and ubuntu_jammy image build in GitHub workflows because previous combined approach did not work. --- .../workflows/clp-execution-image-build.yaml | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index f84c2cbcd..86e0faffb 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -48,31 +48,42 @@ jobs: - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" - build-image: - needs: "filter-relevant-changes" + ubuntu_focal_execution_image: + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" runs-on: "ubuntu-latest" - strategy: - matrix: - include: - - platform_version_id: "focal" - condition: "needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" - - platform_version_id: "jammy" - condition: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" steps: - uses: "actions/checkout@v3" with: submodules: "recursive" - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "focal" + + ubuntu_jammy_execution_image: + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" shell: "bash" + run: "chown $(id -u):$(id -g) -R ." - - name: "Build and Push Docker Image" - if: "'true' == matrix.condition" - uses: "./.github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-execution-image-build" with: image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" platform_id: "ubuntu" - platform_version_id: "${{matrix.platform_version_id}}" + platform_version_id: "jammy" From 119920087ab4e407b4d219f553be7f2efacdfd9c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 20:44:31 -0400 Subject: [PATCH 18/22] Revert Yoda conditions. --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 86e0faffb..01d35b256 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -49,7 +49,7 @@ jobs: - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" ubuntu_focal_execution_image: - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" + if: "needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'true'" runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" @@ -69,7 +69,7 @@ jobs: platform_version_id: "focal" ubuntu_jammy_execution_image: - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" From 1108033bbd95929646c00dca0a2defb322384c0a Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 20:48:03 -0400 Subject: [PATCH 19/22] Ensure execution image builds need filter-relevant-changes. --- .github/workflows/clp-execution-image-build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 01d35b256..2ec6fefe5 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -49,7 +49,8 @@ jobs: - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" ubuntu_focal_execution_image: - if: "needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed == 'true'" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" + needs: "filter-relevant-changes" runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" @@ -69,7 +70,8 @@ jobs: platform_version_id: "focal" ubuntu_jammy_execution_image: - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v3" From 80cf0de232d6df85e63452ca3894e516f8a54501 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 21:55:46 -0400 Subject: [PATCH 20/22] Add step to "Set up Docker Buildx". --- .../clp-execution-image-build/action.yaml | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index f8ff24823..45d87dff2 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -5,15 +5,15 @@ to run the CLP package." inputs: image_registry: default: "ghcr.io" - description: "Container image registry." + description: "Container image registry" required: false image_registry_username: default: "${{github.actor}}" - description: "Container image registry username." + description: "Container image registry username" required: false image_registry_password: default: "" - description: "Container image registry password." + description: "Container image registry password" required: false platform_id: description: "Platform ID of the container (e.g. ubuntu)" @@ -26,13 +26,18 @@ inputs: runs: using: "composite" steps: - - uses: "docker/login-action@v3" + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Login to Image Registry" + uses: "docker/login-action@v3" with: registry: "${{inputs.image_registry}}" username: "${{inputs.image_registry_username}}" password: "${{inputs.image_registry_password}}" - - id: "sanitization" + - name: "Sanitize Repository Name" + id: "sanitization" shell: "bash" run: | # Docker doesn't support repository names with uppercase characters, so we convert the @@ -40,13 +45,15 @@ runs: echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ >> "$GITHUB_OUTPUT" - - id: "meta" + - name: "Update Metadata" + id: "meta" uses: "docker/metadata-action@v5" with: images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" - - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + - name: "Build and Push" + if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@v5" with: context: "./" From 31b86d944a69dd25b51db3d7fda51c8b39efa363 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 22:48:43 -0400 Subject: [PATCH 21/22] Update blob pattern in clp-execution-image-build workflow file. --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 2ec6fefe5..cc229b157 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -5,12 +5,12 @@ on: paths: - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**" + - "tools/docker-images/**/*" push: paths: - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**" + - "tools/docker-images/**/*" workflow_dispatch: concurrency: From a1aa6790fece3a3bf49c8101a41c698b17444dcf Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 10 Mar 2024 22:50:55 -0400 Subject: [PATCH 22/22] Update blob pattern in `dorny/paths-filter` in clp-execution-image-build workflow file. --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index cc229b157..c1fabea5d 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -42,11 +42,11 @@ jobs: ubuntu_focal_image: - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-focal/**" + - "tools/docker-images/clp-execution-base-ubuntu-focal/**/*" ubuntu_jammy_image: - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" ubuntu_focal_execution_image: if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed"