HPCC-29925 GH Actions resets changes in vcpkg_overlays

Signed-off-by: Gordon Smith <[email protected]>
hpcc-systems · Jul 11, 2023 · 5ca1775 · 5ca1775
1 parent 908a215
commit 5ca1775
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 22 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -146,25 +146,38 @@ jobs:
       - name: Create Build Image
         uses: docker/build-push-action@v3
         with:
+          push: true
           builder: ${{ steps.buildx.outputs.name }}
-          file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile
-          context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg
+          file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}/Dockerfile
+          context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}
           load: true
           build-args: |
             VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}
           tags: |
-            build-${{ matrix.os }}:latest
+            build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+
+      - name: Update Docker Image (vcpkg_overlays)
+        run: |
+          mkdir -p ${{ needs.preamble.outputs.folder_build }}
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+          CONTAINER=$(docker run -d \
+            --mount ${{ needs.preamble.outputs.mount_platform }} \
+            --mount ${{ needs.preamble.outputs.mount_build }} \
+            $docker_label \
+            "tail -f /dev/null")
+          docker exec $CONTAINER /bin/bash -c "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_REMBED=ON" 
+          docker commit $CONTAINER build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+          docker rm -f $CONTAINER
 
       - name: CMake Packages
         if: ${{ !matrix.container && !matrix.ln && !matrix.documentation }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
           plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM")
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-            docker_label=build-${{ matrix.os }}:latest
+            docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
             docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF" 
             docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
             # Disabled as not currently needed ---
@@ -179,7 +192,7 @@ jobs:
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker_label=build-${{ matrix.os }}:latest
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" 
@@ -191,7 +204,7 @@ jobs:
           mkdir -p {${{needs.preamble.outputs.folder_build }},EN_US,PT_BR}
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker_label=build-${{ matrix.os }}:latest
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
@@ -213,7 +226,7 @@ jobs:
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker_label=build-${{ matrix.os }}:latest
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "${{ needs.preamble.outputs.gpg_import }} && cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
           # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" 

diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml
@@ -134,23 +134,36 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Create Build Image  
+      - name: Create Build Image
         if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         uses: docker/build-push-action@v3
         with:
           builder: ${{ steps.buildx.outputs.name }}
-          file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile
-          context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg
+          file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}/Dockerfile
+          context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}
           load: true
           build-args: |
             VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}
           tags: |
-            build-${{ matrix.os }}:latest
+            build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+
+      - name: Update Docker Image (vcpkg_overlays)
+        if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
+        run: |
+          mkdir -p ${{ needs.preamble.outputs.folder_build }}
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+          CONTAINER=$(docker run -d \
+            --mount ${{ needs.preamble.outputs.mount_platform }} \
+            --mount ${{ needs.preamble.outputs.mount_build }} \
+            $docker_label \
+            "tail -f /dev/null")
+          docker exec $CONTAINER /bin/bash -c "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_REMBED=ON" 
+          docker commit $CONTAINER build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
+          docker rm -f $CONTAINER
 
       - name: CMake Packages
         if: ${{ !matrix.container && !matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           declare -a plugins
           if [ ${{ needs.preamble.outputs.include_plugins }} == "ON" ]; then
               plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM")
@@ -160,7 +173,7 @@ jobs:
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-            docker_label=build-${{ matrix.os }}:latest
+            docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
             docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF" 
             docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
             # Disabled as not currently needed ---
@@ -171,21 +184,19 @@ jobs:
       - name: CMake Containerized Packages
         if: ${{ matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker_label=build-${{ matrix.os }}:latest
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
 
 
       - name: CMake LN Packages
         if: ${{ matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
-          docker_label=build-${{ matrix.os }}:latest
+          docker_label=build-${{ matrix.os }}:${{ steps.vars.outputs.vcpkg_sha_short }}
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF"
           docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" 
           # Disabled as not currently needed ---

diff --git a/dockerfiles/vcpkg/amazonlinux.dockerfile → dockerfiles/vcpkg/amazonlinux/Dockerfile b/dockerfiles/vcpkg/amazonlinux.dockerfile → dockerfiles/vcpkg/amazonlinux/Dockerfile
diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh
@@ -25,22 +25,28 @@ echo "DOCKER_PASSWORD: $DOCKER_PASSWORD"
 # docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
 
 function doBuild() {
-    docker build --progress plain --pull --rm -f "$SCRIPT_DIR/$1.dockerfile" \
+    docker build --progress plain -f "$SCRIPT_DIR/$1/Dockerfile" \
         -t build-$1:$GITHUB_REF \
         -t build-$1:latest \
         --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \
         --build-arg VCPKG_REF=$VCPKG_REF \
-        "$SCRIPT_DIR/." 
+        "$SCRIPT_DIR/$1/." 
+
+    mkdir -p build-$1
+
+    docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
+        "mkdir -p /hpcc-dev/HPCC-Platform/build-$1 && cp -r /hpcc-dev/vcpkg_installed /hpcc-dev/HPCC-Platform/build-$1/vcpkg_installed"
 
     docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
         "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS}"
+
     docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
         "cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel $(nproc)"
 
 # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash
 }
 
-CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON"
+CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON"
 
 doBuild centos-7
 doBuild centos-8

diff --git a/dockerfiles/vcpkg/centos-7.dockerfile → dockerfiles/vcpkg/centos-7/Dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile → dockerfiles/vcpkg/centos-7/Dockerfile
@@ -1,11 +1,12 @@
 ARG VCPKG_REF=latest
-FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF
+FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF 
 
 RUN yum install -y \
     java-11-openjdk-devel \
     python3-devel \
     wget \
     epel-release
+
 RUN yum update -y && yum install -y R-core-devel
 
 ENV Rcpp_package=Rcpp_0.12.19.tar.gz

diff --git a/dockerfiles/vcpkg/centos-8.dockerfile → dockerfiles/vcpkg/centos-8/Dockerfile b/dockerfiles/vcpkg/centos-8.dockerfile → dockerfiles/vcpkg/centos-8/Dockerfile
@@ -5,6 +5,7 @@ RUN yum remove -y java-1.* && yum install -y \
     java-11-openjdk-devel \
     python3-devel \
     epel-release
+
 RUN yum install -y \
     R-core-devel \
     R-Rcpp-devel \

diff --git a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile → dockerfiles/vcpkg/ubuntu-20.04/Dockerfile b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile → dockerfiles/vcpkg/ubuntu-20.04/Dockerfile
@@ -9,8 +9,11 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     wget \
     r-base \
     r-cran-rcpp
+
 RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package}
+
 RUN R CMD INSTALL ${RInside_package}
+
 RUN rm -f ${RInside_package}
 
 WORKDIR /hpcc-dev

diff --git a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile → dockerfiles/vcpkg/ubuntu-22.04/Dockerfile b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile → dockerfiles/vcpkg/ubuntu-22.04/Dockerfile
diff --git a/dockerfiles/vcpkg/ubuntu-22.10.dockerfile → dockerfiles/vcpkg/ubuntu-22.10/Dockerfile b/dockerfiles/vcpkg/ubuntu-22.10.dockerfile → dockerfiles/vcpkg/ubuntu-22.10/Dockerfile