From 8ca338eb58157e554af44523bba0b830c204019a Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Fri, 3 May 2024 14:27:38 -0600 Subject: [PATCH 1/4] Fix FPM install and pin version to known working --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5a16001..f3f2254 100755 --- a/Dockerfile +++ b/Dockerfile @@ -100,8 +100,8 @@ WORKDIR /tmp/bennu/build RUN cmake -D BUILD_GOBENNU=OFF ../ && make -j$(nproc) install \ && rm -rf /tmp/* -RUN gem install fpm -RUN pip3 install --trusted-host pypy.org --trusted-host files.pythonhosted.org -U aptly-ctl pip setuptools twine wheel +RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.15.1 +RUN python3 -m pip install --no-cache-dir --upgrade aptly-ctl pip setuptools twine wheel WORKDIR /root CMD /bin/bash From ce3f026196260c740bcfd1789c303336c101f478 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Fri, 3 May 2024 14:30:24 -0600 Subject: [PATCH 2/4] Improvements to Dockerfile - Cmake build parallel. This substantially speeds up build if system has multiple cores, e.g. when building locally. - Enable customization of base image and pip index - Disable caching of pip packages - Better apt cleanup - Remove --trusted-host args from pip installs (unsafe and not needed) --- Dockerfile | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3f2254..c3b681e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ +ARG REGISTRY_IMAGE="ubuntu:20.04" +ARG PIP_INDEX="https://pypi.org/" +ARG PIP_INDEX_URL="https://pypi.org/simple" + # create python build image -FROM ubuntu:20.04 AS pybuilder +FROM ${REGISTRY_IMAGE} AS pybuilder -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND="noninteractive" \ + PIP_DISABLE_PIP_VERSION_CHECK=1 -RUN apt update \ - && apt install -y \ +RUN apt-get update \ + && apt-get install -y \ build-essential cmake git wget python3-dev python3-pip \ libfreetype6-dev liblapack-dev libboost-dev \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* # setup ZMQ @@ -26,10 +32,10 @@ RUN wget -O helics.tgz https://github.com/GMLC-TDC/HELICS/releases/download/v${H && tar -C /tmp/helics -xzf helics.tgz \ && rm helics.tgz \ && mkdir -p /tmp/helics/build && cd /tmp/helics/build \ - && cmake -D HELICS_USE_SYSTEM_ZEROMQ_ONLY=ON .. \ + && cmake -j$(nproc) -D HELICS_USE_SYSTEM_ZEROMQ_ONLY=ON .. \ && make -j$(nproc) install -RUN python3 -m pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org pyzmq~=20.0.0 --install-option=--enable-drafts +RUN python3 -m pip install --no-cache-dir pyzmq~=20.0.0 --install-option=--enable-drafts RUN wget -O pyhelics.tgz https://github.com/GMLC-TDC/pyhelics/releases/download/v${HELICS_VERSION}/helics-${HELICS_VERSION}.tar.gz \ && mkdir -p /tmp/pyhelics \ @@ -37,25 +43,26 @@ RUN wget -O pyhelics.tgz https://github.com/GMLC-TDC/pyhelics/releases/download/ && rm pyhelics.tgz \ && cd /tmp/pyhelics/helics-${HELICS_VERSION} \ && sed -i 's/helics-apps/helics-apps~=2.7.1/' setup.py \ - && python3 -m pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org . + && python3 -m pip install --no-cache-dir . #DEBUG build #ADD docker/vendor /tmp/bennu/vendor #WORKDIR /tmp/bennu/vendor/helics-helper -#RUN python3 -m pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org . +#RUN python3 -m pip install . # install Python bennu package ADD src/pybennu /tmp/bennu/src/pybennu WORKDIR /tmp/bennu/src/pybennu -RUN python3 -m pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org . +RUN python3 -m pip install --no-cache-dir . # create final image -FROM ubuntu:20.04 +FROM ${REGISTRY_IMAGE} -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND="noninteractive" \ + PIP_DISABLE_PIP_VERSION_CHECK=1 -RUN apt update \ - && apt install -y \ +RUN apt-get update \ + && apt-get install -y \ # bennu build-essential ca-certificates cmake cmake-curses-gui \ git wget pkg-config libasio-dev libsodium-dev \ @@ -68,6 +75,7 @@ RUN apt update \ libffi-dev ruby-dev ruby-ffi \ # python python3-dev python3-pip python3-setuptools python3-wheel \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* # setup Go @@ -97,7 +105,8 @@ ADD test /tmp/bennu/test # install C++ and Golang bennu package WORKDIR /tmp/bennu/build -RUN cmake -D BUILD_GOBENNU=OFF ../ && make -j$(nproc) install \ +RUN cmake -j$(nproc) -D BUILD_GOBENNU=OFF ../ \ + && make -j$(nproc) install \ && rm -rf /tmp/* RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.15.1 From 3ecd427c271757ed184a9d2d80bf339245740636 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Fri, 3 May 2024 14:30:46 -0600 Subject: [PATCH 3/4] CI: bump versions, a bit of cleanup --- .github/workflows/docker-image.yml | 11 +++++------ .github/workflows/publish-image.yml | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 24f386e..1e4c217 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -5,12 +5,11 @@ on: branches: [ "main", "**" ] jobs: - build: - + name: Build Docker image runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Build the Docker image - run: docker build . --file Dockerfile --tag sceptre-bennu:$(date +%s) + - name: Check out the repo + uses: actions/checkout@v4 + - name: Build the Docker image + run: docker build --pull --file Dockerfile --tag sceptre-bennu:$(date +%s) . diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 2e10537..9403b85 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -13,18 +13,18 @@ env: # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: build-and-push-image: + name: Build and push Docker image runs-on: ubuntu-latest # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: contents: read packages: write - # steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -32,17 +32,16 @@ jobs: # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - name: Build and push Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - From 5efa97fe77d9c2bca28cc2e635683934a27f57f3 Mon Sep 17 00:00:00 2001 From: Christopher Goes Date: Fri, 3 May 2024 14:31:48 -0600 Subject: [PATCH 4/4] add comment about gem install referencing issue --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index c3b681e..489a2d2 100755 --- a/Dockerfile +++ b/Dockerfile @@ -109,7 +109,10 @@ RUN cmake -j$(nproc) -D BUILD_GOBENNU=OFF ../ \ && make -j$(nproc) install \ && rm -rf /tmp/* +# Gem install failures: https://github.com/jordansissel/fpm/issues/2048 +# For now, manually install dotenv 2.8.1 RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.15.1 + RUN python3 -m pip install --no-cache-dir --upgrade aptly-ctl pip setuptools twine wheel WORKDIR /root