From 32c782f094e18b7477866e91722dd54055859843 Mon Sep 17 00:00:00 2001 From: Ed Smith Date: Fri, 6 Aug 2021 17:00:12 +0100 Subject: [PATCH 1/4] Create a new build, test, publish workflow There are three phases to this workflow: - First we build everything, untagged, and cache the resulting layers (using mode=max to make sure we get the layers of the builder image as well as the layers of the runner image). This is a buildx step. - Second, we run the three test suites in parallel. Unfortunately, the ability to reuse yaml between the jobs is quite limited at present as far as I could determine. This consumes the layer cache produced in the previous step three times. - Third, if this is a push to master, the publish job runs, which logs into both Github Containers and DockerHub, and pushes the images to them. This re-uses the layer cache again, which means we're only pushing an image that we have actually tested in the previous steps. --- .github/workflows/docker-publish.yml | 200 +++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..9f4eec98 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,200 @@ +name: Build, test and deploy + +on: + push: + branches: [ master ] + tags: [ 'v*.*.*' ] + pull_request: + branches: [ master ] + workflow_dispatch: + ~ + +env: + DOCKERHUB_TAG: edscollabora/debos + GITHUB_TAG: ghcr.io/${{ github.repository_owner }}/debos + TEST_TAG: ${{ github.repository }} + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Repository checkout + uses: actions/checkout@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Use cache + uses: actions/cache@v2 + with: + path: /tmp/.build-cache + key: ${{ runner.os }}-docker-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-docker- + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: false + file: docker/Dockerfile + cache-from: type=local,src=/tmp/.build-cache + cache-to: type=local,dest=/tmp/.build-cache,mode=max + + unit-test: + name: Run unit tests + needs: [build] + runs-on: ubuntu-latest + steps: + - name: Repository checkout + uses: actions/checkout@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Use cache + uses: actions/cache@v2 + with: + path: /tmp/.build-cache + key: ${{ runner.os }}-docker-${{ github.sha }} + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: ${{ env.TEST_TAG }}:builder + file: docker/Dockerfile + cache-from: type=local,src=/tmp/.build-cache + load: true + target: builder + + - name: Run unit tests + run: | + docker-compose -f docker/unit-tests.test.yml \ + up --exit-code-from=sut + env: + IMAGE: ${{ env.TEST_TAG }}:builder + + recipe-test: + name: Run recipe tests + needs: [build] + runs-on: ubuntu-latest + steps: + - name: Repository checkout + uses: actions/checkout@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Use cache + uses: actions/cache@v2 + with: + path: /tmp/.build-cache + key: ${{ runner.os }}-docker-${{ github.sha }} + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: ${{ env.TEST_TAG }} + file: docker/Dockerfile + cache-from: type=local,src=/tmp/.build-cache + load: true + + - name: Run test recipes on host + run: | + docker-compose -f docker/recipes.test.yml \ + up --exit-code-from=sut + env: + IMAGE: ${{ env.TEST_TAG }} + + uml-test: + name: Run UML tests + needs: [build] + runs-on: ubuntu-latest + steps: + - name: Repository checkout + uses: actions/checkout@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Use cache + uses: actions/cache@v2 + with: + path: /tmp/.build-cache + key: ${{ runner.os }}-docker-${{ github.sha }} + + - name: Build Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: false + tags: ${{ env.TEST_TAG }} + file: docker/Dockerfile + cache-from: type=local,src=/tmp/.build-cache + load: true + + - name: Run test recipes using UML backend + run: | + docker-compose -f docker/recipes-test-uml.yml \ + up --exit-code-from=sut + env: + IMAGE: ${{ env.TEST_TAG }} + + publish: + name: Publish + needs: + - unit-test + - recipe-test + - uml-test + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v3 + with: + images: ${{ env.DOCKERHUB_TAG }},${{ env.GITHUB_TAG }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Use cache + uses: actions/cache@v2 + with: + path: /tmp/.build-cache + key: ${{ runner.os }}-docker-${{ github.sha }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + file: docker/Dockerfile + cache-from: type=local,src=/tmp/.build-cache From f09f3ec028b1d9bf9ce50998ffda9107cce04d89 Mon Sep 17 00:00:00 2001 From: Ed Smith Date: Fri, 6 Aug 2021 17:01:01 +0100 Subject: [PATCH 2/4] Specify the image name in unit tests That way, the test pipeline can pre-deploy those images to the local docker, avoiding a rebuild. This is necessary because buildx with --cache-from seems to be very difficult to use in conjunction with docker-compose. --- docker/recipes-test-uml.yml | 3 ++- docker/recipes.test.yml | 3 ++- docker/unit-tests.test.yml | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/recipes-test-uml.yml b/docker/recipes-test-uml.yml index ff28598f..d0d46c93 100644 --- a/docker/recipes-test-uml.yml +++ b/docker/recipes-test-uml.yml @@ -5,7 +5,8 @@ services: build: context: .. dockerfile: docker/Dockerfile - target: runner + image: + "${IMAGE}" volumes: - type: bind source: ../tests diff --git a/docker/recipes.test.yml b/docker/recipes.test.yml index d10cb74b..9a3b754a 100644 --- a/docker/recipes.test.yml +++ b/docker/recipes.test.yml @@ -5,7 +5,8 @@ services: build: context: .. dockerfile: docker/Dockerfile - target: runner + image: + "${IMAGE}" volumes: - type: bind source: ../tests diff --git a/docker/unit-tests.test.yml b/docker/unit-tests.test.yml index 3eaec86c..23f65dc3 100644 --- a/docker/unit-tests.test.yml +++ b/docker/unit-tests.test.yml @@ -6,5 +6,7 @@ services: context: .. dockerfile: docker/Dockerfile target: builder + image: + "${IMAGE}" working_dir: /usr/local/go/src/github.com/go-debos/debos command: go test -v ./... From 74c39526c003cd1212b08d81e4bdc8ce934b0f53 Mon Sep 17 00:00:00 2001 From: Ed Smith Date: Fri, 6 Aug 2021 17:02:14 +0100 Subject: [PATCH 3/4] Ignore the .git directory when determining docker context Otherwise, the simple act of copying the (possibly unchanged) code will always force a cache invalidation in docker. That in turn will mean that go get -t ./... runs every single build (which takes ~30 seconds). --- .dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6b8710a7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git From 41f3ec7dc00893edfab6060c7df0ebde29fea13a Mon Sep 17 00:00:00 2001 From: Ed Smith Date: Fri, 6 Aug 2021 17:03:38 +0100 Subject: [PATCH 4/4] Remove the old test workflow It's generally slower than the new pipeline, and doesn't have the option to deploy at the end. --- .github/workflows/build-test.yaml | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/build-test.yaml diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml deleted file mode 100644 index 9f6b2d6c..00000000 --- a/.github/workflows/build-test.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build Debos and run tests -on: [pull_request, push] - -jobs: - build-test: - name: Build Debos and run tests - runs-on: ubuntu-latest - steps: - - name: Repository checkout - uses: actions/checkout@v2 - - - name: Build container - run: | - docker build -f docker/Dockerfile -t godebos/debos . - - - name: Run unit tests - run: | - docker-compose -f docker/unit-tests.test.yml \ - up --build --exit-code-from=sut - - - name: Run test recipes on host - run: | - docker-compose -f docker/recipes.test.yml \ - up --build --exit-code-from=sut - - - name: Run test recipes using UML backend - run: | - docker-compose -f docker/recipes-test-uml.yml \ - up --build --exit-code-from=sut