From ccc4608c835750fa9bd416b4c66b07a23f090ad3 Mon Sep 17 00:00:00 2001 From: austinhallett Date: Wed, 15 May 2024 15:59:11 -0400 Subject: [PATCH 1/5] remove target platform env var --- .github/workflows/build-test-deploy.yml | 106 ++++++++++++------------ 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 51a923e..dc34441 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -1,63 +1,61 @@ name: yocto-container build and test on: - push: - branches: - - master - pull_request: + push: + branches: + - master + pull_request: - # Allow for manually running - workflow_dispatch: + # Allow for manually running + workflow_dispatch: - # Run at 00:52 UTC every Saturday - schedule: - - cron: '52 0 * * 6' + # Run at 00:52 UTC every Saturday + schedule: + - cron: "52 0 * * 6" jobs: - build-test-deploy: - runs-on: ubuntu-latest - permissions: - packages: write - strategy: - fail-fast: false - matrix: - distro_to_build: [ - alma-8, - alma-9, - centos-7, - debian-10, - debian-11, - debian-12, - fedora-36, - fedora-37, - fedora-38, - fedora-39, - fedora-40, - opensuse-15.4, - opensuse-15.5, - ubuntu-18.04, - ubuntu-20.04, - ubuntu-22.04 - ] - targetplatform: [ - linux/amd64 - ] - env: - DISTRO_TO_BUILD: ${{ matrix.distro_to_build }} - TARGETPLATFORM: ${{ matrix.targetplatform }} - REPO: crops/yocto - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - GHCR_USERNAME: ${{ github.actor }} - GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + build-test-deploy: + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + fail-fast: false + matrix: + distro_to_build: + [ + alma-8, + alma-9, + centos-7, + debian-10, + debian-11, + debian-12, + fedora-36, + fedora-37, + fedora-38, + fedora-39, + fedora-40, + opensuse-15.4, + opensuse-15.5, + ubuntu-18.04, + ubuntu-20.04, + ubuntu-22.04, + ] + env: + DISTRO_TO_BUILD: ${{ matrix.distro_to_build }} + TARGETPLATFORM: ${{ matrix.targetplatform }} + REPO: crops/yocto + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 + steps: + - uses: actions/checkout@v4 - # Build and test the images - - name: Run build-and-test.sh - run: ./build_container.sh + # Build and test the images + - name: Run build-and-test.sh + run: ./build_container.sh - # Deploy the images - - name: Deploy - if: ${{ env.DOCKER_PASSWORD != '' && env.GHCR_PASSWORD != '' }} - run: ./deploy.sh + # Deploy the images + - name: Deploy + if: ${{ env.DOCKER_PASSWORD != '' && env.GHCR_PASSWORD != '' }} + run: ./deploy.sh From af81cde4d672385406919a04932f84df1b512d6b Mon Sep 17 00:00:00 2001 From: austinhallett Date: Wed, 15 May 2024 16:00:53 -0400 Subject: [PATCH 2/5] add login step --- .github/workflows/build-test-deploy.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index dc34441..4c86e9e 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -51,6 +51,13 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.env.GHCR_USERNAME }} + password: ${{ github.env.GHCR_PASSWORD }} + # Build and test the images - name: Run build-and-test.sh run: ./build_container.sh From 82fe1ba5fb70b26438e4bbc367782075e7723222 Mon Sep 17 00:00:00 2001 From: austinhallett Date: Wed, 15 May 2024 16:01:24 -0400 Subject: [PATCH 3/5] setup qemu and buildx --- .github/workflows/build-test-deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 4c86e9e..57f0123 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -58,6 +58,12 @@ jobs: username: ${{ github.env.GHCR_USERNAME }} password: ${{ github.env.GHCR_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # Build and test the images - name: Run build-and-test.sh run: ./build_container.sh From 31d3192fafb0efc2f02a33da8dfcb4c3149547a5 Mon Sep 17 00:00:00 2001 From: austinhallett Date: Wed, 15 May 2024 16:24:51 -0400 Subject: [PATCH 4/5] create set up steps --- .github/workflows/build-test-deploy.yml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 57f0123..9a1a66f 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -51,6 +51,27 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Generate tag + id: tag + run: echo "TAG=${{ matrix.distro_to_build }}-base" >> $GITHUB_OUTPUT + + - name: Get docker dir + id: dockerdir + run: echo "DOCKER_DIR='find -name ${{ steps.tag.outputs.TAG}}'" >> $GITHUB_OUTPUT + + - name: Get working directory + id: workdir + run: echo "WORK_DIR='mktemp --tmpdir -d tmp-${{ steps.tag.outputs.TAG}}.XXX'" >> $GITHUB_OUTPUT + + - name: Setup working directory + run: | + cp -r ${{ steps.dockerdir.outputs.DOCKER_DIR }} ${{ steps.workdir.outputs.WORK_DIR }} + WORK_DIR=${{ steps.workdir.outputs.WORK_DIR }}/${{ steps.tag.outputs.TAG }} + cp build-install-dumb-init.sh $workdir + cp install-buildtools.sh $WORK_DIR + cp install-buildtools-make.sh $WORK_DIR + echo "WORK_DIR=$WORK_DIR" >> $GITHUB_OUTPUT + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -64,6 +85,22 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build + uses: docker/build-push-action@v5 + with: + platforms: linux/amd64,linux/arm64 + load: true + pull: true + build-args: | + BASE_DISTRO=${{ matrix.base_distro }} + http_proxy=$http_proxy + HTTP_PROXY=$http_proxy + https_proxy=$https_proxy + HTTPS_PROXY=$https_proxy + no_proxy=$no_proxy + NO_PROXY=$no_proxy + tags: ${{ steps.tag.outputs.TAG }} + # Build and test the images - name: Run build-and-test.sh run: ./build_container.sh From 55ccb0751d493361dbeac3074b29bd65ba4e4322 Mon Sep 17 00:00:00 2001 From: austinhallett Date: Thu, 16 May 2024 08:23:10 -0400 Subject: [PATCH 5/5] translate build.sh to gh worflow --- .github/workflows/build-test-deploy.yml | 120 +++++++++++++++++++----- 1 file changed, 98 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-test-deploy.yml b/.github/workflows/build-test-deploy.yml index 9a1a66f..1791da0 100644 --- a/.github/workflows/build-test-deploy.yml +++ b/.github/workflows/build-test-deploy.yml @@ -41,7 +41,6 @@ jobs: ] env: DISTRO_TO_BUILD: ${{ matrix.distro_to_build }} - TARGETPLATFORM: ${{ matrix.targetplatform }} REPO: crops/yocto DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} @@ -51,33 +50,45 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Generate tag - id: tag + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ github.env.DOCKER_USERNAME }} + password: ${{ github.env.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.env.GHCR_USERNAME }} + password: ${{ github.env.GHCR_PASSWORD }} + + - name: Generate base tag + id: base-tag run: echo "TAG=${{ matrix.distro_to_build }}-base" >> $GITHUB_OUTPUT - name: Get docker dir id: dockerdir - run: echo "DOCKER_DIR='find -name ${{ steps.tag.outputs.TAG}}'" >> $GITHUB_OUTPUT + run: echo "DOCKER_DIR='find -name ${{ steps.base-tag.outputs.TAG}}'" >> $GITHUB_OUTPUT - name: Get working directory id: workdir - run: echo "WORK_DIR='mktemp --tmpdir -d tmp-${{ steps.tag.outputs.TAG}}.XXX'" >> $GITHUB_OUTPUT + run: echo "WORK_DIR='mktemp --tmpdir -d tmp-${{ steps.base-tag.outputs.TAG}}.XXX'" >> $GITHUB_OUTPUT - name: Setup working directory + id: setup-workdir run: | cp -r ${{ steps.dockerdir.outputs.DOCKER_DIR }} ${{ steps.workdir.outputs.WORK_DIR }} - WORK_DIR=${{ steps.workdir.outputs.WORK_DIR }}/${{ steps.tag.outputs.TAG }} - cp build-install-dumb-init.sh $workdir + WORK_DIR=${{ steps.workdir.outputs.WORK_DIR }}/${{ steps.base-tag.outputs.TAG }} + cp build-install-dumb-init.sh $WORK_DIR cp install-buildtools.sh $WORK_DIR cp install-buildtools-make.sh $WORK_DIR echo "WORK_DIR=$WORK_DIR" >> $GITHUB_OUTPUT - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.env.GHCR_USERNAME }} - password: ${{ github.env.GHCR_PASSWORD }} + - name: Get baseimage + id: baseimage + working-directory: ${{ steps.setup-workdir.outputs.WORK_DIR }} + run: echo "BASE_IMAGE=`grep FROM Dockerfile | sed -e 's/FROM //'" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -85,12 +96,17 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build + - name: Pull base image + run: docker pull ${{ steps.baseimage.outputs.BASE_IMAGE }} + + - name: Build base uses: docker/build-push-action@v5 with: platforms: linux/amd64,linux/arm64 + push: false load: true pull: true + context: ${{ steps.setup-workdir.outputs.WORK_DIR }} build-args: | BASE_DISTRO=${{ matrix.base_distro }} http_proxy=$http_proxy @@ -99,13 +115,73 @@ jobs: HTTPS_PROXY=$https_proxy no_proxy=$no_proxy NO_PROXY=$no_proxy - tags: ${{ steps.tag.outputs.TAG }} + tags: ${{ github.env.REPO }}/${{ steps.tag.outputs.TAG }} + + - name: Clean up + run: rm ${{ steps.setup-workdir.outputs.WORK_DIR }} -rf - # Build and test the images - - name: Run build-and-test.sh - run: ./build_container.sh + - name: Create builder tag + id: builder-tag + run: echo "TAG=${{ matrix.distro_to_build }}-builder" >> $GITHUB_OUTPUT - # Deploy the images - - name: Deploy - if: ${{ env.DOCKER_PASSWORD != '' && env.GHCR_PASSWORD != '' }} - run: ./deploy.sh + - name: Build builder + id: builder-workdir + run: echo "WORK_DIR='mktemp --tmpdir -d tmp-${{ steps.builder-tag.outputs.TAG}}.XXX'" >> $GITHUB_OUTPUT + + - name: Populate distro specific dockerfile + id: distro-dockerfile + run: | + workdir=${{ steps.builder-workdir.outputs.WORK_DIR }} + cp dockerfiles/templates/Dockerfile.builder $workdir/Dockerfile + cp distro-entry.sh $workdir + sed -i "s/DISTRO_TO_BUILD/$DISTRO_TO_BUILD/g" $workdir/Dockerfile + cp helpers/runbitbake.py $workdir + + - name: Replace rewitt/yocto repo + id: replace-rewitt + working-directory: ${{ steps.builder-workdir.outputs.WORK_DIR }} + run: sed -i -e "s#crops/yocto#$REPO#" Dockerfile + + - name: Build image + uses: docker/build-push-action@v5 + with: + platforms: linux/amd64,linux/arm64 + push: false + load: true + pull: true + context: ${{ steps.builder-workdir.outputs.WORK_DIR }} + build-args: | + http_proxy=$http_proxy + HTTP_PROXY=$http_proxy + https_proxy=$https_proxy + HTTPS_PROXY=$https_proxy + no_proxy=$no_proxy + NO_PROXY=$no_proxy + tags: ${{ github.env.REPO }}/${{ steps.builder-tag.outputs.TAG }} + + - name: Test base container + run: ./tests/container/vnc-test.sh ${{ github.env.REPO}}:${{ steps.base-tag.outputs.TAG}} + + - name: Test builder container + run: ./tests/container/smoke.sh ${{ github.env.REPO}}:${{ steps.builder-tag.outputs.TAG}} + + - name: Cleanup + run: rm ${{ steps.builder-workdir.outputs.WORK_DIR }} -rf + + - name: Push base image + uses: docker/build-push-action@v5 + with: + context: ${{ steps.base-workdir.outputs.WORK_DIR }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.LATEST_TAG }} + + - name: Push builder image + uses: docker/build-push-action@v5 + with: + context: ${{ steps.builder-workdir.outputs.WORK_DIR }} + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/${{ github.env.REPO }}/${{ steps.builder-tag.outputs.TAG }} + ${{ steps.builder-tag.outputs.TAG }}