From 47dcfd54248565303f61e141376d05cab7c10b9c Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 17:55:54 -0600 Subject: [PATCH 01/10] chore: bump ci dependency versions --- .github/workflows/check_format.yml | 20 ++++++++++---------- .github/workflows/integration.yml | 12 ++++++------ .github/workflows/linkcheck.yml | 6 +++--- .github/workflows/packaging-tests.yml | 4 ++-- .github/workflows/scheduled.yml | 8 ++++---- .github/workflows/stale.yml | 2 +- .github/workflows/unit.yml | 4 ++-- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml index c2f5808a53e..f4bf5999c88 100644 --- a/.github/workflows/check_format.yml +++ b/.github/workflows/check_format.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: "Checkout #1" - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v4 - name: "Checkout #2 (for tools/read-version)" run: | @@ -46,10 +46,10 @@ jobs: strategy: fail-fast: false name: Check json format - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: "Checkout #1" - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v4 - name: "Test format" run: | @@ -61,17 +61,17 @@ jobs: strategy: fail-fast: false name: Check docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: "Checkout #1" - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v4 - name: "Checkout #2 (for tools/read-version)" run: | git fetch --unshallow git remote add upstream https://git.launchpad.net/cloud-init - name: "Install Python 3.10" - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10.8' - name: "Install dependencies" @@ -91,10 +91,10 @@ jobs: shell-lint: name: Shell Lint - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v4 - name: Install ShellCheck run: | @@ -106,9 +106,9 @@ jobs: shellcheck ./tools/ds-identify check-cla-signers: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3.0.0 + - uses: actions/checkout@v4 - name: Check CLA signers file run: tools/check-cla-signers diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0f3206f7901..e7788a6b998 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -21,10 +21,10 @@ env: jobs: package-build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: "Checkout" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Fetch all tags for tools/read-version fetch-depth: 0 @@ -48,7 +48,7 @@ jobs: sudo -E su $USER -c 'mk-sbuild ${{ env.RELEASE }}' sudo -E su $USER -c 'DEB_BUILD_OPTIONS=nocheck sbuild --nolog --no-run-lintian --no-run-autopkgtest --verbose --dist=${{ env.RELEASE }} --build-dir=${{ runner.temp }} cloud-init_*.dsc' - name: Archive debs as artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: 'cloud-init-${{ env.RELEASE }}-deb' path: '${{ runner.temp }}/cloud-init*.deb' @@ -56,15 +56,15 @@ jobs: integration-tests: needs: package-build - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: "Checkout" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Fetch all tags for tools/read-version fetch-depth: 0 - name: Retrieve cloud-init package - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: 'cloud-init-${{ env.RELEASE }}-deb' - name: Verify deb package diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index aeae6840a46..8a6ecb06767 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -8,15 +8,15 @@ concurrency: jobs: linkcheck: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 continue-on-error: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.failOnError == 'true') }} steps: - name: Checkout Repository - uses: actions/checkout@v3.0.0 + uses: actions/checkout@v4 - name: "Install Python 3.10" - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10.8' diff --git a/.github/workflows/packaging-tests.yml b/.github/workflows/packaging-tests.yml index 8517d862c7d..2382ca4cc5a 100644 --- a/.github/workflows/packaging-tests.yml +++ b/.github/workflows/packaging-tests.yml @@ -18,10 +18,10 @@ env: jobs: check-patches: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Fetch all branches for merging fetch-depth: 0 diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index aca4f9b7e87..c674ca116af 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies env: DEBIAN_FRONTEND: noninteractive @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Python 3.13-dev - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.13-dev check-latest: true @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt-get -qy update diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c763dafded7..bf253c5750b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v1 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 14 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index d704ae64d7b..4521138f9e8 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -28,7 +28,7 @@ jobs: check-latest: false experimental: false name: Python ${{matrix.python-version}} ${{ matrix.slug }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 continue-on-error: ${{ matrix.experimental }} steps: - name: "Checkout" @@ -37,7 +37,7 @@ jobs: # Fetch all tags for tools/read-version fetch-depth: 0 - name: Install Python ${{matrix.python-version}} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{matrix.python-version}} check-latest: ${{matrix.check-latest}} From 2e175c28a362638894fadc829b7859a085718abe Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:14:47 -0600 Subject: [PATCH 02/10] bump docs python version, use lxd action --- .github/workflows/check_format.yml | 2 +- .github/workflows/integration.yml | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_format.yml b/.github/workflows/check_format.yml index f4bf5999c88..8f2a3357709 100644 --- a/.github/workflows/check_format.yml +++ b/.github/workflows/check_format.yml @@ -73,7 +73,7 @@ jobs: - name: "Install Python 3.10" uses: actions/setup-python@v5 with: - python-version: '3.10.8' + python-version: '3.11.9' - name: "Install dependencies" run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e7788a6b998..c83b120e459 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,6 +63,10 @@ jobs: with: # Fetch all tags for tools/read-version fetch-depth: 0 + - name: Setup LXD + uses: canonical/setup-lxd@v0.1.1 + with: + channel: latest/candidate - name: Retrieve cloud-init package uses: actions/download-artifact@v4 with: @@ -74,16 +78,13 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -y update sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox wireguard - - name: Initialize LXD + - name: Initialize pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - sudo adduser $USER lxd # Jammy GH Action runners have docker installed, which edits iptables # in a way that is incompatible with lxd. # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker - sudo iptables -I DOCKER-USER -j ACCEPT - sudo lxd init --auto - name: Run integration Tests run: | - sg lxd -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci' + 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci' From 2e9f079bb2540a77d3fdbcd103cc9e863c90f3fd Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:24:01 -0600 Subject: [PATCH 03/10] don't do duplicate work --- .github/workflows/integration.yml | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c83b120e459..160520fafd3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -53,24 +53,10 @@ jobs: name: 'cloud-init-${{ env.RELEASE }}-deb' path: '${{ runner.temp }}/cloud-init*.deb' retention-days: 3 - - integration-tests: - needs: package-build - runs-on: ubuntu-24.04 - steps: - - name: "Checkout" - uses: actions/checkout@v4 - with: - # Fetch all tags for tools/read-version - fetch-depth: 0 - name: Setup LXD uses: canonical/setup-lxd@v0.1.1 with: channel: latest/candidate - - name: Retrieve cloud-init package - uses: actions/download-artifact@v4 - with: - name: 'cloud-init-${{ env.RELEASE }}-deb' - name: Verify deb package run: | ls -hal cloud-init*.deb @@ -78,13 +64,10 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get -y update sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox wireguard - - name: Initialize pycloudlib + - name: Set up Pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - # Jammy GH Action runners have docker installed, which edits iptables - # in a way that is incompatible with lxd. - # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker - name: Run integration Tests run: | - 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci' + sh -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci' From fe11f45a6b169292573dc68a72de8140890f9e26 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:32:19 -0600 Subject: [PATCH 04/10] eliminate duplicate apt calls --- .github/workflows/integration.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 160520fafd3..5fc4ee69414 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -32,12 +32,19 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \ - debhelper \ - dh-python \ fakeroot \ - python3-setuptools \ sbuild \ - ubuntu-dev-tools + ubuntu-dev-tools \ + tox \ + wireguard \ + $(\ + ./tools/read-dependencies \ + --requirements-file requirements.txt \ + --requirements-file test-requirements.txt \ + --distro ubuntu \ + --system-pkg-names 2> /dev/null \ + | tr '\n' ' ' \ + ) sudo sbuild-adduser $USER cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc # Install all build and test dependencies @@ -60,10 +67,6 @@ jobs: - name: Verify deb package run: | ls -hal cloud-init*.deb - - name: Prepare test tools - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get -y update - sudo DEBIAN_FRONTEND=noninteractive apt-get -y install tox wireguard - name: Set up Pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa From 5f16db737dde0d5b3b2a30be04d55a7a5343fd79 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:37:49 -0600 Subject: [PATCH 05/10] fix deb location --- .github/workflows/integration.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 5fc4ee69414..7a439de8784 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -47,8 +47,6 @@ jobs: ) sudo sbuild-adduser $USER cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc - # Install all build and test dependencies - ./tools/read-dependencies -r requirements.txt -r test-requirements.txt -d ubuntu -s -i - name: Build package run: | ./packages/bddeb -S -d --release ${{ env.RELEASE }} @@ -66,11 +64,11 @@ jobs: channel: latest/candidate - name: Verify deb package run: | - ls -hal cloud-init*.deb + ls -hal '${{ runner.temp }}/cloud-init*.deb' - name: Set up Pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa echo "[lxd]" > /home/$USER/.config/pycloudlib.toml - name: Run integration Tests run: | - sh -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls cloud-init*.deb)" tox -e integration-tests-ci' + sh -c 'CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init*.deb)" tox -e integration-tests-ci' From 0c1fff2eaf1e687addbed3519f47daccba935f66 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:47:12 -0600 Subject: [PATCH 06/10] inspect --- .github/workflows/integration.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7a439de8784..e545e5cec7b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,6 +28,11 @@ jobs: with: # Fetch all tags for tools/read-version fetch-depth: 0 + - name: Inspect + run: | + free -h + cat /proc/mounts + df -h - name: Prepare dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update @@ -64,7 +69,9 @@ jobs: channel: latest/candidate - name: Verify deb package run: | - ls -hal '${{ runner.temp }}/cloud-init*.deb' + ls -hal '${{ runner.temp }}' + echo '${{ runner.temp }}/cloud-init*.deb || true + ls -hal '${{ runner.temp }}/cloud-init*.deb' || true - name: Set up Pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa From 93af591e76a7199f3c454c2a6726f6ff363d84ff Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Tue, 3 Sep 2024 18:55:53 -0600 Subject: [PATCH 07/10] wip --- .github/workflows/integration.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e545e5cec7b..3069fed4177 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -67,11 +67,18 @@ jobs: uses: canonical/setup-lxd@v0.1.1 with: channel: latest/candidate + + # this is really dumb, but I'm out of time for now + # - name: Retrieve cloud-init package + # uses: actions/download-artifact@v3 + # with: + # name: 'cloud-init-${{ env.RELEASE }}-deb' + - name: Verify deb package run: | ls -hal '${{ runner.temp }}' - echo '${{ runner.temp }}/cloud-init*.deb || true - ls -hal '${{ runner.temp }}/cloud-init*.deb' || true + echo ${{ runner.temp }}/cloud-init*.deb || true + ls -hal ${{ runner.temp }}/cloud-init*.deb || true - name: Set up Pycloudlib run: | ssh-keygen -P "" -q -f ~/.ssh/id_rsa From 6aca9670c32fba69468fa41e33eef9103fd15585 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Wed, 4 Sep 2024 08:31:00 -0600 Subject: [PATCH 08/10] wip --- .github/workflows/integration.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3069fed4177..7986d546afd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,6 +33,7 @@ jobs: free -h cat /proc/mounts df -h + sudo du -sh /var/lib/schroot | true - name: Prepare dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update @@ -52,6 +53,15 @@ jobs: ) sudo sbuild-adduser $USER cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc + - name: Create tmpfs for schroot + run: | + sudo tee -a /etc/fstab< Date: Wed, 4 Sep 2024 09:00:29 -0600 Subject: [PATCH 09/10] wip --- .github/workflows/integration.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7986d546afd..835e5b9b6e4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,12 +28,6 @@ jobs: with: # Fetch all tags for tools/read-version fetch-depth: 0 - - name: Inspect - run: | - free -h - cat /proc/mounts - df -h - sudo du -sh /var/lib/schroot | true - name: Prepare dependencies run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update From a9caa4ef42022dc900a5914209d03be571403118 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 5 Sep 2024 08:45:30 -0600 Subject: [PATCH 10/10] comments --- .github/workflows/integration.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 835e5b9b6e4..6b9b9730d8c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ env: RELEASE: focal jobs: - package-build: + build-package-and-run-tests: runs-on: ubuntu-24.04 steps: - name: "Checkout" @@ -47,15 +47,6 @@ jobs: ) sudo sbuild-adduser $USER cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc - - name: Create tmpfs for schroot - run: | - sudo tee -a /etc/fstab<