From e3f44b9cd3af8801ae208532a0159e71395f2d07 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 15:53:24 -0400 Subject: [PATCH 01/22] handle docker-compose v2, free up more space Signed-off-by: Bob Callaway --- .github/workflows/tests.yaml | 2 ++ .github/workflows/validate-release.yml | 2 ++ test/e2e_test.sh | 11 ++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ef6f815888c..ed5ff54238a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -135,6 +135,8 @@ jobs: php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \ php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true + apt-get remove -y '^llvm.*' || true + apt-get remove -y '^dotnet.*' || true apt-get remove -y 'php.*' || true apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 481f49b9c56..8061ee09798 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -107,6 +107,8 @@ jobs: php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \ php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true + apt-get remove -y '^llvm.*' || true + apt-get remove -y '^dotnet.*' || true apt-get remove -y 'php.*' || true apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 3c60c76da5e..2156fdac4a7 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -16,6 +16,11 @@ set -ex +docker_compose="docker compose" +if ! ${docker_compose} version >/dev/null 2>&1; then + docker_compose="docker-compose" +fi + echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) @@ -59,10 +64,10 @@ export FULCIO_METRICS_PORT=2113 export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo - docker-compose up -d + ${docker_compose} up -d echo -n "waiting up to 60 sec for system to start" count=0 - until [ $(docker-compose ps | grep -c "(healthy)") == 3 ]; + until [ $(${docker_compose} ps | grep -c "(healthy)") == 3 ]; do if [ $count -eq 6 ]; then echo "! timeout reached" @@ -80,7 +85,7 @@ cleanup_services() { cleanup_oidc for repo in rekor fulcio; do pushd $HOME/$repo - docker-compose down + ${docker_compose} down popd done } From f5c75a272b71a8b32d3b7170bc9f2ec32f1d6874 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 16:23:19 -0400 Subject: [PATCH 02/22] set docker compose labels on network Signed-off-by: Bob Callaway --- test/e2e_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 2156fdac4a7..bc30207dda4 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -24,7 +24,7 @@ fi echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) -docker network ls | grep fulcio_default || docker network create fulcio_default +docker network ls | grep fulcio_default || docker network create fulcio_default --label "com.docker.compose.network=fulcio_default,com.docker.compose.project=fulcio" docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg cleanup_oidc() { echo "cleaning up oidc" From 9b4207f71a588e638b080cce55af89ec108eef9b Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 16:25:04 -0400 Subject: [PATCH 03/22] llvm is needed Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 8061ee09798..fc002dd34d3 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -107,7 +107,6 @@ jobs: php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \ php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true - apt-get remove -y '^llvm.*' || true apt-get remove -y '^dotnet.*' || true apt-get remove -y 'php.*' || true apt-get autoremove -y >/dev/null 2>&1 || true From 4629cbd3b9c8894f630dd1fdacdc3cbb67d45d3c Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:13:56 -0400 Subject: [PATCH 04/22] try again Signed-off-by: Bob Callaway --- .github/workflows/tests.yaml | 2 +- .github/workflows/validate-release.yml | 2 +- test/e2e_test.sh | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ed5ff54238a..32ae07d2965 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -92,6 +92,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space run: | + rm -rf /usr/local/lib/android/ rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" @@ -136,7 +137,6 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y '^llvm.*' || true - apt-get remove -y '^dotnet.*' || true apt-get remove -y 'php.*' || true apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index fc002dd34d3..dbc401cacd9 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,6 +64,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | + rm -rf /usr/local/lib/android/ rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" @@ -107,7 +108,6 @@ jobs: php-igbinary php-memcache php-memcached php-mongodb php-redis php-xdebug \ php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true - apt-get remove -y '^dotnet.*' || true apt-get remove -y 'php.*' || true apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true diff --git a/test/e2e_test.sh b/test/e2e_test.sh index bc30207dda4..bb673629d78 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -24,7 +24,7 @@ fi echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) -docker network ls | grep fulcio_default || docker network create fulcio_default --label "com.docker.compose.network=fulcio_default,com.docker.compose.project=fulcio" +docker network ls | grep fulcio_default || docker network create fulcio_default docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg cleanup_oidc() { echo "cleaning up oidc" @@ -64,6 +64,9 @@ export FULCIO_METRICS_PORT=2113 export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo + if [ "$repo" == "fulcio" ]; then + yq -y '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml | sponge docker-compose.yml + fi ${docker_compose} up -d echo -n "waiting up to 60 sec for system to start" count=0 From 0f8fd6381aee72653a2c1f0a7b42a4e726598bfc Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:23:46 -0400 Subject: [PATCH 05/22] sudo Signed-off-by: Bob Callaway --- .github/workflows/tests.yaml | 2 +- .github/workflows/validate-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 32ae07d2965..290d725d720 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -92,7 +92,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space run: | - rm -rf /usr/local/lib/android/ + sudo rm -rf /usr/local/lib/android/ rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index dbc401cacd9..dbfd5d5a14e 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,7 +64,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | - rm -rf /usr/local/lib/android/ + sudo rm -rf /usr/local/lib/android/ rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" From fa08b296b987c5e7345ca5066192693dda4d7923 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:32:06 -0400 Subject: [PATCH 06/22] try again Signed-off-by: Bob Callaway --- .github/workflows/tests.yaml | 2 +- .github/workflows/validate-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 290d725d720..5d1f9d167f2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -92,7 +92,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space run: | - sudo rm -rf /usr/local/lib/android/ + rm -rf /usr/local/.ghcup rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index dbfd5d5a14e..ce34ea94563 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,7 +64,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | - sudo rm -rf /usr/local/lib/android/ + rm -rf /usr/local/.ghcup rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" From 19cd7cd97fca4fc54255abf1dc0ee406f84ca96a Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:35:24 -0400 Subject: [PATCH 07/22] try codeql Signed-off-by: Bob Callaway --- .github/workflows/tests.yaml | 2 +- .github/workflows/validate-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5d1f9d167f2..de5ab337360 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -92,7 +92,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space run: | - rm -rf /usr/local/.ghcup + rm -rf /opt/hostedtoolcache/CodeQL rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index ce34ea94563..86edc56314e 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,7 +64,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | - rm -rf /usr/local/.ghcup + rm -rf /opt/hostedtoolcache/CodeQL rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" rm -rf "/usr/local/share/boost" From b449dd52a5d1affedf49d61eb55580535d4a3396 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:41:44 -0400 Subject: [PATCH 08/22] fix yq Signed-off-by: Bob Callaway --- test/e2e_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_test.sh b/test/e2e_test.sh index bb673629d78..85eb981e6b4 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -65,7 +65,7 @@ export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo if [ "$repo" == "fulcio" ]; then - yq -y '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml | sponge docker-compose.yml + yq -i '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml fi ${docker_compose} up -d echo -n "waiting up to 60 sec for system to start" From dbb6143d7ce55bfd5d187982e72278ee5f6f09b6 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 17:42:23 -0400 Subject: [PATCH 09/22] yq e Signed-off-by: Bob Callaway --- test/e2e_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 85eb981e6b4..da71e98a3fa 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -65,7 +65,7 @@ export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo if [ "$repo" == "fulcio" ]; then - yq -i '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml + yq -i e '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml fi ${docker_compose} up -d echo -n "waiting up to 60 sec for system to start" From c8e1142829a7d966c222a753f65401367a57893d Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 18:37:17 -0400 Subject: [PATCH 10/22] try again Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 1 + test/e2e_test.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 86edc56314e..41c8c4d34a5 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,6 +109,7 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true + sdkmanager --uninstall "platform-tools" "platforms" "buildtools" apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true - name: check disk space diff --git a/test/e2e_test.sh b/test/e2e_test.sh index da71e98a3fa..a2ec9ea6c16 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -24,7 +24,7 @@ fi echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) -docker network ls | grep fulcio_default || docker network create fulcio_default +docker network ls | grep fulcio_default || docker network create fulcio_default --label "com.docker.compose.network=fulcio_default" docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg cleanup_oidc() { echo "cleaning up oidc" From b8491567bd81f4f10a3dcc95188a985cdb03c06e Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 18:40:58 -0400 Subject: [PATCH 11/22] use full path to sdkmanager Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 41c8c4d34a5..eaa6b0f043f 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,7 +109,7 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - sdkmanager --uninstall "platform-tools" "platforms" "buildtools" + /usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --uninstall "platform-tools" "platforms" "buildtools" apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true - name: check disk space From 175fa5a4edbf6f8c94446789196fc45c6a2a3d74 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 18:53:21 -0400 Subject: [PATCH 12/22] explore filepath Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index eaa6b0f043f..e4c09d28063 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,6 +109,7 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true + find /usr/local/lib/android/ /usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --uninstall "platform-tools" "platforms" "buildtools" apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true From b0bf9f19810a2173e47665c872d0cb8664b6fea5 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 18:57:10 -0400 Subject: [PATCH 13/22] use env var Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index e4c09d28063..5f603a229ad 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,7 +109,7 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - find /usr/local/lib/android/ + find $ANDROID_HOME /usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --uninstall "platform-tools" "platforms" "buildtools" apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true From 94f96e84207270a7db946a36acc06481ccbf1160 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 19:01:12 -0400 Subject: [PATCH 14/22] try sdk root Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 5f603a229ad..d1c9c490032 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,7 +109,7 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - find $ANDROID_HOME + find $ANDROID_SDK_ROOT /usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --uninstall "platform-tools" "platforms" "buildtools" apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true From 74a0704839ca264522a849445eb037a8fbfc8552 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 19:11:10 -0400 Subject: [PATCH 15/22] find offenders Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index d1c9c490032..b826a3b9eec 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,12 +109,11 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - find $ANDROID_SDK_ROOT - /usr/local/lib/android/sdk/cmdline-tools/latest/bin/sdkmanager --uninstall "platform-tools" "platforms" "buildtools" + cd / && du -sch * .[!.]* | sort -rh apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true - name: check disk space - run: df -h + run: df -hZZ - name: goreleaser snapshot run: make snapshot From 6f300bcf819ff66250225b0e9e1a4ff8c35bd20e Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 19:15:29 -0400 Subject: [PATCH 16/22] zoom Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index b826a3b9eec..087df045700 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,11 +109,11 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - cd / && du -sch * .[!.]* | sort -rh + cd /usr && du -sch * .[!.]* | sort -rh apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true - name: check disk space - run: df -hZZ + run: df -h - name: goreleaser snapshot run: make snapshot From 3709195fb6bd1d171a01a0e6457134289493e283 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 20:11:42 -0400 Subject: [PATCH 17/22] more investigation Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 3 ++- test/e2e_test.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 087df045700..aef3fcc63f0 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -109,7 +109,8 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - cd /usr && du -sch * .[!.]* | sort -rh + cd /usr/lib && du -sch * .[!.]* | sort -rh + cd /usr/lib/local && du -sch * .[!.]* | sort -rh apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true - name: check disk space diff --git a/test/e2e_test.sh b/test/e2e_test.sh index a2ec9ea6c16..3c75ace5e74 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -24,7 +24,7 @@ fi echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) -docker network ls | grep fulcio_default || docker network create fulcio_default --label "com.docker.compose.network=fulcio_default" +docker network ls | grep fulcio_default || docker network create fulcio_default docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg cleanup_oidc() { echo "cleaning up oidc" @@ -66,6 +66,7 @@ for repo in rekor fulcio; do pushd $repo if [ "$repo" == "fulcio" ]; then yq -i e '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml + tail docker-compose.yml fi ${docker_compose} up -d echo -n "waiting up to 60 sec for system to start" From 483862c9e6798903ac105c71dbd5923ba0b660fd Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 4 Aug 2024 20:27:24 -0400 Subject: [PATCH 18/22] attempt Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 1 + test/e2e_test.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index aef3fcc63f0..41bfaac3724 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,6 +64,7 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | + rm -rf /usr/lib/google-cloud-sdk rm -rf /opt/hostedtoolcache/CodeQL rm -rf /usr/share/dotnet/ rm -rf "$AGENT_TOOLSDIRECTORY" diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 3c75ace5e74..6d795ac2318 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -66,6 +66,7 @@ for repo in rekor fulcio; do pushd $repo if [ "$repo" == "fulcio" ]; then yq -i e '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml + yq -i e '.services.fulcio-server.networks=["fulcio_default"]' docker-compose.yml tail docker-compose.yml fi ${docker_compose} up -d From 080f2d8db52fe2e4ff2390b4f5bc19e4300bb28a Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Mon, 5 Aug 2024 20:33:22 -0400 Subject: [PATCH 19/22] try again Signed-off-by: Bob Callaway --- test/e2e_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 6d795ac2318..c9fafcec575 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -24,7 +24,7 @@ fi echo "setting up OIDC provider" pushd ./test/fakeoidc oidcimg=$(ko build main.go --local) -docker network ls | grep fulcio_default || docker network create fulcio_default +docker network ls | grep fulcio_default || docker network create fulcio_default --label "com.docker.compose.network=fulcio_default" docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg cleanup_oidc() { echo "cleaning up oidc" @@ -65,8 +65,8 @@ export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo if [ "$repo" == "fulcio" ]; then - yq -i e '.networks={"fulcio_default":{ "name":"fulcio_default","external":true }}' docker-compose.yml - yq -i e '.services.fulcio-server.networks=["fulcio_default"]' docker-compose.yml + yq -i e '.networks={"default":{ "name":"fulcio_default","external":true }}' docker-compose.yml + yq -i e '.services.fulcio-server.networks=["default"]' docker-compose.yml tail docker-compose.yml fi ${docker_compose} up -d From f8b76508a934e115705e3fdfdd7f575879b91133 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Mon, 5 Aug 2024 21:06:55 -0400 Subject: [PATCH 20/22] try deleting more Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 41bfaac3724..16cbff32abe 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -64,6 +64,18 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | + sudo rm -rf /usr/share/dotnet || true + sudo rm -rf /usr/share/swift || true + sudo rm -rf /usr/local/lib/android || true + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/local/.ghcup || true + sudo rm -rf "/usr/local/share/boost" || true + sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true + sudo rm -rf /opt/hostedtoolcache/ || true + sudo rm -rf /usr/local/graalvm/ || true + sudo rm -rf /usr/local/share/powershell || true + sudo rm -rf /usr/local/share/chromium || true + sudo rm -rf /usr/local/lib/node_modules || true rm -rf /usr/lib/google-cloud-sdk rm -rf /opt/hostedtoolcache/CodeQL rm -rf /usr/share/dotnet/ @@ -110,10 +122,9 @@ jobs: php-zmq snmp pollinate libpq-dev postgresql-client powershell ruby-full \ sphinxsearch subversion mongodb-org -yq >/dev/null 2>&1 || true apt-get remove -y 'php.*' || true - cd /usr/lib && du -sch * .[!.]* | sort -rh - cd /usr/lib/local && du -sch * .[!.]* | sort -rh apt-get autoremove -y >/dev/null 2>&1 || true apt-get autoclean -y >/dev/null 2>&1 || true + apt-get clean -y >/dev/null 2>&1 || true - name: check disk space run: df -h From ae0418641cf4f5a5600aaa0921f2b8457b392492 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Mon, 5 Aug 2024 21:44:29 -0400 Subject: [PATCH 21/22] try deleting from host mount Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 36 ++++++++++++-------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 16cbff32abe..cd6bf66cb55 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -44,6 +44,8 @@ jobs: container: image: ghcr.io/gythialy/golang-cross:v1.22.5-0@sha256:5cf8fca7fe80392c8d1597fe89d291d49120507390f25507746f73d4b7f8a8f2 + volumes: + - /:/host permissions: {} @@ -64,27 +66,23 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | - sudo rm -rf /usr/share/dotnet || true - sudo rm -rf /usr/share/swift || true - sudo rm -rf /usr/local/lib/android || true - sudo rm -rf /opt/ghc || true - sudo rm -rf /usr/local/.ghcup || true - sudo rm -rf "/usr/local/share/boost" || true - sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true - sudo rm -rf /opt/hostedtoolcache/ || true - sudo rm -rf /usr/local/graalvm/ || true - sudo rm -rf /usr/local/share/powershell || true - sudo rm -rf /usr/local/share/chromium || true - sudo rm -rf /usr/local/lib/node_modules || true - rm -rf /usr/lib/google-cloud-sdk - rm -rf /opt/hostedtoolcache/CodeQL - rm -rf /usr/share/dotnet/ - rm -rf "$AGENT_TOOLSDIRECTORY" - rm -rf "/usr/local/share/boost" - rm -rf /opt/ghc + rm -rf /host/usr/share/dotnet || true + rm -rf /host/usr/share/swift || true + rm -rf /host/usr/local/lib/android || true + rm -rf /host/opt/ghc || true + rm -rf /host/usr/local/.ghcup || true + rm -rf /host/opt/hostedtoolcache/ || true + rm -rf /host/usr/local/graalvm/ || true + rm -rf /host/usr/local/share/powershell || true + rm -rf /host/usr/local/share/chromium || true + rm -rf /host/usr/local/lib/node_modules || true + rm -rf /host/usr/lib/google-cloud-sdk + rm -rf /host/opt/hostedtoolcache/CodeQL + rm -rf "/host/$AGENT_TOOLSDIRECTORY" + rm -rf "/host/usr/local/share/boost" docker rmi $(docker image ls -aq) || true swapoff /swapfile || true - rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc || true + rm -rf /host/swapfile || true apt purge aria2 ansible hhvm mono-devel azure-cli shellcheck rpm xorriso zsync \ clang-6.0 lldb-6.0 lld-6.0 clang-format-6.0 clang-8 lldb-8 lld-8 clang-format-8 \ clang-9 lldb-9 lld-9 clangd-9 clang-format-9 dotnet-sdk-3.0 dotnet-sdk-3.1=3.1.101-1 \ From 5f285685159fda70f320b3b6df049f29bfeddee0 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Mon, 5 Aug 2024 21:49:25 -0400 Subject: [PATCH 22/22] scope host mounts Signed-off-by: Bob Callaway --- .github/workflows/validate-release.yml | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index cd6bf66cb55..653b55dbfcc 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -45,7 +45,8 @@ jobs: container: image: ghcr.io/gythialy/golang-cross:v1.22.5-0@sha256:5cf8fca7fe80392c8d1597fe89d291d49120507390f25507746f73d4b7f8a8f2 volumes: - - /:/host + - /usr:/host_usr + - /opt:/host_opt permissions: {} @@ -66,23 +67,22 @@ jobs: # Related to https://github.com/sigstore/cosign/issues/3149 - name: free up disk space for the release run: | - rm -rf /host/usr/share/dotnet || true - rm -rf /host/usr/share/swift || true - rm -rf /host/usr/local/lib/android || true - rm -rf /host/opt/ghc || true - rm -rf /host/usr/local/.ghcup || true - rm -rf /host/opt/hostedtoolcache/ || true - rm -rf /host/usr/local/graalvm/ || true - rm -rf /host/usr/local/share/powershell || true - rm -rf /host/usr/local/share/chromium || true - rm -rf /host/usr/local/lib/node_modules || true - rm -rf /host/usr/lib/google-cloud-sdk - rm -rf /host/opt/hostedtoolcache/CodeQL - rm -rf "/host/$AGENT_TOOLSDIRECTORY" - rm -rf "/host/usr/local/share/boost" - docker rmi $(docker image ls -aq) || true - swapoff /swapfile || true - rm -rf /host/swapfile || true + rm -rf /host_usr/share/dotnet || true + rm -rf /host_usr/share/swift || true + rm -rf /host_usr/local/lib/android || true + rm -rf /host_opt/ghc || true + rm -rf /host_usr/local/.ghcup || true + rm -rf /host_opt/hostedtoolcache/ || true + rm -rf /host_usr/local/graalvm/ || true + rm -rf /host_usr/local/share/powershell || true + rm -rf /host_usr/local/share/chromium || true + rm -rf /host_usr/local/lib/node_modules || true + rm -rf /host_usr/lib/google-cloud-sdk || true + rm -rf /host_opt/hostedtoolcache/CodeQL || true + rm -rf "/host_usr/local/share/boost" || true + # docker rmi $(docker image ls -aq) || true + # swapoff /swapfile || true + # rm -rf /host/swapfile || true apt purge aria2 ansible hhvm mono-devel azure-cli shellcheck rpm xorriso zsync \ clang-6.0 lldb-6.0 lld-6.0 clang-format-6.0 clang-8 lldb-8 lld-8 clang-format-8 \ clang-9 lldb-9 lld-9 clangd-9 clang-format-9 dotnet-sdk-3.0 dotnet-sdk-3.1=3.1.101-1 \