From ace27a3f2aa9b4ca3f02b4efb10b351fafeb0c24 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 19 Nov 2024 23:14:54 +0100 Subject: [PATCH 01/13] allow to sign debian when promoting. --- buildkite/scripts/debian/promote.sh | 6 +- buildkite/scripts/debian/publish.sh | 8 +- buildkite/src/Command/MinaArtifact.dhall | 3 + buildkite/src/Constants/DebianRepo.dhall | 98 +++++++++++++++++++++++- scripts/debian/publish.sh | 30 ++++++-- scripts/debian/reversion.sh | 15 ++-- scripts/debian/verify.sh | 4 +- scripts/docker/build.sh | 1 + 8 files changed, 149 insertions(+), 16 deletions(-) diff --git a/buildkite/scripts/debian/promote.sh b/buildkite/scripts/debian/promote.sh index 491e77f562e..0e869728d9b 100755 --- a/buildkite/scripts/debian/promote.sh +++ b/buildkite/scripts/debian/promote.sh @@ -12,6 +12,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -s|--from-component) FROM_COMPONENT="$2"; shift;; -t|--to-component) TO_COMPONENT="$2"; shift;; --new-name) NEW_NAME="$2"; shift;; + --repo-key) REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -28,6 +29,7 @@ function usage() { echo " -s, --from-component The source channel in which package currently resides" echo " -t, --to-component The target channel for package (unstable, alpha, beta etc.)" echo " -c, --codename The Debian codename (bullseye, focal etc.)" + echo " --repo-key The Debian target repo key" echo "" echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly" exit 1 @@ -40,6 +42,7 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi; if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi; if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi; +if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi; # check for AWS Creds if [ -z "$AWS_ACCESS_KEY_ID" ]; then @@ -65,5 +68,6 @@ else --new-version $NEW_VERSION \ --suite $FROM_COMPONENT \ --new-suite $TO_COMPONENT \ - --new-name $NEW_NAME + --new-name $NEW_NAME \ + --sign $REPO_KEY fi \ No newline at end of file diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index 8b96a974591..bd72972fdc1 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,8 +15,14 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi + +sudo chown -R opam ~/.gnupg/ + +gpg --batch --yes --import /var/secrets/debian/key.gpg + source scripts/debian/publish.sh \ --names "${DOWNLOAD_FOLDER}/mina-*.deb" \ --release $MINA_DEB_RELEASE \ --version $MINA_DEB_VERSION \ - --codename $MINA_DEB_CODENAME + --codename $MINA_DEB_CODENAME \ + --sign $SIGN diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 0f7e43f1256..8d7b762b075 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -46,6 +46,7 @@ let MinaBuildSpec = , mode : PipelineMode.Type , tags : List PipelineTag.Type , channel : DebianChannel.Type + , debianRepo : DebianRepo.Type } , default = { prefix = "MinaArtifact" @@ -58,6 +59,7 @@ let MinaBuildSpec = , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] , channel = DebianChannel.Type.Unstable + , debianRepo = DebianRepo.Type.PackagesO1Test } } @@ -117,6 +119,7 @@ let publish_to_debian_repo = , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + , "${DebianRepo.keyIdEnv spec.debianRepo}" ] "./buildkite/scripts/debian/publish.sh" , label = diff --git a/buildkite/src/Constants/DebianRepo.dhall b/buildkite/src/Constants/DebianRepo.dhall index 98239c16858..9030478c7e4 100644 --- a/buildkite/src/Constants/DebianRepo.dhall +++ b/buildkite/src/Constants/DebianRepo.dhall @@ -1,3 +1,9 @@ +let Prelude = ../External/Prelude.dhall + +let Optional/map = Prelude.Optional.map + +let Optional/default = Prelude.Optional.default + let DebianRepo : Type = < Local | PackagesO1Test > @@ -10,4 +16,94 @@ let address = } repo -in { Type = DebianRepo, address = address } +let bucket = + \(repo : DebianRepo) + -> merge + { Local = None Text, PackagesO1Test = Some "packages.o1test.net" } + repo + +let bucket_or_default = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let bucketArg = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> "--bucket " ++ bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let keyId = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddress = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddressArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(key : Text) -> "--key-path " ++ key) + (keyAddress repo) + + in Optional/default Text "" maybeKey + +let keyArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "--sign " ++ repo) + (keyId repo) + + in Optional/default Text "" maybeKey + +let keyIdEnv = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "SIGN=" ++ repo) + (keyId repo) + + in Optional/default Text "" maybeKey + +let bucketEnv = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "BUCKET=" ++ repo) + (bucket repo) + + in Optional/default Text "" maybeKey + +in { Type = DebianRepo + , keyIdEnv = keyIdEnv + , keyAddressArg = keyAddressArg + , address = address + , bucket = bucket + , bucket_or_default = bucket_or_default + , bucketArg = bucketArg + , bucketEnv = bucketEnv + , keyId = keyId + , keyArg = keyArg + } diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 624a82d4f33..f3177c88d12 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -11,6 +11,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -r|--release) DEB_RELEASE="$2"; shift;; -v|--version) DEB_VERSION="$2"; shift;; -c|--codename) DEB_CODENAME="$2"; shift;; + -s|--sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -23,6 +24,7 @@ function usage() { echo " -r, --release The Debian release" echo " -v, --version The Debian version" echo " -c, --codename The Debian codename" + echo " -s, --sign The Debian key id used for sign" echo "" echo "Example: $0 --name mina-archive --release unstable --version 2.0.0-rc1-48efea4 --codename bullseye " exit 1 @@ -34,6 +36,12 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi; +if [[ -z "$SIGN" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign=$SIGN" +fi + BUCKET_ARG="--bucket=packages.o1test.net" S3_REGION_ARG="--s3-region=us-west-2" # utility for publishing deb repo with commons options @@ -47,20 +55,28 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ --fail-if-exists \ --lock \ --preserve-versions \ - --cache-control=max-age=120" + --cache-control=max-age=120 \ + $SIGN_ARG" + +if [[ -z "${PASSPHRASE:-}" ]]; then + GPG_OPTS="" +else + GPG_OPTS="--gpg-options=\"--batch --pinentry-mode=loopback --yes " +fi + + echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}" # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - ${DEBS3_UPLOAD} \ - --component "${DEB_RELEASE}" \ - --codename "${DEB_CODENAME}" \ - "${DEB_NAMES}" + "${DEBS3_UPLOAD}" \ + --component "${DEB_RELEASE}" \ + --codename "${DEB_CODENAME}" \ + "${GPG_OPTS}" \ + "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done -debs=() - for deb in $DEB_NAMES do # extracting name from debian package path. E.g: diff --git a/scripts/debian/reversion.sh b/scripts/debian/reversion.sh index 5fec028f490..a17e46a5719 100755 --- a/scripts/debian/reversion.sh +++ b/scripts/debian/reversion.sh @@ -3,6 +3,7 @@ set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -d|--deb) DEB="$2"; shift;; @@ -14,6 +15,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --new-version) NEW_VERSION="$2"; shift;; --suite) SUITE="$2"; shift;; --new-suite) NEW_SUITE="$2"; shift;; + --sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -30,6 +32,7 @@ function usage() { echo " --new-version The New Debian version" echo " --suite The Current Debian suite" echo " --new-suite The New Debian suite" + echo " --sign The Public Key id, which is used to sign package. Key must be stored locally" echo "" echo "Example: $0 --deb mina-archive --version 2.0.0-rc1-48efea4 --new-version 2.0.0-rc1 --codename bullseye --release unstable --new-release umt" exit 1 @@ -39,18 +42,21 @@ if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$NEW_RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$NEW_SUITE" ]]; then NEW_SUITE=$SUITE; fi; - if [[ -z "$DEB" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi; - +if [[ -z "$SIGN" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign $SIGN" +fi function rebuild_deb() { rm -f "${DEB}_${VERSION}.deb" rm -rf "${NEW_NAME}_${NEW_VERSION}" - wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb + wget https://s3.us-west-2.amazonaws.com/${BUCKET}/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}" sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" @@ -59,5 +65,4 @@ function rebuild_deb() { } rebuild_deb - -source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} --bucket ${BUCKET} ${SIGN_ARG} diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index c61d3636a7e..5364e817097 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -4,12 +4,14 @@ set -eox pipefail CHANNEL=umt-mainnet VERSION=3.0.0-f872d85 CODENAME=bullseye +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -c|--channel) CHANNEL="$2"; shift;; -v|--version) VERSION="$2"; shift;; -p|--package) PACKAGE="$2"; shift;; -m|--codename) CODENAME="$2"; shift;; + -b|--bucket) BUCKET="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -29,7 +31,7 @@ SCRIPT=' set -x \ && echo installing mina \ && apt-get update > /dev/null \ && apt-get install -y lsb-release ca-certificates > /dev/null \ - && echo "deb [trusted=yes] http://packages.o1test.net '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ + && echo "deb [trusted=yes] https://'$BUCKET' '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ && apt-get update > /dev/null \ && apt list -a '$PACKAGE' \ && apt-get install -y --allow-downgrades '$PACKAGE'='$VERSION' \ diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 34f8f62a0eb..4c400e7e396 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -47,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-profile) DEB_PROFILE="$2"; shift;; --deb-repo) INPUT_REPO="$2"; shift;; --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + --deb-repo-key) DEB_REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done From efe10036129f0b0a61d2e03fffa4f204dba22032 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 20 Nov 2024 21:58:16 +0100 Subject: [PATCH 02/13] freeze build id for debugging --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index 8257094f4b2..1e4fcfdd206 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} +PREFIX=gs://buildkite_k8s/coda/shared/0193492f-2c3f-4dde-8e38-b1c9c36ccab5 FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From 730e197aa8f8cb86cf03aba17a264ebe0be3824a Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 21 Nov 2024 23:03:10 +0100 Subject: [PATCH 03/13] WIP --- buildkite/scripts/debian/publish.sh | 13 ++++++++----- scripts/debian/publish.sh | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index bd72972fdc1..04901c7a054 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,14 +15,17 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi - -sudo chown -R opam ~/.gnupg/ - -gpg --batch --yes --import /var/secrets/debian/key.gpg +if [ -z "$SIGN" ]; then + sudo chown -R opam ~/.gnupg/ + gpg --batch --yes --import /var/secrets/debian/key.gpg + SIGN_ARG="--sign $SIGN" +else + SIGN_ARG="" +fi source scripts/debian/publish.sh \ --names "${DOWNLOAD_FOLDER}/mina-*.deb" \ --release $MINA_DEB_RELEASE \ --version $MINA_DEB_VERSION \ --codename $MINA_DEB_CODENAME \ - --sign $SIGN + $SIGN_ARG diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index f3177c88d12..f22c973a7fc 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eo pipefail +set -eox pipefail CLEAR='\033[0m' RED='\033[0;31m' From 2d7aca634df4e69082458ad793b4813cb8d6323a Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 20:05:56 +0100 Subject: [PATCH 04/13] fix usage empty debian repo key --- buildkite/src/Command/MinaArtifact.dhall | 16 +++++++++------- buildkite/src/Constants/DebianRepo.dhall | 8 +++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 8d7b762b075..321989856ee 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -114,13 +114,15 @@ let publish_to_debian_repo = Toolchain.select spec.toolchainSelectMode spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - , "${DebianRepo.keyIdEnv spec.debianRepo}" - ] + ( [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName + spec.channel}" + ] + # DebianRepo.keyIdEnvList spec.debianRepo + ) "./buildkite/scripts/debian/publish.sh" , label = "Publish Mina for ${DebianVersions.capitalName diff --git a/buildkite/src/Constants/DebianRepo.dhall b/buildkite/src/Constants/DebianRepo.dhall index 9030478c7e4..c994563ca20 100644 --- a/buildkite/src/Constants/DebianRepo.dhall +++ b/buildkite/src/Constants/DebianRepo.dhall @@ -4,6 +4,8 @@ let Optional/map = Prelude.Optional.map let Optional/default = Prelude.Optional.default +let Optional/toList = Prelude.Optional.toList + let DebianRepo : Type = < Local | PackagesO1Test > @@ -74,7 +76,7 @@ let keyArg = in Optional/default Text "" maybeKey -let keyIdEnv = +let keyIdEnvList = \(repo : DebianRepo) -> let maybeKey = Optional/map @@ -83,7 +85,7 @@ let keyIdEnv = (\(repo : Text) -> "SIGN=" ++ repo) (keyId repo) - in Optional/default Text "" maybeKey + in Optional/toList Text maybeKey let bucketEnv = \(repo : DebianRepo) @@ -97,7 +99,7 @@ let bucketEnv = in Optional/default Text "" maybeKey in { Type = DebianRepo - , keyIdEnv = keyIdEnv + , keyIdEnvList = keyIdEnvList , keyAddressArg = keyAddressArg , address = address , bucket = bucket From 6938f9ff69ff8a42300fb3a6edca509a18bce53e Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 22:52:23 +0100 Subject: [PATCH 05/13] revert SIGN arg handling --- buildkite/scripts/debian/publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index 04901c7a054..b69799b47b9 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -16,11 +16,11 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then fi if [ -z "$SIGN" ]; then + SIGN_ARG="" +else sudo chown -R opam ~/.gnupg/ gpg --batch --yes --import /var/secrets/debian/key.gpg SIGN_ARG="--sign $SIGN" -else - SIGN_ARG="" fi source scripts/debian/publish.sh \ From 9b0d2e5c779cf35036ac78391da0d636ca004b12 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 22:58:41 +0100 Subject: [PATCH 06/13] use -n --- buildkite/scripts/debian/publish.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index b69799b47b9..042e2a4f5a2 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,12 +15,12 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi -if [ -z "$SIGN" ]; then - SIGN_ARG="" -else +if [ -n "$SIGN" ]; then sudo chown -R opam ~/.gnupg/ gpg --batch --yes --import /var/secrets/debian/key.gpg SIGN_ARG="--sign $SIGN" +else + SIGN_ARG="" fi source scripts/debian/publish.sh \ From 5841cd7682af328956f00b5f9d86f5df70d8f4c4 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 23:14:37 +0100 Subject: [PATCH 07/13] fix env var handling again --- buildkite/scripts/debian/publish.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index 042e2a4f5a2..75ddad3c1da 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,12 +15,12 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi -if [ -n "$SIGN" ]; then +if [ -z "${SIGN:-}" ]; then + SIGN_ARG="" +else sudo chown -R opam ~/.gnupg/ gpg --batch --yes --import /var/secrets/debian/key.gpg SIGN_ARG="--sign $SIGN" -else - SIGN_ARG="" fi source scripts/debian/publish.sh \ From 57b5d9502f20af1fef62d916590ac80b7b3e6925 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 23:31:34 +0100 Subject: [PATCH 08/13] fix empty SIGN --- scripts/debian/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index f22c973a7fc..137d6dca43f 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -36,7 +36,7 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi; -if [[ -z "$SIGN" ]]; then +if [[ -z "${SIGN:-}" ]]; then SIGN_ARG="" else SIGN_ARG="--sign=$SIGN" From d63069578fd099bd027c3c831b2b4262b4c6409f Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 23:42:41 +0100 Subject: [PATCH 09/13] fix deb-s3 usage --- scripts/debian/publish.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 137d6dca43f..c83c05c7256 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -70,11 +70,7 @@ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${D # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - "${DEBS3_UPLOAD}" \ - --component "${DEB_RELEASE}" \ - --codename "${DEB_CODENAME}" \ - "${GPG_OPTS}" \ - "${DEB_NAMES}" + ${DEBS3_UPLOAD} "--component ${DEB_RELEASE} --codename ${DEB_CODENAME} ${GPG_OPTS} ${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done for deb in $DEB_NAMES From 4cf77034dbcd956784aebec451a6445dd0791160 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 25 Nov 2024 23:50:38 +0100 Subject: [PATCH 10/13] fix command exec once again --- scripts/debian/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index c83c05c7256..be548e97ffb 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -70,7 +70,7 @@ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${D # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - ${DEBS3_UPLOAD} "--component ${DEB_RELEASE} --codename ${DEB_CODENAME} ${GPG_OPTS} ${DEB_NAMES}" + ${DEBS3_UPLOAD} "--component ${DEB_RELEASE} --codename ${DEB_CODENAME} ${GPG_OPTS}" "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done for deb in $DEB_NAMES From b23ccb1c7ad7f2ce4093e07321ebabcc77935be9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 26 Nov 2024 18:38:01 +0100 Subject: [PATCH 11/13] fix args extends --- scripts/debian/publish.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index be548e97ffb..55098675923 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -59,9 +59,9 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ $SIGN_ARG" if [[ -z "${PASSPHRASE:-}" ]]; then - GPG_OPTS="" + GPG_OPTS=() else - GPG_OPTS="--gpg-options=\"--batch --pinentry-mode=loopback --yes " + GPG_OPTS=("--gpg-options=\"--batch" "--pinentry-mode=loopback" "--yes") fi @@ -70,7 +70,7 @@ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${D # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - ${DEBS3_UPLOAD} "--component ${DEB_RELEASE} --codename ${DEB_CODENAME} ${GPG_OPTS}" "${DEB_NAMES}" + ${DEBS3_UPLOAD} --component "${DEB_RELEASE}" --codename "${DEB_CODENAME}" "${GPG_OPTS[@]}" "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done for deb in $DEB_NAMES From a0893fa5d9171b9d44d51e4bf3cab3950bc1ba25 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 26 Nov 2024 18:47:58 +0100 Subject: [PATCH 12/13] Revert "freeze build id for debugging" This reverts commit 3c7d4497ace243b0c57da5b17c17937086bee333. --- buildkite/scripts/download-artifact-from-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/download-artifact-from-cache.sh b/buildkite/scripts/download-artifact-from-cache.sh index 1e4fcfdd206..8257094f4b2 100755 --- a/buildkite/scripts/download-artifact-from-cache.sh +++ b/buildkite/scripts/download-artifact-from-cache.sh @@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then fi DOWNLOAD_BIN=gsutil -PREFIX=gs://buildkite_k8s/coda/shared/0193492f-2c3f-4dde-8e38-b1c9c36ccab5 +PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID} FILE="$1" REMOTE_LOCATION="$2" OPTS=${3:-""} From 75ce2241aac51bc47aa7889b10e9066127c586f1 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 26 Nov 2024 18:49:29 +0100 Subject: [PATCH 13/13] revert debugging --- scripts/debian/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 55098675923..0b5c669ab40 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eox pipefail +set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m'