diff --git a/buildkite/scripts/debian/promote.sh b/buildkite/scripts/debian/promote.sh index 491e77f562e..0e869728d9b 100755 --- a/buildkite/scripts/debian/promote.sh +++ b/buildkite/scripts/debian/promote.sh @@ -12,6 +12,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -s|--from-component) FROM_COMPONENT="$2"; shift;; -t|--to-component) TO_COMPONENT="$2"; shift;; --new-name) NEW_NAME="$2"; shift;; + --repo-key) REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -28,6 +29,7 @@ function usage() { echo " -s, --from-component The source channel in which package currently resides" echo " -t, --to-component The target channel for package (unstable, alpha, beta etc.)" echo " -c, --codename The Debian codename (bullseye, focal etc.)" + echo " --repo-key The Debian target repo key" echo "" echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly" exit 1 @@ -40,6 +42,7 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi; if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi; if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi; +if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi; # check for AWS Creds if [ -z "$AWS_ACCESS_KEY_ID" ]; then @@ -65,5 +68,6 @@ else --new-version $NEW_VERSION \ --suite $FROM_COMPONENT \ --new-suite $TO_COMPONENT \ - --new-name $NEW_NAME + --new-name $NEW_NAME \ + --sign $REPO_KEY fi \ No newline at end of file diff --git a/buildkite/scripts/debian/publish.sh b/buildkite/scripts/debian/publish.sh index 8b96a974591..75ddad3c1da 100755 --- a/buildkite/scripts/debian/publish.sh +++ b/buildkite/scripts/debian/publish.sh @@ -15,8 +15,17 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then exit 0 fi +if [ -z "${SIGN:-}" ]; then + SIGN_ARG="" +else + sudo chown -R opam ~/.gnupg/ + gpg --batch --yes --import /var/secrets/debian/key.gpg + SIGN_ARG="--sign $SIGN" +fi + source scripts/debian/publish.sh \ --names "${DOWNLOAD_FOLDER}/mina-*.deb" \ --release $MINA_DEB_RELEASE \ --version $MINA_DEB_VERSION \ - --codename $MINA_DEB_CODENAME + --codename $MINA_DEB_CODENAME \ + $SIGN_ARG diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index e98fdca16a8..7f582d8b0b7 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -46,6 +46,7 @@ let MinaBuildSpec = , mode : PipelineMode.Type , tags : List PipelineTag.Type , channel : DebianChannel.Type + , debianRepo : DebianRepo.Type } , default = { prefix = "MinaArtifact" @@ -58,6 +59,7 @@ let MinaBuildSpec = , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] , channel = DebianChannel.Type.Unstable + , debianRepo = DebianRepo.Type.PackagesO1Test } } @@ -132,12 +134,15 @@ let publish_to_debian_repo = Toolchain.select spec.toolchainSelectMode spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] + ( [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName + spec.channel}" + ] + # DebianRepo.keyIdEnvList spec.debianRepo + ) "./buildkite/scripts/debian/publish.sh" , label = "Publish Mina for ${DebianVersions.capitalName diff --git a/buildkite/src/Constants/DebianRepo.dhall b/buildkite/src/Constants/DebianRepo.dhall index 98239c16858..c994563ca20 100644 --- a/buildkite/src/Constants/DebianRepo.dhall +++ b/buildkite/src/Constants/DebianRepo.dhall @@ -1,3 +1,11 @@ +let Prelude = ../External/Prelude.dhall + +let Optional/map = Prelude.Optional.map + +let Optional/default = Prelude.Optional.default + +let Optional/toList = Prelude.Optional.toList + let DebianRepo : Type = < Local | PackagesO1Test > @@ -10,4 +18,94 @@ let address = } repo -in { Type = DebianRepo, address = address } +let bucket = + \(repo : DebianRepo) + -> merge + { Local = None Text, PackagesO1Test = Some "packages.o1test.net" } + repo + +let bucket_or_default = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let bucketArg = + \(repo : DebianRepo) + -> let maybeBucket = + Optional/map + Text + Text + (\(bucket : Text) -> "--bucket " ++ bucket) + (bucket repo) + + in Optional/default Text "" maybeBucket + +let keyId = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddress = + \(repo : DebianRepo) + -> merge { Local = None Text, PackagesO1Test = None Text } repo + +let keyAddressArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(key : Text) -> "--key-path " ++ key) + (keyAddress repo) + + in Optional/default Text "" maybeKey + +let keyArg = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "--sign " ++ repo) + (keyId repo) + + in Optional/default Text "" maybeKey + +let keyIdEnvList = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "SIGN=" ++ repo) + (keyId repo) + + in Optional/toList Text maybeKey + +let bucketEnv = + \(repo : DebianRepo) + -> let maybeKey = + Optional/map + Text + Text + (\(repo : Text) -> "BUCKET=" ++ repo) + (bucket repo) + + in Optional/default Text "" maybeKey + +in { Type = DebianRepo + , keyIdEnvList = keyIdEnvList + , keyAddressArg = keyAddressArg + , address = address + , bucket = bucket + , bucket_or_default = bucket_or_default + , bucketArg = bucketArg + , bucketEnv = bucketEnv + , keyId = keyId + , keyArg = keyArg + } diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index 624a82d4f33..0b5c669ab40 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -11,6 +11,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -r|--release) DEB_RELEASE="$2"; shift;; -v|--version) DEB_VERSION="$2"; shift;; -c|--codename) DEB_CODENAME="$2"; shift;; + -s|--sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -23,6 +24,7 @@ function usage() { echo " -r, --release The Debian release" echo " -v, --version The Debian version" echo " -c, --codename The Debian codename" + echo " -s, --sign The Debian key id used for sign" echo "" echo "Example: $0 --name mina-archive --release unstable --version 2.0.0-rc1-48efea4 --codename bullseye " exit 1 @@ -34,6 +36,12 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi; if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi; +if [[ -z "${SIGN:-}" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign=$SIGN" +fi + BUCKET_ARG="--bucket=packages.o1test.net" S3_REGION_ARG="--s3-region=us-west-2" # utility for publishing deb repo with commons options @@ -47,20 +55,24 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ --fail-if-exists \ --lock \ --preserve-versions \ - --cache-control=max-age=120" + --cache-control=max-age=120 \ + $SIGN_ARG" + +if [[ -z "${PASSPHRASE:-}" ]]; then + GPG_OPTS=() +else + GPG_OPTS=("--gpg-options=\"--batch" "--pinentry-mode=loopback" "--yes") +fi + + echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}" # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. for _ in {1..10}; do ( - ${DEBS3_UPLOAD} \ - --component "${DEB_RELEASE}" \ - --codename "${DEB_CODENAME}" \ - "${DEB_NAMES}" + ${DEBS3_UPLOAD} --component "${DEB_RELEASE}" --codename "${DEB_CODENAME}" "${GPG_OPTS[@]}" "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done -debs=() - for deb in $DEB_NAMES do # extracting name from debian package path. E.g: diff --git a/scripts/debian/reversion.sh b/scripts/debian/reversion.sh index bd937ca5387..94cb3e27aa8 100755 --- a/scripts/debian/reversion.sh +++ b/scripts/debian/reversion.sh @@ -3,6 +3,7 @@ set -eo pipefail CLEAR='\033[0m' RED='\033[0;31m' +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -d|--deb) DEB="$2"; shift;; @@ -14,6 +15,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --new-version) NEW_VERSION="$2"; shift;; --suite) SUITE="$2"; shift;; --new-suite) NEW_SUITE="$2"; shift;; + --sign) SIGN="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -30,6 +32,7 @@ function usage() { echo " --new-version The New Debian version" echo " --suite The Current Debian suite" echo " --new-suite The New Debian suite" + echo " --sign The Public Key id, which is used to sign package. Key must be stored locally" echo "" echo "Example: $0 --deb mina-archive --version 2.0.0-rc1-48efea4 --new-version 2.0.0-rc1 --codename bullseye --release unstable --new-release umt" exit 1 @@ -39,18 +42,21 @@ if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$NEW_RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$NEW_SUITE" ]]; then NEW_SUITE=$SUITE; fi; - if [[ -z "$DEB" ]]; then NEW_NAME=$DEB; fi; if [[ -z "$RELEASE" ]]; then NEW_RELEASE=$RELEASE; fi; if [[ -z "$VERSION" ]]; then NEW_VERSION=$VERSION; fi; if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi; - +if [[ -z "$SIGN" ]]; then + SIGN_ARG="" +else + SIGN_ARG="--sign $SIGN" +fi function rebuild_deb() { rm -f "${DEB}_${VERSION}.deb" rm -rf "${NEW_NAME}_${NEW_VERSION}" - - wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/"${CODENAME}"/m/mi/"${DEB}"_"${VERSION}".deb + + wget https://s3.us-west-2.amazonaws.com/${BUCKET}/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}" sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" @@ -59,5 +65,4 @@ function rebuild_deb() { } rebuild_deb - -source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" --bucket ${BUCKET} ${SIGN_ARG} diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index 25877aa3f67..b0837a142a6 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -4,12 +4,14 @@ set -eox pipefail CHANNEL=umt-mainnet VERSION=3.0.0-f872d85 CODENAME=bullseye +BUCKET=packages.o1test.net while [[ "$#" -gt 0 ]]; do case $1 in -c|--channel) CHANNEL="$2"; shift;; -v|--version) VERSION="$2"; shift;; -p|--package) PACKAGE="$2"; shift;; -m|--codename) CODENAME="$2"; shift;; + -b|--bucket) BUCKET="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -29,7 +31,7 @@ SCRIPT=' set -x \ && echo installing mina \ && apt-get update > /dev/null \ && apt-get install -y lsb-release ca-certificates > /dev/null \ - && echo "deb [trusted=yes] http://packages.o1test.net '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ + && echo "deb [trusted=yes] https://'$BUCKET' '$CODENAME' '$CHANNEL'" > /etc/apt/sources.list.d/mina.list \ && apt-get update > /dev/null \ && apt list -a '$PACKAGE' \ && apt-get install -y --allow-downgrades '$PACKAGE'='$VERSION' \ diff --git a/scripts/docker/build.sh b/scripts/docker/build.sh index 34f8f62a0eb..4c400e7e396 100755 --- a/scripts/docker/build.sh +++ b/scripts/docker/build.sh @@ -47,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in --deb-profile) DEB_PROFILE="$2"; shift;; --deb-repo) INPUT_REPO="$2"; shift;; --deb-build-flags) DEB_BUILD_FLAGS="$2"; shift;; + --deb-repo-key) DEB_REPO_KEY="$2"; shift;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done