allow to use new debian repos [dev]

buildkite/scripts/cache-artifact.sh

@@ -9,7 +9,7 @@ if [[ $# -ne 2 ]]; then
 fi
 
 UPLOAD_BIN=gsutil
-PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID}
+PREFIX=gs://buildkite_k8s/coda/shared/019365c5-e59b-4d54-941c-3d0f5adde52c
 FILE="$1"
 REMOTE_LOCATION="$2"
 

buildkite/scripts/debian/promote.sh

@@ -12,6 +12,8 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -s|--from-component) FROM_COMPONENT="$2"; shift;;
   -t|--to-component) TO_COMPONENT="$2"; shift;;
   --new-name) NEW_NAME="$2"; shift;;
+  --new-repo) NEW_REPO="$2"; shift;;
+  -r|--repo) REPO="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -28,6 +30,8 @@ function usage() {
   echo "  -s, --from-component   The source channel in which package currently resides"
   echo "  -t, --to-component     The target channel for package (unstable, alpha, beta etc.)"
   echo "  -c, --codename         The Debian codename (bullseye, focal etc.)"
+  echo "  -r, --repo             The Debian source repo"
+  echo "  --new-repo             The Debian target repo. By default equal to repo"
   echo ""
   echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly"
   exit 1
@@ -40,6 +44,8 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi;
 if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi;
 if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi;
+if [[ -z "$REPO" ]]; then usage "Repository is not set!"; fi;
+if [[ -z "$NEW_REPO" ]]; then NEW_REPO=$REPO; fi;
 
 # check for AWS Creds
 if [ -z "$AWS_ACCESS_KEY_ID" ]; then
@@ -65,5 +71,7 @@ else
     --new-version $NEW_VERSION \
     --suite $FROM_COMPONENT \
     --new-suite $TO_COMPONENT \
-    --new-name $NEW_NAME
+    --new-name $NEW_NAME \
+    --repo $REPO \
+    --new-repo $NEW_REPO
 fi

buildkite/scripts/debian/publish.sh

@@ -19,4 +19,5 @@ source scripts/debian/publish.sh \
   --names "${DOWNLOAD_FOLDER}/mina-*.deb" \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \
-  --codename $MINA_DEB_CODENAME  
+  --codename $MINA_DEB_CODENAME \
+  --bucket $BUCKET

buildkite/scripts/download-artifact-from-cache.sh

@@ -9,7 +9,7 @@ if [[ $# -lt 2 ]]; then
 fi
 
 DOWNLOAD_BIN=gsutil
-PREFIX=gs://buildkite_k8s/coda/shared/${BUILDKITE_BUILD_ID}
+PREFIX=gs://buildkite_k8s/coda/shared/019365c5-e59b-4d54-941c-3d0f5adde52c
 FILE="$1"
 REMOTE_LOCATION="$2"
 OPTS=${3:-""}

buildkite/src/Command/DockerImage.dhall

@@ -113,18 +113,23 @@ let generateStep =
                 ++  " --deb-build-flags ${BuildFlags.lowerName
                                             spec.build_flags}"
 
+          let remoteRepoCmds =
+                [ Cmd.run
+                    (     exportMinaDebCmd
+                      ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
+                      ++  " && "
+                      ++  buildDockerCmd
+                      ++  " && "
+                      ++  releaseDockerCmd
+                    )
+                ]
+
           let commands =
                 merge
-                  { PackagesO1Test =
-                    [ Cmd.run
-                        (     exportMinaDebCmd
-                          ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
-                          ++  " && "
-                          ++  buildDockerCmd
-                          ++  " && "
-                          ++  releaseDockerCmd
-                        )
-                    ]
+                  { PackagesO1Test = remoteRepoCmds
+                  , Unstable = remoteRepoCmds
+                  , Nightly = remoteRepoCmds
+                  , Stable = remoteRepoCmds
                   , Local =
                     [ Cmd.run
                         (     exportMinaDebCmd

buildkite/src/Command/MinaArtifact.dhall

@@ -46,6 +46,7 @@ let MinaBuildSpec =
           , mode : PipelineMode.Type
           , tags : List PipelineTag.Type
           , channel : DebianChannel.Type
+          , debianRepo : DebianRepo.Type
           }
       , default =
           { prefix = "MinaArtifact"
@@ -58,6 +59,7 @@ let MinaBuildSpec =
           , mode = PipelineMode.Type.PullRequest
           , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
           , channel = DebianChannel.Type.Unstable
+          , debianRepo = DebianRepo.Type.Unstable
           }
       }
 
@@ -137,6 +139,8 @@ let publish_to_debian_repo =
                   , "MINA_DEB_CODENAME=${DebianVersions.lowerName
                                            spec.debVersion}"
                   , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}"
+                  , "${DebianRepo.keyIdEnv spec.debianRepo}"
+                  , "${DebianRepo.bucketEnv spec.debianRepo}"
                   ]
                   "./buildkite/scripts/debian/publish.sh"
             , label =

buildkite/src/Command/PromotePackage.dhall

@@ -59,7 +59,7 @@ let PromoteDebianSpec =
           , network = Network.Type.Devnet
           , codename = DebianVersions.DebVersion.Bullseye
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyDevelop
+          , to_channel = DebianChannel.Type.Alpha
           , profile = Profiles.Type.Standard
           , remove_profile_from_name = False
           , step_key = "promote-debian-package"

buildkite/src/Command/Promotion/PromoteDebian.dhall

@@ -12,6 +12,8 @@ let Profiles = ../../Constants/Profiles.dhall
 
 let DebianVersions = ../../Constants/DebianVersions.dhall
 
+let DebianRepo = ../../Constants/DebianRepo.dhall
+
 let Toolchain = ../../Constants/Toolchain.dhall
 
 let Command = ../Base.dhall
@@ -29,6 +31,8 @@ let PromoteDebianSpec =
           , codename : DebianVersions.DebVersion
           , from_channel : DebianChannel.Type
           , to_channel : DebianChannel.Type
+          , source_repo : DebianRepo.Type
+          , target_repo : DebianRepo.Type
           , profile : Profiles.Type
           , remove_profile_from_name : Bool
           , step_key : Text
@@ -43,7 +47,9 @@ let PromoteDebianSpec =
           , network = Network.Type.Devnet
           , codename = DebianVersions.DebVersion.Bullseye
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyCompatible
+          , to_channel = DebianChannel.Type.Compatible
+          , source_repo = DebianRepo.Type.Unstable
+          , target_repo = DebianRepo.Type.Nightly
           , profile = Profiles.Type.Standard
           , remove_profile_from_name = False
           , step_key = "promote-debian-package"
@@ -76,10 +82,12 @@ let promoteDebianStep =
                       , "AWS_SECRET_ACCESS_KEY"
                       , "FROM_VERSION_MANUAL"
                       ]
-                      ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/debian/promote.sh --package ${package_name} --version ${spec.version}  --new-version ${spec.new_version}  --architecture ${spec.architecture}  --codename ${DebianVersions.lowerName
-                                                                                                                                                                                                                                                         spec.codename}  --from-component ${DebianChannel.lowerName
-                                                                                                                                                                                                                                                                                              spec.from_channel}  --to-component ${DebianChannel.lowerName
-                                                                                                                                                                                                                                                                                                                                     spec.to_channel} ${new_name}"
+                      ". ./buildkite/scripts/export-git-env-vars.sh && ./buildkite/scripts/debian/promote.sh --repo ${DebianRepo.bucket_or_default
+                                                                                                                        spec.source_repo} --new-repo ${DebianRepo.bucket_or_default
+                                                                                                                                                         spec.target_repo} --package ${package_name} --version ${spec.version}  --new-version ${spec.new_version}  --architecture ${spec.architecture}  --codename ${DebianVersions.lowerName
+                                                                                                                                                                                                                                                                                                                       spec.codename}  --from-component ${DebianChannel.lowerName
+                                                                                                                                                                                                                                                                                                                                                            spec.from_channel}  --to-component ${DebianChannel.lowerName
+                                                                                                                                                                                                                                                                                                                                                                                                   spec.to_channel} ${new_name}"
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key
                 , target = Size.Small

buildkite/src/Command/Promotion/PromotePackages.dhall

@@ -58,7 +58,7 @@ let PromotePackagesSpec =
           , network = Network.Type.Mainnet
           , codenames = [] : List DebianVersions.DebVersion
           , from_channel = DebianChannel.Type.Unstable
-          , to_channel = DebianChannel.Type.NightlyCompatible
+          , to_channel = DebianChannel.Type.Compatible
           , new_tags = [] : List Text
           , remove_profile_from_name = False
           , publish = False

buildkite/src/Command/Promotion/VerifyDebian.dhall

@@ -2,6 +2,8 @@ let Package = ../../Constants/DebianPackage.dhall
 
 let DebianChannel = ../../Constants/DebianChannel.dhall
 
+let DebianRepo = ../../Constants/DebianRepo.dhall
+
 let Profiles = ../../Constants/Profiles.dhall
 
 let DebianVersions = ../../Constants/DebianVersions.dhall
@@ -30,9 +32,10 @@ let promoteDebianVerificationStep =
                 Command.Config::{
                 , commands =
                   [ Cmd.run
-                      "source buildkite/scripts/export-git-env-vars.sh && ./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName
-                                                                                                                                                                    spec.codename}  --channel ${DebianChannel.lowerName
-                                                                                                                                                                                                  spec.to_channel}"
+                      "source buildkite/scripts/export-git-env-vars.sh && ./scripts/debian/verify.sh --bucket ${DebianRepo.bucket_or_default
+                                                                                                                  spec.target_repo}--package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName
+                                                                                                                                                                                                  spec.codename}  --channel ${DebianChannel.lowerName
+                                                                                                                                                                                                                                spec.to_channel}"
                   ]
                 , label = "Debian: ${spec.step_key}"
                 , key = spec.step_key

buildkite/src/Command/Promotion/VerifyPackages.dhall

@@ -58,7 +58,7 @@ let VerifyPackagesSpec =
           , profile = Profiles.Type.Standard
           , network = Network.Type.Mainnet
           , codenames = [] : List DebianVersions.DebVersion
-          , channel = DebianChannel.Type.NightlyCompatible
+          , channel = DebianChannel.Type.Compatible
           , new_tags = [] : List Text
           , remove_profile_from_name = False
           , published = False

buildkite/src/Constants/DebianChannel.dhall

@@ -1,8 +1,8 @@
 let Channel
     : Type
     = < Unstable
-      | NightlyDevelop
-      | NightlyCompatible
+      | Develop
+      | Compatible
       | Itn
       | Umt
       | UmtMainnet
@@ -17,8 +17,8 @@ let capitalName =
           \(channel : Channel)
       ->  merge
             { Unstable = "Unstable"
-            , NightlyDevelop = "NightlyDevelop"
-            , NightlyCompatible = "NightlyCompatible"
+            , Develop = "Develop"
+            , Compatible = "Compatible"
             , Itn = "Itn"
             , Umt = "Umt"
             , UmtMainnet = "UmtMainnet"
@@ -34,8 +34,8 @@ let lowerName =
           \(channel : Channel)
       ->  merge
             { Unstable = "unstable"
-            , NightlyDevelop = "nightly-develop"
-            , NightlyCompatible = "nightly-compatible"
+            , Develop = "develop"
+            , Compatible = "compatible"
             , Itn = "itn"
             , Umt = "umt"
             , UmtMainnet = "umt-mainnet"

buildkite/src/Constants/DebianRepo.dhall

@@ -1,13 +1,133 @@
+let Prelude = ../External/Prelude.dhall
+
+let Optional/map = Prelude.Optional.map
+
+let Optional/default = Prelude.Optional.default
+
 let DebianRepo
     : Type
-    = < Local | PackagesO1Test >
+    = < Local | PackagesO1Test | Unstable | Nightly | Stable >
 
 let address =
           \(repo : DebianRepo)
       ->  merge
             { Local = "http://localhost:8080"
             , PackagesO1Test = "http://packages.o1test.net"
+            , Unstable = "https://unstable.apt.packages.minaprotocol.com"
+            , Nightly = "https://nightly.apt.packages.minaprotocol.com"
+            , Stable = "https://stable.apt.packages.minaprotocol.com"
+            }
+            repo
+
+let bucket =
+          \(repo : DebianRepo)
+      ->  merge
+            { Local = None Text
+            , PackagesO1Test = Some "packages.o1test.net"
+            , Unstable = Some "unstable.apt.packages.minaprotocol.com"
+            , Nightly = Some "nightly.apt.packages.minaprotocol.com"
+            , Stable = Some "stable.apt.packages.minaprotocol.com"
+            }
+            repo
+
+let bucket_or_default =
+          \(repo : DebianRepo)
+      ->  let maybeBucket =
+                Optional/map
+                  Text
+                  Text
+                  (\(bucket : Text) -> bucket)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeBucket
+
+let bucketArg =
+          \(repo : DebianRepo)
+      ->  let maybeBucket =
+                Optional/map
+                  Text
+                  Text
+                  (\(bucket : Text) -> "--bucket " ++ bucket)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeBucket
+
+let keyId =
+          \(repo : DebianRepo)
+      ->  merge
+            { Local = None Text
+            , PackagesO1Test = None Text
+            , Unstable = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
+            , Nightly = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
+            , Stable = Some "B40D16B1A4773DE415DAF9DBFE236881C07523DC"
             }
             repo
 
-in  { Type = DebianRepo, address = address }
+let keyAddress =
+          \(repo : DebianRepo)
+      ->  let keyPath = "/key.asc"
+
+          in  merge
+                { Local = None Text
+                , PackagesO1Test = None Text
+                , Unstable = Some (address repo ++ keyPath)
+                , Nightly = Some (address repo ++ keyPath)
+                , Stable = Some (address repo ++ keyPath)
+                }
+                repo
+
+let keyAddressArg =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(key : Text) -> "--key-path " ++ key)
+                  (keyAddress repo)
+
+          in  Optional/default Text "" maybeKey
+
+let keyArg =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "--sign " ++ repo)
+                  (keyId repo)
+
+          in  Optional/default Text "" maybeKey
+
+let keyIdEnv =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "SIGN=" ++ repo)
+                  (keyId repo)
+
+          in  Optional/default Text "" maybeKey
+
+let bucketEnv =
+          \(repo : DebianRepo)
+      ->  let maybeKey =
+                Optional/map
+                  Text
+                  Text
+                  (\(repo : Text) -> "BUCKET=" ++ repo)
+                  (bucket repo)
+
+          in  Optional/default Text "" maybeKey
+
+in  { Type = DebianRepo
+    , keyIdEnv = keyIdEnv
+    , keyAddressArg = keyAddressArg
+    , address = address
+    , bucket = bucket
+    , bucket_or_default = bucket_or_default
+    , bucketArg = bucketArg
+    , bucketEnv = bucketEnv
+    , keyId = keyId
+    , keyArg = keyArg
+    }