Merge pull request #16390 from MinaProtocol/dkijania/rosetta_start_is…

…sue_dev

port rosetta docker-start fix and test to dev

buildkite/src/Command/Rosetta/Connectivity.dhall

@@ -0,0 +1,113 @@
+let Cmd = ../../Lib/Cmds.dhall
+
+let B = ../../External/Buildkite.dhall
+
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+
+let Size = ../../Command/Size.dhall
+
+let Network = ../../Constants/Network.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Dockers = ../../Constants/DockerVersions.dhall
+
+let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
+
+let Spec =
+      { Type =
+          { dockerType : Dockers.Type
+          , network : Network.Type
+          , mode : PipelineMode.Type
+          , additionalDirtyWhen : List S.Type
+          , softFail : B/SoftFail
+          , timeout : Natural
+          }
+      , default =
+          { dockerType = Dockers.Type.Bullseye
+          , network = Network.Type.Devnet
+          , mode = PipelineMode.Type.Stable
+          , additionalDirtyWhen = [] : List S.Type
+          , softFail = B/SoftFail.Boolean False
+          , timeout = 1000
+          }
+      }
+
+let command
+    : Spec.Type -> Command.Type
+    =     \(spec : Spec.Type)
+      ->  Command.build
+            Command.Config::{
+            , commands =
+              [ Cmd.chain
+                  [ "export MINA_DEB_CODENAME=${Dockers.lowerName
+                                                  spec.dockerType}"
+                  , "source ./buildkite/scripts/export-git-env-vars.sh"
+                  , "scripts/tests/rosetta-connectivity.sh --network ${Network.lowerName
+                                                                         spec.network} --tag \\\${MINA_DOCKER_TAG} --timeout ${Natural/show
+                                                                                                                                 spec.timeout}"
+                  ]
+              ]
+            , label =
+                "Rosetta ${Network.lowerName spec.network} connectivity test "
+            , key =
+                "rosetta-${Network.lowerName spec.network}-connectivity-test"
+            , target = Size.XLarge
+            , soft_fail = Some spec.softFail
+            , depends_on =
+                Dockers.dependsOn
+                  spec.dockerType
+                  spec.network
+                  Profiles.Type.Standard
+                  Artifacts.Type.Rosetta
+            }
+
+let pipeline
+    : Spec.Type -> Pipeline.Config.Type
+    =     \(spec : Spec.Type)
+      ->  Pipeline.Config::{
+          , spec = JobSpec::{
+            , dirtyWhen =
+                  [ S.strictlyStart (S.contains "src")
+                  , S.exactly
+                      "buildkite/src/Jobs/Test/RosettaIntegrationTests"
+                      "dhall"
+                  , S.exactly
+                      "buildkite/src/Jobs/Test/Rosetta${Network.capitalName
+                                                          spec.network}Connect"
+                      "dhall"
+                  , S.exactly
+                      "buildkite/src/Command/Rosetta/Connectivity"
+                      "dhall"
+                  , S.exactly "scripts/tests/rosetta-connectivity" "sh"
+                  , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh"
+                  , S.exactly
+                      "buildkite/scripts/rosetta-integration-tests-full"
+                      "sh"
+                  ]
+                # spec.additionalDirtyWhen
+            , path = "Test"
+            , name = "Rosetta${Network.capitalName spec.network}Connect"
+            , mode = spec.mode
+            , tags =
+              [ PipelineTag.Type.Long
+              , PipelineTag.Type.Test
+              , PipelineTag.Type.Stable
+              ]
+            }
+          , steps = [ command spec ]
+          }
+
+in  { command = command, pipeline = pipeline, Spec = Spec }

buildkite/src/Constants/DockerVersions.dhall

@@ -28,17 +28,16 @@ let lowerName =
             }
             docker
 
-let dependsOn =
+let dependsOnStep =
           \(docker : Docker)
+      ->  \(prefix : Text)
       ->  \(network : Network.Type)
       ->  \(profile : Profiles.Type)
       ->  \(binary : Artifacts.Type)
       ->  let network = "${Network.capitalName network}"
 
           let profileSuffix = "${Profiles.toSuffixUppercase profile}"
 
-          let prefix = "MinaArtifact"
-
           let suffix = "docker-image"
 
           let key = "${Artifacts.lowerName binary}-${suffix}"
@@ -77,8 +76,16 @@ let dependsOn =
                 }
                 docker
 
+let dependsOn =
+          \(docker : Docker)
+      ->  \(network : Network.Type)
+      ->  \(profile : Profiles.Type)
+      ->  \(binary : Artifacts.Type)
+      ->  dependsOnStep docker "MinaArtifact" network profile binary
+
 in  { Type = Docker
     , capitalName = capitalName
     , lowerName = lowerName
     , dependsOn = dependsOn
+    , dependsOnStep = dependsOnStep
     }

buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall

@@ -0,0 +1,10 @@
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Network = ../../Constants/Network.dhall
+
+let Connectivity = ../../Command/Rosetta/Connectivity.dhall
+
+in  Pipeline.build
+      ( Connectivity.pipeline
+          Connectivity.Spec::{ network = Network.Type.Devnet }
+      )

buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall

@@ -0,0 +1,10 @@
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let Network = ../../Constants/Network.dhall
+
+let Connectivity = ../../Command/Rosetta/Connectivity.dhall
+
+in  Pipeline.build
+      ( Connectivity.pipeline
+          Connectivity.Spec::{ network = Network.Type.Mainnet, timeout = 2400 }
+      )

scripts/tests/rosetta-connectivity.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# end to end test for rosetta connectivity with given network 
+set -x
+CLEAR='\033[0m'
+RED='\033[0;31m'
+
+NETWORK=devnet
+TIMEOUT=900
+
+while [[ "$#" -gt 0 ]]; do case $1 in
+  -n|--network) NETWORK="$2"; shift;;
+  -t|--tag) TAG="$2"; shift;;
+  --timeout) TIMEOUT="$2"; shift;;
+  -h|--help) usage; exit 0;;
+  *) echo "Unknown parameter passed: $1"; usage; exit 1;;
+esac; shift; done
+
+
+function usage() {
+  if [[ -n "$1" ]]; then
+    echo -e "${RED}☞  $1${CLEAR}\n";
+  fi
+  echo "Usage: $0 [-t docker-tag] [-n network]"
+  echo "  -t, --version             The version to be used in the docker image tag"
+  echo "  -n, --network             The network configuration to use (devnet or mainnet). Default=$NETWORK"
+  echo "  -h, --help                Show help"
+  echo ""
+  echo "Example: $0 --network devnet --tag 3.0.3-bullseye-berkeley "
+
+}
+
+if [[ -z "$TAG" ]]; then usage "Docker tag is not set!"; usage; exit 1; fi;
+
+container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK )
+
+stop_docker() {
+    { docker stop "$container_id" ; docker rm "$container_id" ; } || true
+}
+
+trap stop_docker ERR
+
+# Command to run the process
+process_command="docker logs $container_id -f"
+
+# Pattern to wait for in the output
+pattern="Added block with state hash"
+
+# Timeout duration in seconds
+timeout_duration=$TIMEOUT
+
+# Required number of matches
+required_matches=10
+
+# Count of pattern matches
+match_count=0
+
+# Start time
+start_time=$(date +%s)
+
+# Wait timeout for next line in output
+next_line_timeout=60
+
+# Run the rosetta docker and check its output
+while  IFS= read -t $next_line_timeout -r line; do
+    # Get the current time
+    current_time=$(date +%s)
+
+    # Check if the timeout has been exceeded
+    elapsed_time=$((current_time - start_time))
+    if [ "$elapsed_time" -ge "$timeout_duration" ]; then
+        echo "Timeout reached! The pattern ('$pattern') was not found $required_matches times."
+        exit 1
+    fi
+
+    # If the pattern is found, increment the match count
+    if [[ "$line" =~ $pattern ]]; then
+        match_count=$((match_count+1))
+        echo "Pattern found ($match_count of $required_matches): $line"
+    fi
+
+    # If the required number of matches is reached, exit the loop
+    if [ "$match_count" -ge "$required_matches" ]; then
+        echo "Pattern found $required_matches times. Exiting..."
+        break
+    fi
+done < <($process_command)
+
+stop_docker
+
+# Check if we met the required match count
+if [ "$match_count" -ge "$required_matches" ]; then
+    echo "Successfully found the pattern ('$pattern') $required_matches times."
+    exit 0
+else
+    echo "Failed to find the pattern ('$pattern') $required_matches times within the timeout."
+    exit 1
+fi