diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall new file mode 100644 index 00000000000..0335aeeced1 --- /dev/null +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -0,0 +1,113 @@ +let Cmd = ../../Lib/Cmds.dhall + +let B = ../../External/Buildkite.dhall + +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineMode = ../../Pipeline/Mode.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall + +let Size = ../../Command/Size.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + +let Dockers = ../../Constants/DockerVersions.dhall + +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type + +let Spec = + { Type = + { dockerType : Dockers.Type + , network : Network.Type + , mode : PipelineMode.Type + , additionalDirtyWhen : List S.Type + , softFail : B/SoftFail + , timeout : Natural + } + , default = + { dockerType = Dockers.Type.Bullseye + , network = Network.Type.Devnet + , mode = PipelineMode.Type.Stable + , additionalDirtyWhen = [] : List S.Type + , softFail = B/SoftFail.Boolean False + , timeout = 1000 + } + } + +let command + : Spec.Type -> Command.Type + = \(spec : Spec.Type) + -> Command.build + Command.Config::{ + , commands = + [ Cmd.chain + [ "export MINA_DEB_CODENAME=${Dockers.lowerName + spec.dockerType}" + , "source ./buildkite/scripts/export-git-env-vars.sh" + , "scripts/tests/rosetta-connectivity.sh --network ${Network.lowerName + spec.network} --tag \\\${MINA_DOCKER_TAG} --timeout ${Natural/show + spec.timeout}" + ] + ] + , label = + "Rosetta ${Network.lowerName spec.network} connectivity test " + , key = + "rosetta-${Network.lowerName spec.network}-connectivity-test" + , target = Size.XLarge + , soft_fail = Some spec.softFail + , depends_on = + Dockers.dependsOn + spec.dockerType + spec.network + Profiles.Type.Standard + Artifacts.Type.Rosetta + } + +let pipeline + : Spec.Type -> Pipeline.Config.Type + = \(spec : Spec.Type) + -> Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly + "buildkite/src/Jobs/Test/RosettaIntegrationTests" + "dhall" + , S.exactly + "buildkite/src/Jobs/Test/Rosetta${Network.capitalName + spec.network}Connect" + "dhall" + , S.exactly + "buildkite/src/Command/Rosetta/Connectivity" + "dhall" + , S.exactly "scripts/tests/rosetta-connectivity" "sh" + , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" + , S.exactly + "buildkite/scripts/rosetta-integration-tests-full" + "sh" + ] + # spec.additionalDirtyWhen + , path = "Test" + , name = "Rosetta${Network.capitalName spec.network}Connect" + , mode = spec.mode + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = [ command spec ] + } + +in { command = command, pipeline = pipeline, Spec = Spec } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 926a077b77d..eb2dfeb9600 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -28,8 +28,9 @@ let lowerName = } docker -let dependsOn = +let dependsOnStep = \(docker : Docker) + -> \(prefix : Text) -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) @@ -37,8 +38,6 @@ let dependsOn = let profileSuffix = "${Profiles.toSuffixUppercase profile}" - let prefix = "MinaArtifact" - let suffix = "docker-image" let key = "${Artifacts.lowerName binary}-${suffix}" @@ -77,8 +76,16 @@ let dependsOn = } docker +let dependsOn = + \(docker : Docker) + -> \(network : Network.Type) + -> \(profile : Profiles.Type) + -> \(binary : Artifacts.Type) + -> dependsOnStep docker "MinaArtifact" network profile binary + in { Type = Docker , capitalName = capitalName , lowerName = lowerName , dependsOn = dependsOn + , dependsOnStep = dependsOnStep } diff --git a/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall new file mode 100644 index 00000000000..a41f4cb5484 --- /dev/null +++ b/buildkite/src/Jobs/Test/RosettaDevnetConnect.dhall @@ -0,0 +1,10 @@ +let Pipeline = ../../Pipeline/Dsl.dhall + +let Network = ../../Constants/Network.dhall + +let Connectivity = ../../Command/Rosetta/Connectivity.dhall + +in Pipeline.build + ( Connectivity.pipeline + Connectivity.Spec::{ network = Network.Type.Devnet } + ) diff --git a/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall new file mode 100644 index 00000000000..7c081efd7c2 --- /dev/null +++ b/buildkite/src/Jobs/Test/RosettaMainnetConnect.dhall @@ -0,0 +1,10 @@ +let Pipeline = ../../Pipeline/Dsl.dhall + +let Network = ../../Constants/Network.dhall + +let Connectivity = ../../Command/Rosetta/Connectivity.dhall + +in Pipeline.build + ( Connectivity.pipeline + Connectivity.Spec::{ network = Network.Type.Mainnet, timeout = 2400 } + ) diff --git a/scripts/tests/rosetta-connectivity.sh b/scripts/tests/rosetta-connectivity.sh new file mode 100755 index 00000000000..d48bb918941 --- /dev/null +++ b/scripts/tests/rosetta-connectivity.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# end to end test for rosetta connectivity with given network +set -x +CLEAR='\033[0m' +RED='\033[0;31m' + +NETWORK=devnet +TIMEOUT=900 + +while [[ "$#" -gt 0 ]]; do case $1 in + -n|--network) NETWORK="$2"; shift;; + -t|--tag) TAG="$2"; shift;; + --timeout) TIMEOUT="$2"; shift;; + -h|--help) usage; exit 0;; + *) echo "Unknown parameter passed: $1"; usage; exit 1;; +esac; shift; done + + +function usage() { + if [[ -n "$1" ]]; then + echo -e "${RED}☞ $1${CLEAR}\n"; + fi + echo "Usage: $0 [-t docker-tag] [-n network]" + echo " -t, --version The version to be used in the docker image tag" + echo " -n, --network The network configuration to use (devnet or mainnet). Default=$NETWORK" + echo " -h, --help Show help" + echo "" + echo "Example: $0 --network devnet --tag 3.0.3-bullseye-berkeley " + +} + +if [[ -z "$TAG" ]]; then usage "Docker tag is not set!"; usage; exit 1; fi; + +container_id=$(docker run -d --env MINA_NETWORK=$NETWORK gcr.io/o1labs-192920/mina-rosetta:$TAG-$NETWORK ) + +stop_docker() { + { docker stop "$container_id" ; docker rm "$container_id" ; } || true +} + +trap stop_docker ERR + +# Command to run the process +process_command="docker logs $container_id -f" + +# Pattern to wait for in the output +pattern="Added block with state hash" + +# Timeout duration in seconds +timeout_duration=$TIMEOUT + +# Required number of matches +required_matches=10 + +# Count of pattern matches +match_count=0 + +# Start time +start_time=$(date +%s) + +# Wait timeout for next line in output +next_line_timeout=60 + +# Run the rosetta docker and check its output +while IFS= read -t $next_line_timeout -r line; do + # Get the current time + current_time=$(date +%s) + + # Check if the timeout has been exceeded + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout_duration" ]; then + echo "Timeout reached! The pattern ('$pattern') was not found $required_matches times." + exit 1 + fi + + # If the pattern is found, increment the match count + if [[ "$line" =~ $pattern ]]; then + match_count=$((match_count+1)) + echo "Pattern found ($match_count of $required_matches): $line" + fi + + # If the required number of matches is reached, exit the loop + if [ "$match_count" -ge "$required_matches" ]; then + echo "Pattern found $required_matches times. Exiting..." + break + fi +done < <($process_command) + +stop_docker + +# Check if we met the required match count +if [ "$match_count" -ge "$required_matches" ]; then + echo "Successfully found the pattern ('$pattern') $required_matches times." + exit 0 +else + echo "Failed to find the pattern ('$pattern') $required_matches times within the timeout." + exit 1 +fi \ No newline at end of file