Gennet/Kurtosis info into WLS with master changes

.gitignore

@@ -27,3 +27,5 @@ config/topology_generated/
 config/waku_config_files/
 kurtosisrun_log.txt
 summary.json
+wakurtosis_logs/
+log_trace_test

README.md

@@ -9,7 +9,7 @@ More info about Kurtosis: https://docs.kurtosis.com/
 
 #### Before using this repository note that: 
 
-- **You are using Kurtosis version 0.67.0**. This is important, as they are working on it and changes can be huge depending on different versions. You can find all Kurtosis versions [here](https://github.com/kurtosis-tech/kurtosis-cli-release-artifacts/releases).
+- **You are using Kurtosis version 0.67.3**. This is important, as they are working on it and changes can be huge depending on different versions. You can find all Kurtosis versions [here](https://github.com/kurtosis-tech/kurtosis-cli-release-artifacts/releases).
 - The topology files that will be used by default are defined in `config/topology_generated/`. This topology is created with the [gennet](gennet-module/Readme.md) module.
 - Kurtosis can set up services in a parallel manner, defined in the `config.json` file (see below).
 - Only `kurtosis` and `docker` are needed to run this.

build.sh

@@ -2,8 +2,10 @@
 sudo apt-get update
 sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
+apt-get install -y jq
+
 # Install the suitable kurtosis-cli
-kurtosis_version=0.67.0
+kurtosis_version=0.67.3
 echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
 sudo apt update
 sudo apt-mark unhold kurtosis-cli

config/config.json

@@ -25,7 +25,8 @@
     "benchmark": "False"
   },
   "wls": {
-    "simulation_time": 6,
+    "debug_level": "DEBUG",
+    "simulation_time": 60,
     "message_rate": 10,
     "min_packet_size": 2,
     "max_packet_size": 1024,

config/test_files/nwaku_0.toml

@@ -1,4 +1,4 @@
-topics="test"
+topics="test asd"
 rpc-admin=true
 keep-alive=true
 metrics-server=true

config/test_files/nwaku_2.toml

@@ -0,0 +1,5 @@
+topics="test"
+rpc-admin=true
+keep-alive=true
+metrics-server=true
+nodekey="80fb8f9b71c808bc186bc8c6639a12446e667d031e8ac4896efad603e21728b4"

config/test_files/nwaku_3.toml

@@ -0,0 +1,5 @@
+topics="test"
+rpc-admin=true
+keep-alive=true
+metrics-server=true
+nodekey="80fb8f9b71c808bc186bc8c6639a12446e667d031e8ac4896efad603e21728b5"

config/test_files/test_config_file.json

@@ -0,0 +1,14 @@
+{
+  "general":{
+    "prng_seed" : 67
+  },
+  "kurtosis": {
+    "enclave_name": "wakurtosis",
+  },
+  "gennet": {
+    "num_nodes": 5
+  },
+  "wls": {
+    "simulation_time": 60
+  }
+}

config/test_files/test_network_data.json

@@ -1,18 +1,58 @@
 {
-  "nwaku_0": {
-    "static_nodes": [
-      "nwaku_1"
+  "containers": {
+    "container_0": [
+      "nwaku_0",
+      "nwaku_2"
     ],
-    "subnetwork": "subnetwork_0",
-    "image": "nim-waku",
-    "configuration": "nwaku_0.toml"
+    "container_1": [
+      "nwaku_1",
+      "nwaku_3"
+    ]
   },
-  "nwaku_1": {
-    "static_nodes": [
-      "nwaku_0"
-    ],
-    "subnetwork": "subnetwork_0",
-    "image": "nim-waku",
-    "configuration": "nwaku_1.toml"
+  "nodes": {
+    "nwaku_0": {
+      "static_nodes": [
+        "nwaku_1"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_0.toml",
+      "node_log": "node_0.log",
+      "port_shift": 0,
+      "container_id": "container_0"
+    },
+    "nwaku_1": {
+      "static_nodes": [
+        "nwaku_0"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_1.toml",
+      "node_log": "node_1.log",
+      "port_shift": 0,
+      "container_id": "container_1"
+    },
+    "nwaku_2": {
+      "static_nodes": [
+        "nwaku_3"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_2.toml",
+      "node_log": "node_1.log",
+      "port_shift": 1,
+      "container_id": "container_0"
+    },
+    "nwaku_3": {
+      "static_nodes": [
+        "nwaku_2"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_3.toml",
+      "node_log": "node_3.log",
+      "port_shift": 1,
+      "container_id": "container_1"
+    }
   }
 }

config/test_files/test_network_data_2.json

@@ -1,18 +1,34 @@
 {
-  "nwaku_0_2": {
-    "static_nodes": [
-      "nwaku_1_2"
-    ],
-    "subnetwork": "subnetwork_0",
-    "image": "nim-waku",
-    "configuration": "nwaku_0_2.toml"
-  },
-  "nwaku_1_2": {
-    "static_nodes": [
+  "containers": {
+    "cid1": [
       "nwaku_0_2"
     ],
-    "subnetwork": "subnetwork_0",
-    "image": "nim-waku",
-    "configuration": "nwaku_1_2.toml"
+    "cid2": [
+      "nwaku_1_2"
+    ]
+  },
+  "nodes": {
+    "nwaku_0_2": {
+      "static_nodes": [
+        "nwaku_1_2"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_0_2.toml",
+      "node_log": "node_0_2.log",
+      "port_shift": 0,
+      "container_id": "cid1"
+    },
+    "nwaku_1_2": {
+      "static_nodes": [
+        "nwaku_0_2"
+      ],
+      "subnetwork": "subnetwork_0",
+      "image": "nim-waku",
+      "node_config": "nwaku_1_2.toml",
+      "node_log": "node_1_2.log",
+      "port_shift": 0,
+      "container_id": "cid2"
+    }
   }
 }

main.star

@@ -17,7 +17,6 @@ def run(plan, args):
     config = json.decode(config_json)
 
     kurtosis_config = config[vars.KURTOSIS_KEY]
-    wls_config = config[vars.WLS_KEY]
     interconnection_batch = kurtosis_config[vars.INTERCONNECTION_BATCH_KEY]
 
     # Load network topology
@@ -35,4 +34,4 @@ def run(plan, args):
     nodes.interconnect_nodes(plan, network_topology, interconnection_batch)
 
     # Setup WLS & Start the Simulation
-    wls_service = wls.init(plan, network_topology, wls_config)
+    wls_service = wls.init(plan, network_topology, config_file)

monitoring/prometheus.yml

@@ -10,4 +10,8 @@ scrape_configs:
     scrape_interval: 1s
     file_sd_configs:
     - files:
-      - '/tmp/targets.json'
+      - '/tmp/targets.json'
+  - job_name: 'cadvisor'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['cadvisor:8080']

prepare_env.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+kurtosis engine stop
+
+ulimit -n $(ulimit -n -H)
+ulimit -u $(ulimit -u -H)
+
+sudo sysctl -w net.ipv4.neigh.default.gc_thresh3=4096
+sudo sysctl fs.inotify.max_user_instances=1048576
+sudo sysctl -w vm.max_map_count=262144
+
+sudo docker container rm -f $(docker container ls -aq)
+sudo docker volume rm -f $(docker volume ls -q)
+
+kurtosis engine start

run.sh

@@ -2,10 +2,6 @@
 
 dir=$(pwd)
 
-# Set up Cadvisor
-docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/var/lib/docker/:/var/lib/docker:ro --volume=/dev/disk/:/dev/disk:ro --volume=/sys:/sys:ro --volume=/etc/machine-id:/etc/machine-id:ro --publish=8080:8080 --detach=true --name=cadvisor --privileged --device=/dev/kmsg gcr.io/cadvisor/cadvisor
-
-
 # Parse arg if any
 ARGS1=${1:-"wakurtosis"}
 ARGS2=${2:-"config.json"}
@@ -18,9 +14,45 @@ loglevel="error"
 echo "- Enclave name: " $enclave_name
 echo "- Configuration file: " $wakurtosis_config_file
 
-# Run Gennet 
-echo -e "\nRunning Gennet"
+# Delete the enclave just in case
+echo -e "\nCleaning up Kurtosis environment "$enclave_name
+docker container stop cadvisor > /dev/null 2>&1
+docker container rm cadvisor > /dev/null 2>&1
+kurtosis enclave rm -f $enclave_name > /dev/null 2>&1
+# kurtosis clean -a > /dev/null 2>&1 we do not want to delete all enclaves, just the one we will execute
+
+# Delete previous logs
+echo -e "\Deleting previous logs in ${enclave_name}_logs"
+rm -rf ./${enclave_name}_logs > /dev/null 2>&1
+rm ./kurtosisrun_log.txt > /dev/null 2>&1
+
+# Preparing enclave
+echo "Preparing enclave..."
+kurtosis enclave add --name ${enclave_name}
+enclave_preffix="$(kurtosis enclave inspect --full-uuids $enclave_name | grep UUID: | awk '{print $2}')"
+echo "Enclave network: "$enclave_preffix
+
+# Get enclave last IP
+subnet="$(docker network inspect $enclave_preffix | jq -r '.[].IPAM.Config[0].Subnet')"
+echo "Enclave subnetork: $subnet"
+last_ip="$(ipcalc $subnet | grep HostMax | awk '{print $2}')"
+echo "cAdvisor IP: $last_ip"
+
+
+# Set up Cadvisor
+# docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/var/lib/docker/:/var/lib/docker:ro --volume=/dev/disk/:/dev/disk:ro --volume=/sys:/sys:ro --volume=/etc/machine-id:/etc/machine-id:ro --publish=8080:8080 --detach=true --name=cadvisor --privileged --device=/dev/kmsg gcr.io/cadvisor/cadvisor
+docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/var/lib/docker/:/var/lib/docker:ro --volume=/dev/disk/:/dev/disk:ro --volume=/sys:/sys:ro --volume=/etc/machine-id:/etc/machine-id:ro --publish=8080:8080 --detach=true --name=cadvisor --privileged --device=/dev/kmsg --network $enclave_preffix --ip=$last_ip gcr.io/cadvisor/cadvisor:v0.47.0
+
+
+# Delete topology
+rm -rf ./config/topology_generated > /dev/null 2>&1
+# Remove previous logs
+rm -rf ./$enclave_name_logs > /dev/null 2>&1
+
+# Create and run Gennet docker container
+echo -e "\nRunning network generation"
 docker rm gennet-container > /dev/null 2>&1  # cleanup the old docker if any
+
 cd gennet-module
 docker run --name gennet-container -v ${dir}/config/:/config gennet --config-file /config/${wakurtosis_config_file} --output-dir /config/topology_generated
 err=$?
@@ -34,10 +66,7 @@ fi
 
 docker rm gennet-container > /dev/null 2>&1
 
-# Delete the enclave just in case
-kurtosis enclave rm -f $enclave_name > /dev/null 2>&1
-
-# Create a new enclave and run the simulation
+# Create the new enclave and run the simulation
 jobs=$(cat config/${wakurtosis_config_file} | jq -r ".kurtosis.jobs")
 
 echo -e "\nSetting up the enclave: $enclave_name"
@@ -54,37 +83,36 @@ wls_service_name=$(kurtosis --cli-log-level $loglevel enclave inspect $enclave_n
 # kurtosis service logs $enclave_name $wls_service_name
 echo -e "\n--> To see simulation logs run: kurtosis service logs $enclave_name $wls_service_name <--"
 
-# Extract the Grafana host:port
-# TODO: needs fixing
-grafana_host=$(kurtosis --cli-log-level $loglevel enclave inspect $enclave_name | grep grafana | awk '{print $6}')
+
+# Fetch the Grafana address & port
+grafana_host=$(kurtosis enclave inspect $enclave_name | grep grafana | awk '{print $6}')
 echo -e "\n--> Statistics in Grafana server at http://$grafana_host/ <--"
 
-# echo "Output of kurtosis run command written in kurtosisrun_log.txt"
+echo "Output of kurtosis run command written in kurtosisrun_log.txt"
 
 ### Wait for WLS to finish
 
 # Get the container prefix/uffix for the WLS service
-enclave_prefix="$(kurtosis --cli-log-level $loglevel  enclave inspect --full-uuids $enclave_name | grep UUID: | awk '{print $2}')"
-cid_suffix="$(kurtosis --cli-log-level $loglevel  enclave inspect --full-uuids $enclave_name | grep $wls_service_name | cut -f 1 -d ' ')"
+service_name="$(kurtosis --cli-log-level $loglevel  enclave inspect $enclave_name | grep $wls_service_name | awk '{print $2}')"
+service_uuid="$(kurtosis --cli-log-level $loglevel  enclave inspect --full-uuids $enclave_name | grep $wls_service_name | awk '{print $1}')"
 
 # Construct the fully qualified container name that kurtosis has created
-cid="$enclave_prefix--user-service--$cid_suffix"
+cid="$service_name--$service_uuid"
 
 # Wait for the container to halt; this will block
-echo "Waiting for simulation to finish ..."
+echo -e "Waiting for simulation to finish ..."
 status_code="$(docker container wait $cid)"
 
+### Logs
+kurtosis enclave dump ${enclave_name} ${enclave_name}_logs > /dev/null 2>&1
+echo -e "Simulation ended with code $status_code Results in ./${enclave_name}_logs"
+
 END2=$(date +%s)
 DIFF2=$(( $END2 - $END1 ))
 
 echo "Simulation took $DIFF1 + $DIFF2 = $(( $END2 - $START)) secs"
 # Copy simulation results
-#docker cp "$cid:/wls/summary.json" "./"
-#echo "Simulation ended with code $status_code Results in ./summary.json"
-
-# Stop and delete the enclave
-# kurtosis enclave stop $enclave_name > /dev/null 2>&1
-# kurtosis enclave rm -f $enclave_name > /dev/null 2>&1
-# echo "Enclave $enclave_name stopped and deleted."
+# docker cp "$cid:/wls/summary.json" "./${enclave_name}_logs" > /dev/null 2>&1
+docker cp "$cid:/wls/messages.json" "./${enclave_name}_logs"
 
 echo "Done."