From 32c1e299fda07d37a12f1f77f896d3e1874dfbeb Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Wed, 11 Dec 2024 18:13:32 +0100
Subject: [PATCH 01/30] Adding basic acceptance testing script

---
 .nextmv/benchmark.py | 63 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 .nextmv/benchmark.py

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
new file mode 100644
index 0000000..0af5035
--- /dev/null
+++ b/.nextmv/benchmark.py
@@ -0,0 +1,63 @@
+import os
+from datetime import datetime, timezone
+
+from nextmv import cloud
+
+APP_ID = "nextroute-bench"
+API_KEY = os.environ["NEXTMV_API_KEY"]
+TAG = os.getenv("TAG", "untagged")
+
+METRICS = [
+    cloud.Metric(
+        field="result.value",
+        metric_type=cloud.MetricType.direct_comparison,
+        params=cloud.MetricParams(
+            tolerance=cloud.MetricTolerance(
+                value=0.05,
+                type=cloud.ToleranceType.relative,
+            ),
+            operator=cloud.Comparison.less_than_or_equal_to,
+        ),
+        statistic=cloud.StatisticType.mean,
+    )
+]
+
+
+def run_acceptance_test() -> cloud.AcceptanceTest:
+    client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
+    app = cloud.Application(client=client, id=APP_ID)
+    ts = (
+        datetime.now(timezone.utc)
+        .replace(microsecond=0)
+        .isoformat()
+        .replace("+00:00", "Z")
+        .replace(":", "")
+        .replace("-", "")
+    )
+    id = f"auto-{TAG}-{ts}"
+    print(f"Running acceptance test with ID: {id}")
+    print("Waiting for the test to complete...")
+    result = app.new_acceptance_test_with_result(
+        candidate_instance_id="candidate",
+        baseline_instance_id="baseline",
+        id=id,
+        metrics=METRICS,
+        name=f"Auto-test {TAG}",
+        description=f"Automated test for {TAG}",
+        input_set_id="nextroute-bench-v20",
+        polling_options=cloud.PollingOptions(
+            max_duration=600,  # 10 minutes
+            max_tries=1000,  # basically forever - we'll stop by duration
+        ),
+    )
+    passed = "passed" if result and result.results and result.results.passed else "unknown"
+    print(f"Acceptance test completed with status: {passed}")
+    return result
+
+
+def main():
+    run_acceptance_test()
+
+
+if __name__ == "__main__":
+    main()

From 63127915d77957ef8b5ccd9acf2899e1b8803b63 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Sun, 15 Dec 2024 04:21:19 +0100
Subject: [PATCH 02/30] Preparing test entrypoint for benchmarking (app.yaml,
 input embedded options)

---
 .nextmv/benchmark.py | 14 ++++++++++++++
 cmd/app.yaml         | 10 ++++++++++
 cmd/main.go          | 41 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+)
 create mode 100644 cmd/app.yaml

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 0af5035..3886379 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -1,3 +1,12 @@
+# Description:
+# This script does the following:
+# - Make sure the working directory is clean.
+# - Pushes a new version of the app (if it does not already exist; uses git sha as version).
+# - Updates the candidate instance to use the new version.
+# - Runs an acceptance test between the candidate and baseline instances.
+# - Waits for the test to complete.
+# - Posts the result to Slack (if requested).
+
 import os
 from datetime import datetime, timezone
 
@@ -23,6 +32,11 @@
 ]
 
 
+def check_clean_working_directory():
+    if os.system("git diff --quiet") != 0 or os.system("git diff --cached --quiet") != 0:
+        raise Exception("Working directory is not clean")
+
+
 def run_acceptance_test() -> cloud.AcceptanceTest:
     client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
     app = cloud.Application(client=client, id=APP_ID)
diff --git a/cmd/app.yaml b/cmd/app.yaml
new file mode 100644
index 0000000..297dff9
--- /dev/null
+++ b/cmd/app.yaml
@@ -0,0 +1,10 @@
+# This manifest holds the information the app needs to run on the Nextmv Cloud.
+type: go
+runtime: ghcr.io/nextmv-io/runtime/default:latest
+build:
+  command: go build -o main .
+  environment:
+    GOOS: linux
+    GOARCH: arm64
+files:
+  - main
diff --git a/cmd/main.go b/cmd/main.go
index e489b6c..45bf878 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -5,10 +5,12 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/nextmv-io/nextroute"
 	"github.com/nextmv-io/nextroute/check"
@@ -39,31 +41,70 @@ type options struct {
 	Check  check.Options                  `json:"check,omitempty"`
 }
 
+type customOptions struct {
+	MaxDuration *float64 `json:"max_duration,omitempty"`
+}
+
+// applyCustomOptions applies the extended custom options from the input to the
+// actual options.
+func applyCustomOptions(opts options, customOpts any) (options, error) {
+	jOpts, err := json.Marshal(customOpts)
+	if err != nil {
+		return opts, err
+	}
+	var custom customOptions
+	err = json.Unmarshal(jOpts, &custom)
+	if err != nil {
+		return opts, err
+	}
+	if custom.MaxDuration != nil {
+		opts.Solve.Duration = time.Duration(*custom.MaxDuration * float64(time.Second))
+	}
+	return opts, nil
+}
+
 func solver(
 	ctx context.Context,
 	input schema.Input,
 	options options,
 ) (runSchema.Output, error) {
+	// Apply input embedded options, if any. This is used internally for
+	// benchmarking and testing.
+	if input.Options != nil {
+		opts, err := applyCustomOptions(options, input.Options)
+		if err != nil {
+			return runSchema.Output{}, err
+		}
+		options = opts
+	}
+
+	// Create the model from the input and options.
 	model, err := factory.NewModel(input, options.Model)
 	if err != nil {
 		return runSchema.Output{}, err
 	}
 
+	// Create the solver from the model.
 	solver, err := nextroute.NewParallelSolver(model)
 	if err != nil {
 		return runSchema.Output{}, err
 	}
 
+	// Solve the model.
 	solutions, err := solver.Solve(ctx, options.Solve)
 	if err != nil {
 		return runSchema.Output{}, err
 	}
 
+	// Get the last solution.
+	// This call is blocking until the solver terminates. Alternatively,
+	// solutions can be ranged over (see All() method).
 	last, err := solutions.Last()
 	if err != nil {
 		return runSchema.Output{}, err
 	}
 
+	// Process the solution for output.
 	output, err := check.Format(
 		ctx,
 		options,

From 937be9164983db88412ab62c7e9f537e7fd285f7 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Sun, 15 Dec 2024 04:48:05 +0100
Subject: [PATCH 03/30] Completing auto benchmark

---
 .nextmv/benchmark.py | 96 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 79 insertions(+), 17 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 3886379..16ae34a 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -8,13 +8,14 @@
 # - Posts the result to Slack (if requested).
 
 import os
+import subprocess
 from datetime import datetime, timezone
 
 from nextmv import cloud
 
 APP_ID = "nextroute-bench"
 API_KEY = os.environ["NEXTMV_API_KEY"]
-TAG = os.getenv("TAG", "untagged")
+
 
 METRICS = [
     cloud.Metric(
@@ -32,23 +33,67 @@
 ]
 
 
-def check_clean_working_directory():
+def ensure_clean_working_directory():
+    """
+    Ensure the working directory is clean by throwing an exception if it is not.
+    """
     if os.system("git diff --quiet") != 0 or os.system("git diff --cached --quiet") != 0:
         raise Exception("Working directory is not clean")
 
 
-def run_acceptance_test() -> cloud.AcceptanceTest:
-    client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
-    app = cloud.Application(client=client, id=APP_ID)
-    ts = (
-        datetime.now(timezone.utc)
-        .replace(microsecond=0)
-        .isoformat()
-        .replace("+00:00", "Z")
-        .replace(":", "")
-        .replace("-", "")
+def get_tag(app: cloud.Application) -> str:
+    """
+    Get the tag for the new version.
+    If the version already exists, we append a timestamp to the tag.
+    """
+    # If we don't have a tag, we generate one based on the git sha and a timestamp
+    git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()[0:8]
+    # If the version already exists, we append a timestamp to the tag
+    exists = False
+    try:
+        app.version(git_sha)
+        exists = True
+    except Exception:
+        pass
+    if exists:
+        ts = (
+            datetime.now(timezone.utc)
+            .replace(microsecond=0)
+            .isoformat()
+            .replace("+00:00", "Z")
+            .replace(":", "")
+            .replace("-", "")
+        )
+        return f"{git_sha}-{ts}"
+    # Otherwise, we just use the git sha
+    return git_sha
+
+
+def push_new_version(app: cloud.Application, tag: str) -> None:
+    """
+    Push a new version of the app and update the candidate instance to use it.
+    """
+    app.push(app_dir=".")
+    version_id = f"auto-{tag}"
+    app.new_version(
+        id=version_id,
+        name=f"Auto version {tag}",
+        description=f"Automatically generated version {tag}",
+    )
+    app.update_instance(
+        id="candidate",
+        version_id=version_id,
     )
-    id = f"auto-{TAG}-{ts}"
+
+
+def run_acceptance_test(
+    app: cloud.Application,
+    tag: str,
+) -> cloud.AcceptanceTest:
+    """
+    Run an acceptance test between the candidate and baseline instances.
+    """
+    id = f"auto-{tag}"
     print(f"Running acceptance test with ID: {id}")
     print("Waiting for the test to complete...")
     result = app.new_acceptance_test_with_result(
@@ -56,21 +101,38 @@ def run_acceptance_test() -> cloud.AcceptanceTest:
         baseline_instance_id="baseline",
         id=id,
         metrics=METRICS,
-        name=f"Auto-test {TAG}",
-        description=f"Automated test for {TAG}",
+        name=f"Auto-test {tag}",
+        description=f"Automated test for {tag}",
         input_set_id="nextroute-bench-v20",
         polling_options=cloud.PollingOptions(
             max_duration=600,  # 10 minutes
             max_tries=1000,  # basically forever - we'll stop by duration
         ),
     )
-    passed = "passed" if result and result.results and result.results.passed else "unknown"
+    passed = "unknown"
+    if result and result.results:
+        passed = "passed" if result.results.passed else "failed"
     print(f"Acceptance test completed with status: {passed}")
     return result
 
 
 def main():
-    run_acceptance_test()
+    """
+    Main function that runs the benchmark.
+    """
+    # Change to the directory of the app (sibling directory of this script)
+    os.chdir(os.path.join(os.path.dirname(__file__), "..", "cmd"))
+
+    ensure_clean_working_directory()
+
+    client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
+    app = cloud.Application(client=client, id=APP_ID)
+
+    tag = get_tag(app)
+
+    push_new_version(app, tag)
+
+    run_acceptance_test(app, tag)
 
 
 if __name__ == "__main__":

From 5c6ebfa43068c704bc260871e787566b5841f274 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 06:11:54 +0100
Subject: [PATCH 04/30] Fixing version tag

---
 .nextmv/benchmark.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 16ae34a..36f2ea0 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -64,9 +64,9 @@ def get_tag(app: cloud.Application) -> str:
             .replace(":", "")
             .replace("-", "")
         )
-        return f"{git_sha}-{ts}"
+        return f"auto-{git_sha}-{ts}"
     # Otherwise, we just use the git sha
-    return git_sha
+    return f"auto-{git_sha}"
 
 
 def push_new_version(app: cloud.Application, tag: str) -> None:
@@ -74,15 +74,14 @@ def push_new_version(app: cloud.Application, tag: str) -> None:
     Push a new version of the app and update the candidate instance to use it.
     """
     app.push(app_dir=".")
-    version_id = f"auto-{tag}"
     app.new_version(
-        id=version_id,
+        id=tag,
         name=f"Auto version {tag}",
         description=f"Automatically generated version {tag}",
     )
     app.update_instance(
         id="candidate",
-        version_id=version_id,
+        version_id=tag,
     )
 
 

From ca1ce8efa58c2f46d783faae5b510d31b7866230 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 06:18:04 +0100
Subject: [PATCH 05/30] Fixing version test

---
 .nextmv/benchmark.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 36f2ea0..3394c28 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -48,6 +48,7 @@ def get_tag(app: cloud.Application) -> str:
     """
     # If we don't have a tag, we generate one based on the git sha and a timestamp
     git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()[0:8]
+    version_id = f"auto-{git_sha}"
     # If the version already exists, we append a timestamp to the tag
     exists = False
     try:
@@ -64,9 +65,9 @@ def get_tag(app: cloud.Application) -> str:
             .replace(":", "")
             .replace("-", "")
         )
-        return f"auto-{git_sha}-{ts}"
+        return f"{version_id}-{ts}"
     # Otherwise, we just use the git sha
-    return f"auto-{git_sha}"
+    return version_id
 
 
 def push_new_version(app: cloud.Application, tag: str) -> None:

From df58314a9deb969a5766940ab3ee808987e05f88 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 06:19:04 +0100
Subject: [PATCH 06/30] Cleaning code

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 3394c28..4b1b17e 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -65,7 +65,7 @@ def get_tag(app: cloud.Application) -> str:
             .replace(":", "")
             .replace("-", "")
         )
-        return f"{version_id}-{ts}"
+        version_id = f"{version_id}-{ts}"
     # Otherwise, we just use the git sha
     return version_id
 

From 1adc5f901a564b9479264807276ce53cbbe5983b Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 07:25:55 +0100
Subject: [PATCH 07/30] Fixing version test

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 4b1b17e..b8e674c 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -52,7 +52,7 @@ def get_tag(app: cloud.Application) -> str:
     # If the version already exists, we append a timestamp to the tag
     exists = False
     try:
-        app.version(git_sha)
+        app.version(version_id)
         exists = True
     except Exception:
         pass

From f2fab6ad24b4bc21adc64c8ead80aea74d3cc77e Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 07:38:21 +0100
Subject: [PATCH 08/30] Reorganizing prints

---
 .nextmv/benchmark.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index b8e674c..6f00b9f 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -88,13 +88,12 @@ def push_new_version(app: cloud.Application, tag: str) -> None:
 
 def run_acceptance_test(
     app: cloud.Application,
+    id: str,
     tag: str,
 ) -> cloud.AcceptanceTest:
     """
     Run an acceptance test between the candidate and baseline instances.
     """
-    id = f"auto-{tag}"
-    print(f"Running acceptance test with ID: {id}")
     print("Waiting for the test to complete...")
     result = app.new_acceptance_test_with_result(
         candidate_instance_id="candidate",
@@ -109,10 +108,6 @@ def run_acceptance_test(
             max_tries=1000,  # basically forever - we'll stop by duration
         ),
     )
-    passed = "unknown"
-    if result and result.results:
-        passed = "passed" if result.results.passed else "failed"
-    print(f"Acceptance test completed with status: {passed}")
     return result
 
 
@@ -123,6 +118,7 @@ def main():
     # Change to the directory of the app (sibling directory of this script)
     os.chdir(os.path.join(os.path.dirname(__file__), "..", "cmd"))
 
+    print("Making sure the working directory is clean...")
     ensure_clean_working_directory()
 
     client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
@@ -130,9 +126,17 @@ def main():
 
     tag = get_tag(app)
 
+    print(f"Pushing new version with tag: {tag}")
     push_new_version(app, tag)
 
-    run_acceptance_test(app, tag)
+    id = f"auto-{tag}"
+    print(f"Running acceptance test with ID: {id}")
+    print("Waiting for it to complete...")
+    result = run_acceptance_test(app, tag)
+    passed = "unknown"
+    if result and result.results:
+        passed = "passed" if result.results.passed else "failed"
+    print(f"Acceptance test completed with status: {passed}")
 
 
 if __name__ == "__main__":

From dca29b0b9a33c69cf0f862a36c3c670bab48f5c9 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 07:39:46 +0100
Subject: [PATCH 09/30] Adding required name parameter

---
 .nextmv/benchmark.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 6f00b9f..1e41d57 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -80,9 +80,11 @@ def push_new_version(app: cloud.Application, tag: str) -> None:
         name=f"Auto version {tag}",
         description=f"Automatically generated version {tag}",
     )
+    instance = app.instance("candidate")
     app.update_instance(
         id="candidate",
         version_id=tag,
+        name=instance.name,  # Name is required, but we don't want to change it
     )
 
 

From 46f046369d17b55f25e5feb272db8f7424eeb6c7 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 07:43:10 +0100
Subject: [PATCH 10/30] Adding missing param

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 1e41d57..f65e3fa 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -134,7 +134,7 @@ def main():
     id = f"auto-{tag}"
     print(f"Running acceptance test with ID: {id}")
     print("Waiting for it to complete...")
-    result = run_acceptance_test(app, tag)
+    result = run_acceptance_test(app, id, tag)
     passed = "unknown"
     if result and result.results:
         passed = "passed" if result.results.passed else "failed"

From 6e366f1ae0bcbb8e4e8e9e81ad47cac2f8cf3109 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 08:10:16 +0100
Subject: [PATCH 11/30] Removing obsolete print

---
 .nextmv/benchmark.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index f65e3fa..c9796c5 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -96,7 +96,6 @@ def run_acceptance_test(
     """
     Run an acceptance test between the candidate and baseline instances.
     """
-    print("Waiting for the test to complete...")
     result = app.new_acceptance_test_with_result(
         candidate_instance_id="candidate",
         baseline_instance_id="baseline",

From 183e41a68a273220eb026e01074c193dbff78b68 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 16:21:24 +0100
Subject: [PATCH 12/30] Adding auto benchmark workflow

---
 .github/workflows/auto-benchmark.yml | 31 ++++++++++++++++++++++++++++
 .nextmv/benchmark.py                 | 21 ++++++++++++++++++-
 .nextmv/benchmark.requirements.txt   |  2 ++
 3 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/auto-benchmark.yml
 create mode 100644 .nextmv/benchmark.requirements.txt

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
new file mode 100644
index 0000000..54143b7
--- /dev/null
+++ b/.github/workflows/auto-benchmark.yml
@@ -0,0 +1,31 @@
+name: auto benchmark
+on: [push]
+
+env:
+  GO_VERSION: 1.23
+  PYTHON_VERSION: 3.12
+
+jobs:
+  python-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: git clone
+        uses: actions/checkout@v4
+
+      - name: set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r .nextmv/benchmark.requirements.txt
+
+      - name: run acceptance test
+        run: python .nextmv/benchmark.py
diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index c9796c5..e38c507 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -11,10 +11,13 @@
 import subprocess
 from datetime import datetime, timezone
 
+import requests
 from nextmv import cloud
 
 APP_ID = "nextroute-bench"
-API_KEY = os.environ["NEXTMV_API_KEY"]
+API_KEY = os.environ["BENCHMARK_API_KEY_PROD"]
+SLACK_WEBHOOK = os.getenv("SLACK_URL_DEV_SCIENCE", None)
+BRANCH_NAME = os.getenv("BRANCH_NAME", None)
 
 
 METRICS = [
@@ -139,6 +142,22 @@ def main():
         passed = "passed" if result.results.passed else "failed"
     print(f"Acceptance test completed with status: {passed}")
 
+    if SLACK_WEBHOOK and BRANCH_NAME == "develop":
+        print("Posting to Slack...")
+        response = requests.post(
+            SLACK_WEBHOOK,
+            json={
+                "text": f"Acceptance test {result.id} completed with status: {passed}",
+            },
+        )
+
+        if response.status_code != 200:
+            print(f"Failed to send notification to Slack: {response.text}")
+        else:
+            print("Notification sent to Slack")
+
+    print("Done")
+
 
 if __name__ == "__main__":
     main()
diff --git a/.nextmv/benchmark.requirements.txt b/.nextmv/benchmark.requirements.txt
new file mode 100644
index 0000000..52921fa
--- /dev/null
+++ b/.nextmv/benchmark.requirements.txt
@@ -0,0 +1,2 @@
+nextmv>=0.14.1
+requests>=2.32.3

From 47091dd4ce0723a44e56807ac81764af0be067c8 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 16:21:57 +0100
Subject: [PATCH 13/30] Adding branch name for pacing slack notifications

---
 .github/workflows/auto-benchmark.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index 54143b7..c50860e 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -28,4 +28,6 @@ jobs:
           pip install -r .nextmv/benchmark.requirements.txt
 
       - name: run acceptance test
-        run: python .nextmv/benchmark.py
+        run: |
+          export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
+          python .nextmv/benchmark.py

From 738a6c25260d5e62d42138d4c2d44ff85a4fbd6d Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Mon, 16 Dec 2024 16:25:01 +0100
Subject: [PATCH 14/30] Pass secrets

---
 .github/workflows/auto-benchmark.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index c50860e..6c84d3b 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -30,4 +30,6 @@ jobs:
       - name: run acceptance test
         run: |
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
+          export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}
+          export SLACK_URL_DEV_SCIENCE=${{ secrets.SLACK_URL_DEV_SCIENCE }}
           python .nextmv/benchmark.py

From 71d1a2bbc509775177209804e2c6506606c5c4a1 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 03:09:02 +0100
Subject: [PATCH 15/30] Using output summary, improved slack message

---
 .nextmv/benchmark.py | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index e38c507..cf4a148 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -17,6 +17,7 @@
 APP_ID = "nextroute-bench"
 API_KEY = os.environ["BENCHMARK_API_KEY_PROD"]
 SLACK_WEBHOOK = os.getenv("SLACK_URL_DEV_SCIENCE", None)
+ACCOUNT_ID = os.getenv("BENCHMARK_ACCOUNT_ID", None)
 BRANCH_NAME = os.getenv("BRANCH_NAME", None)
 
 
@@ -115,6 +116,26 @@ def run_acceptance_test(
     return result
 
 
+def create_test_url(result_id: str) -> str:
+    """
+    Create a URL to the acceptance test result.
+    """
+    if ACCOUNT_ID:
+        return f"https://cloud.nextmv.io/acc/{ACCOUNT_ID}/app/nextroute-bench/experiments/acceptance/{result_id}"
+    return "unavailable"
+
+
+def write_to_summary(content):
+    """Appends content to the GitHub Actions step summary (if available)."""
+    summary_file = os.getenv("GITHUB_STEP_SUMMARY")
+    if not summary_file:
+        return
+
+    # Write content to the summary file
+    with open(summary_file, "a") as f:
+        f.write(content + "\n")
+
+
 def main():
     """
     Main function that runs the benchmark.
@@ -134,6 +155,11 @@ def main():
     push_new_version(app, tag)
 
     id = f"auto-{tag}"
+    write_to_summary("# Acceptance Test Report")
+    write_to_summary("")
+    write_to_summary(f"ID: {id}")
+    write_to_summary(f"Link: [link]({create_test_url(id)})")
+
     print(f"Running acceptance test with ID: {id}")
     print("Waiting for it to complete...")
     result = run_acceptance_test(app, id, tag)
@@ -147,7 +173,8 @@ def main():
         response = requests.post(
             SLACK_WEBHOOK,
             json={
-                "text": f"Acceptance test {result.id} completed with status: {passed}",
+                "text": f"Acceptance test {result and result.id} completed with status: {passed}"
+                + f" (<{create_test_url(result and result.id)}|View results>)",
             },
         )
 
@@ -156,6 +183,17 @@ def main():
         else:
             print("Notification sent to Slack")
 
+    write_to_summary("")
+    write_to_summary(f"Result: {passed}")
+    if result and result.results:
+        if result.results.error:
+            write_to_summary(f"Error: {result.results.error}")
+        else:
+            write_to_summary("Metrics:")
+            write_to_summary("")
+            for metric in result.results.metric_results:
+                write_to_summary(f"- {metric.metric.field}: {metric.passed}")
+
     print("Done")
 
 

From c68feb2f47ff546dc4dd0e62ed2f0173446e45f5 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 03:10:36 +0100
Subject: [PATCH 16/30] More specific slack message

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index cf4a148..0f18d67 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -173,7 +173,7 @@ def main():
         response = requests.post(
             SLACK_WEBHOOK,
             json={
-                "text": f"Acceptance test {result and result.id} completed with status: {passed}"
+                "text": f"nextroute acceptance test {result and result.id} completed with status: {passed}"
                 + f" (<{create_test_url(result and result.id)}|View results>)",
             },
         )

From dd6b5237254323563a42e4aa1dc5602f607bb348 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 03:14:03 +0100
Subject: [PATCH 17/30] Bump nextmv version

---
 .nextmv/benchmark.requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.requirements.txt b/.nextmv/benchmark.requirements.txt
index 52921fa..dba0196 100644
--- a/.nextmv/benchmark.requirements.txt
+++ b/.nextmv/benchmark.requirements.txt
@@ -1,2 +1,2 @@
-nextmv>=0.14.1
+nextmv>=v0.14.2-dev.0
 requests>=2.32.3

From a5fe09b95df13c093533c79b3506c9f7624f65d3 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 03:18:45 +0100
Subject: [PATCH 18/30] Fixing api key

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 0f18d67..3cc0590 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -146,7 +146,7 @@ def main():
     print("Making sure the working directory is clean...")
     ensure_clean_working_directory()
 
-    client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
+    client = cloud.Client(api_key=API_KEY)
     app = cloud.Application(client=client, id=APP_ID)
 
     tag = get_tag(app)

From 5bc5cadfbd545b99b4bb61b0afd7603a88d9e6a9 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 03:29:34 +0100
Subject: [PATCH 19/30] Forward account id

---
 .github/workflows/auto-benchmark.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index 6c84d3b..bee1659 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -32,4 +32,5 @@ jobs:
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
           export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}
           export SLACK_URL_DEV_SCIENCE=${{ secrets.SLACK_URL_DEV_SCIENCE }}
+          export BENCHMARK_ACCOUNT_ID=${{ secrets.BENCHMARK_ACCOUNT_ID }}
           python .nextmv/benchmark.py

From 92bc04273ee561e47715d233fff233aa753e25c8 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 15:29:19 +0100
Subject: [PATCH 20/30] Moving account id to env vars

---
 .github/workflows/auto-benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index bee1659..b2ed50e 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -32,5 +32,5 @@ jobs:
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
           export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}
           export SLACK_URL_DEV_SCIENCE=${{ secrets.SLACK_URL_DEV_SCIENCE }}
-          export BENCHMARK_ACCOUNT_ID=${{ secrets.BENCHMARK_ACCOUNT_ID }}
+          export BENCHMARK_ACCOUNT_ID=${{ env.BENCHMARK_ACCOUNT_ID }}
           python .nextmv/benchmark.py

From bd8cf40b5afe34663e6d252d72bbff2ea92534e5 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Tue, 17 Dec 2024 15:30:17 +0100
Subject: [PATCH 21/30] Renaming job

---
 .github/workflows/auto-benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index b2ed50e..b9216b3 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -6,7 +6,7 @@ env:
   PYTHON_VERSION: 3.12
 
 jobs:
-  python-test:
+  auto-benchmark:
     runs-on: ubuntu-latest
     steps:
       - name: git clone

From 037545be1903925d96c91cfdf8bdcc4901c4210c Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Wed, 18 Dec 2024 00:52:15 +0100
Subject: [PATCH 22/30] Fixing use of variable

---
 .github/workflows/auto-benchmark.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index b9216b3..cd3cf8a 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -8,6 +8,8 @@ env:
 jobs:
   auto-benchmark:
     runs-on: ubuntu-latest
+    env:
+      BENCHMARK_ACCOUNT_ID: ${{ vars.BENCHMARK_ACCOUNT_ID }}
     steps:
       - name: git clone
         uses: actions/checkout@v4
@@ -32,5 +34,4 @@ jobs:
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
           export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}
           export SLACK_URL_DEV_SCIENCE=${{ secrets.SLACK_URL_DEV_SCIENCE }}
-          export BENCHMARK_ACCOUNT_ID=${{ env.BENCHMARK_ACCOUNT_ID }}
           python .nextmv/benchmark.py

From 3c4a84a2a5d25b323e3d0af95f5a6e7c67860e00 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Wed, 18 Dec 2024 01:07:19 +0100
Subject: [PATCH 23/30] Use official nextmv-py release

---
 .nextmv/benchmark.requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.requirements.txt b/.nextmv/benchmark.requirements.txt
index dba0196..f517d0c 100644
--- a/.nextmv/benchmark.requirements.txt
+++ b/.nextmv/benchmark.requirements.txt
@@ -1,2 +1,2 @@
-nextmv>=v0.14.2-dev.0
+nextmv>=v0.14.2
 requests>=2.32.3

From 6990c5a30bc630291d30014d6f101617e78644a7 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Wed, 18 Dec 2024 01:15:45 +0100
Subject: [PATCH 24/30] Upgrade baseline when running on main branch

---
 .nextmv/benchmark.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 3cc0590..6346224 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -92,6 +92,18 @@ def push_new_version(app: cloud.Application, tag: str) -> None:
     )
 
 
+def upgrade_baseline(app: cloud.Application, version_id: str) -> None:
+    """
+    Upgrade the baseline instance to use the new version.
+    """
+    instance = app.instance("baseline")
+    app.update_instance(
+        id="baseline",
+        version_id=version_id,
+        name=instance.name,  # Name is required, but we don't want to change it
+    )
+
+
 def run_acceptance_test(
     app: cloud.Application,
     id: str,
@@ -194,6 +206,10 @@ def main():
             for metric in result.results.metric_results:
                 write_to_summary(f"- {metric.metric.field}: {metric.passed}")
 
+    if BRANCH_NAME == "develop":
+        print("Upgrading baseline instance to use the new version...")
+        upgrade_baseline(app, tag)
+
     print("Done")
 
 

From 87984e5c8869cb51e20b700b8a00c133e74458f0 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Wed, 18 Dec 2024 15:39:20 +0100
Subject: [PATCH 25/30] Move env var to step that needs it

---
 .github/workflows/auto-benchmark.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index cd3cf8a..89d6fad 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -8,8 +8,6 @@ env:
 jobs:
   auto-benchmark:
     runs-on: ubuntu-latest
-    env:
-      BENCHMARK_ACCOUNT_ID: ${{ vars.BENCHMARK_ACCOUNT_ID }}
     steps:
       - name: git clone
         uses: actions/checkout@v4
@@ -30,6 +28,8 @@ jobs:
           pip install -r .nextmv/benchmark.requirements.txt
 
       - name: run acceptance test
+        env:
+          BENCHMARK_ACCOUNT_ID: ${{ vars.BENCHMARK_ACCOUNT_ID }}
         run: |
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
           export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}

From 225677839551d4a8f439d7afe18d9b5a9ff1ca05 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Thu, 19 Dec 2024 11:16:15 +0100
Subject: [PATCH 26/30] Moving some env vars to GH env section

---
 .github/workflows/auto-benchmark.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/auto-benchmark.yml b/.github/workflows/auto-benchmark.yml
index 89d6fad..795876c 100644
--- a/.github/workflows/auto-benchmark.yml
+++ b/.github/workflows/auto-benchmark.yml
@@ -30,8 +30,8 @@ jobs:
       - name: run acceptance test
         env:
           BENCHMARK_ACCOUNT_ID: ${{ vars.BENCHMARK_ACCOUNT_ID }}
+          BENCHMARK_API_KEY_PROD: ${{ secrets.BENCHMARK_API_KEY_PROD }}
+          SLACK_URL_DEV_SCIENCE: ${{ secrets.SLACK_URL_DEV_SCIENCE }}
         run: |
           export BRANCH_NAME=$(echo $GITHUB_REF | awk -F'/' '{print $3}')
-          export BENCHMARK_API_KEY_PROD=${{ secrets.BENCHMARK_API_KEY_PROD }}
-          export SLACK_URL_DEV_SCIENCE=${{ secrets.SLACK_URL_DEV_SCIENCE }}
           python .nextmv/benchmark.py

From d54638fd4350e568c8e544025de63e69c886f444 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Thu, 19 Dec 2024 11:22:09 +0100
Subject: [PATCH 27/30] Using shifted geometric mean instead

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 6346224..58f5657 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -32,7 +32,7 @@
             ),
             operator=cloud.Comparison.less_than_or_equal_to,
         ),
-        statistic=cloud.StatisticType.mean,
+        statistic=cloud.StatisticType.shifted_geometric_mean,
     )
 ]
 

From 38a70a5dc0db0076182960ba76f98f83aab4c91a Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Thu, 19 Dec 2024 11:30:57 +0100
Subject: [PATCH 28/30] Fixing id and tag handling

---
 .nextmv/benchmark.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 58f5657..5aaae07 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -45,15 +45,15 @@ def ensure_clean_working_directory():
         raise Exception("Working directory is not clean")
 
 
-def get_tag(app: cloud.Application) -> str:
+def get_id(app: cloud.Application) -> tuple[str, str]:
     """
-    Get the tag for the new version.
-    If the version already exists, we append a timestamp to the tag.
+    Get the ID for the new version (and just the tag).
+    If the version already exists, we append a timestamp to the ID.
     """
-    # If we don't have a tag, we generate one based on the git sha and a timestamp
-    git_sha = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()[0:8]
-    version_id = f"auto-{git_sha}"
-    # If the version already exists, we append a timestamp to the tag
+    # Create ID based on git sha.
+    tag = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode().strip()[0:8]
+    version_id = f"auto-{tag}"
+    # If the version already exists, we append a timestamp to the ID.
     exists = False
     try:
         app.version(version_id)
@@ -70,8 +70,9 @@ def get_tag(app: cloud.Application) -> str:
             .replace("-", "")
         )
         version_id = f"{version_id}-{ts}"
-    # Otherwise, we just use the git sha
-    return version_id
+        tag = f"{tag}-{ts}"
+    # Otherwise, we just use the git sha.
+    return version_id, tag
 
 
 def push_new_version(app: cloud.Application, tag: str) -> None:
@@ -161,12 +162,11 @@ def main():
     client = cloud.Client(api_key=API_KEY)
     app = cloud.Application(client=client, id=APP_ID)
 
-    tag = get_tag(app)
+    id, tag = get_id(app)  # id is used as version and acceptance test ID
 
-    print(f"Pushing new version with tag: {tag}")
-    push_new_version(app, tag)
+    print(f"Pushing new version with ID: {id}")
+    push_new_version(app, id)
 
-    id = f"auto-{tag}"
     write_to_summary("# Acceptance Test Report")
     write_to_summary("")
     write_to_summary(f"ID: {id}")
@@ -208,7 +208,7 @@ def main():
 
     if BRANCH_NAME == "develop":
         print("Upgrading baseline instance to use the new version...")
-        upgrade_baseline(app, tag)
+        upgrade_baseline(app, id)
 
     print("Done")
 

From b6738762fe1c73a0429ccffc5787b1c5ab221e7c Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Thu, 19 Dec 2024 11:39:36 +0100
Subject: [PATCH 29/30] Pre-output url

---
 .nextmv/benchmark.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index 5aaae07..ee2f52c 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -134,7 +134,7 @@ def create_test_url(result_id: str) -> str:
     Create a URL to the acceptance test result.
     """
     if ACCOUNT_ID:
-        return f"https://cloud.nextmv.io/acc/{ACCOUNT_ID}/app/nextroute-bench/experiments/acceptance/{result_id}"
+        return f"https://cloud.nextmv.io/acc/{ACCOUNT_ID}/app/nextroute-bench/experiment/acceptance/{result_id}"
     return "unavailable"
 
 
@@ -170,7 +170,9 @@ def main():
     write_to_summary("# Acceptance Test Report")
     write_to_summary("")
     write_to_summary(f"ID: {id}")
-    write_to_summary(f"Link: [link]({create_test_url(id)})")
+    url = create_test_url(id)
+    write_to_summary(f"Link: [link]({url})")
+    print(f"::notice::Acceptance test URL: {url}")
 
     print(f"Running acceptance test with ID: {id}")
     print("Waiting for it to complete...")

From 632b196edf9d60d93c0dba99b37eb593f1bec663 Mon Sep 17 00:00:00 2001
From: Marius Merschformann <marius.merschformann@gmail.com>
Date: Thu, 19 Dec 2024 11:41:14 +0100
Subject: [PATCH 30/30] Flushing url

---
 .nextmv/benchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nextmv/benchmark.py b/.nextmv/benchmark.py
index ee2f52c..47fb9d3 100644
--- a/.nextmv/benchmark.py
+++ b/.nextmv/benchmark.py
@@ -172,7 +172,7 @@ def main():
     write_to_summary(f"ID: {id}")
     url = create_test_url(id)
     write_to_summary(f"Link: [link]({url})")
-    print(f"::notice::Acceptance test URL: {url}")
+    print(f"::notice::Acceptance test URL: {url}", flush=True)
 
     print(f"Running acceptance test with ID: {id}")
     print("Waiting for it to complete...")