Fix: Collect restart test log.

.github/workflows/slow_test.yml

@@ -100,18 +100,21 @@ jobs:
     
       - name: Run restart test
         if: ${{ !cancelled() && !failure() }}
+        id: run_restart_test
         run : |
           sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn"
           sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
           sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity"
+        #   sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_restart_test.py --infinity_path=cmake-build-release/src/infinity --slow=true"
 
       - name: Collect restart test output
         if: ${{ !cancelled() }} # always run this step even if previous steps failed
         # remove symbolic link
         # find all log file like [debug.log.*] in directory, and cat to stdout
         run: |
           sudo rm -f test/data/benchmark/enwiki-10w.csv
-          find . -name "restart_test.log.*" -exec cat {} \;
+          failure="${{ steps.run_restart_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_restart_log.py --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Destroy builder container
         if: always()  # always run this step even if previous steps failed

.github/workflows/tests.yml

@@ -74,6 +74,7 @@ jobs:
 
       - name: Run restart test
         if: ${{ !cancelled() && !failure() }}
+        id: run_restart_test
         run : sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity"
 
       - name: Collect restart test output
@@ -82,7 +83,8 @@ jobs:
         # find all log file like [debug.log.*] in directory, and cat to stdout
         run: |
           sudo rm -f test/data/benchmark/enwiki-10w.csv
-          find . -name "restart_test.log.*" -exec cat {} \;
+          failure="${{ steps.run_restart_test.outcome == 'failure'}}"
+          sudo python3 scripts/collect_restart_log.py --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Start infinity debug version
         if: ${{ !cancelled() && !failure() }}

scripts/collect_restart_log.py

@@ -0,0 +1,52 @@
+
+
+import argparse
+import os
+import random
+import re
+import shutil
+import string
+
+
+parser = argparse.ArgumentParser(description="Python restart test for infinity")
+parser.add_argument(
+    "--output_dir",
+    type=str,
+    required=True,
+    help="Path to the output directory",
+)
+parser.add_argument("--failure", type=str, required=True, help="If the test failured")
+args = parser.parse_args()
+
+output_dir = args.output_dir
+failure = args.failure == "true" or args.failure == "True"
+
+if not os.path.isdir(output_dir):
+    os.makedirs(output_dir)
+
+if failure:
+    random_name = "".join(random.choices(string.ascii_lowercase + string.digits, k=8))
+    print(f"Random log file name: {random_name}")
+
+show_lines = 1000
+
+
+log_files = []
+for root, dirs, files in os.walk("."):
+    for file in files:
+        match = re.match(r"restart_test\.log\.(\d+)", file)
+        if match:
+            log_files.append((os.path.join(root, file), int(match.group(1))))
+
+log_files = sorted(log_files, key=lambda x: x[1])
+print(f"Found log files: {log_files}")
+
+for log_file, i in log_files:
+    if failure:
+        shutil.copy(log_file, f"{output_dir}/{random_name}_{i}.log")
+    if i == len(log_files) - 1:
+        print(f"Last {show_lines} lines from {log_file}:")
+        with open(log_file, "r") as f:
+            lines = f.readlines()
+            for line in lines[-show_lines:]:
+                print(line.strip())

tools/run_restart_test.py

@@ -13,27 +13,45 @@
         type=str,
         default="./build/Debug/src/infinity",
     )
+    parser.add_argument(
+        "--slow",
+        type=bool,
+        default=False,
+    )
 
     args = parser.parse_args()
     infinity_path = args.infinity_path
+    slow = args.slow
 
     current_path = os.getcwd()
     python_test_dir = current_path + "/python"
 
-    process = subprocess.Popen(
-        [
-            python_executable,
-            "-m",
-            "pytest",
-            # "--capture=tee-sys",
-            f"{python_test_dir}/restart_test",
-            f"--infinity_path={infinity_path}",
-            "-x",
-            "-s",
-            "-m",
-            "not slow"
-        ]
-    )
+    if not slow:
+        process = subprocess.Popen(
+            [
+                python_executable,
+                "-m",
+                "pytest",
+                f"{python_test_dir}/restart_test",
+                f"--infinity_path={infinity_path}",
+                "-x",
+                "-s",
+                "-m",
+                "not slow",
+            ]
+        )
+    else:
+        process = subprocess.Popen(
+            [
+                python_executable,
+                "-m",
+                "pytest",
+                f"{python_test_dir}/restart_test",
+                f"--infinity_path={infinity_path}",
+                "-x",
+                "-s",
+            ]
+        )
     process.wait()
     if process.returncode != 0:
-        raise Exception(f"An error occurred: {process.stderr}")
+        raise Exception(f"An error occurred: {process.stderr}")