Fix bug (#1913)

### What problem does this PR solve? 1. Fix: invalidate fulltext cache before cleanup. 2. Refactor base entry. 3. Fix buffer ptr. 4. Fix miss cleanup after restart because of delta log merge. 5. Add cleanup testcase. 6. Fix: move manual cleanup task(and its delta checkpoint task) to background_process. So manual cleanup can cleanup all deleted content before called. 7. Fix: cleanup task is not async task. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] Refactoring - [x] Test cases
infiniflow · Sep 25, 2024 · 48b0408 · 48b0408
1 parent 5369cbf
commit 48b0408
Show file tree

Hide file tree

Showing 42 changed files with 340 additions and 352 deletions.
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -57,15 +57,16 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: |
           # Run a command in the background
-          # store the log in "/var/infinity/log/infinity.log" and set log level "trace"
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_continuous_conf.toml > release.log 2>&1" &
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_continuous_conf.toml > release.log 2> release_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version continously
         if: ${{ !cancelled() && !failure() }}
+        id: run_py_tests
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel_continuous.py" && sleep 1s
 
       - name: Stop infinity release
         if: ${{ !cancelled() }}
+        id: stop_py_tests
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
@@ -76,9 +77,10 @@ jobs:
           fi
 
       - name: Collect infinity release output
-        if: ${{ !cancelled()  && failure() }} # always run this step even if previous steps failed
+        if: ${{ !cancelled() }} # always run this step even if previous steps failed
         run: |
-          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log
+          failure="${{ steps.run_py_tests.outcome == 'failure' || steps.stop_py_tests.outcome == 'failure' }}"
+          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -88,14 +88,16 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: |
           # Run a command in the background
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > debug.log 2>&1" &
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > debug.log 2> debug_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test debug version
         if: ${{ !cancelled() && !failure() }}
+        id: run_tests_debug
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pytest_parallel.py" && sleep 1s
 
       - name: Stop infinity pysdk & http_api & sqllogictest debug
         if: ${{ !cancelled() }}
+        id: stop_tests_debug
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
@@ -106,23 +108,26 @@ jobs:
           fi
 
       - name: Collect infinity debug output
-        if: ${{ !cancelled() && failure() }}
+        if: ${{ !cancelled() }}
         # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected.
         run: |
-          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=debug.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log
+          failure="${{ steps.run_tests_debug.outcome == 'failure' || steps.stop_tests_debug.outcome == 'failure' }}"
+          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=debug.log --stderror_path=debug_error.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Start infinity debug version with vfs off
         if: ${{ !cancelled() && !failure() }}
         run: |
           # Run a command in the background
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_vfs_off.toml > vfs_debug.log 2>&1" &
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_vfs_off.toml > vfs_debug.log 2> vfs_debug_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test debug version
         if: ${{ !cancelled() && !failure() }}
+        id: run_tests_debug_vfs_off
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pytest_parallel.py" && sleep 1s
 
       - name: Stop infinity pysdk & http_api & sqllogictest debug
         if: ${{ !cancelled() }}
+        id: stop_tests_debug_vfs_off
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
@@ -133,10 +138,11 @@ jobs:
           fi
 
       - name: Collect infinity debug output
-        if: ${{ !cancelled() && failure() }}
+        if: ${{ !cancelled() }}
         # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected.
         run: |
-          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=vfs_debug.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log
+          failure="${{ steps.run_tests_debug_vfs_off.outcome == 'failure' || steps.stop_tests_debug_vfs_off.outcome == 'failure' }}"
+          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=vfs_debug.log --stderror_path=vfs_debug_error.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Unit test debug version
         if: ${{ !cancelled() && !failure() }}
@@ -229,14 +235,16 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: |
           # Run a command in the background
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > release.log 2>&1" &
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > release.log 2> release_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version
         if: ${{ !cancelled() && !failure() }}
+        id: run_tests_release
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel.py" && sleep 1s
 
       - name: Stop infinity release
         if: ${{ !cancelled() }}
+        id: stop_tests_release
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
@@ -247,23 +255,26 @@ jobs:
           fi
 
       - name: Collect infinity release output
-        if: ${{ !cancelled() && failure() }} # always run this step even if previous steps failed
+        if: ${{ !cancelled() }} # always run this step even if previous steps failed
         # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected. 
         run: |
-          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log
+          failure="${{ steps.run_tests_release.outcome == 'failure' || steps.stop_tests_release.outcome == 'failure' }}"
+          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
       - name: Start infinity release version with vfs off
         if: ${{ !cancelled() && !failure() }}
         run: |
           # Run a command in the background
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_vfs_off.toml > vfs_release.log 2>&1" &
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_infinity_vfs_off.toml > vfs_release.log 2> vfs_release_error.log" &
 
       - name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version
-        if: ${{ !cancelled() && !failure() }}
+        if: ${{ !cancelled() }}
+        id: run_tests_release_vfs_off
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel.py" && sleep 1s
 
       - name: Stop infinity release
         if: ${{ !cancelled() }}
+        id: stop_tests_release_vfs_off
         run: |
           pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo)
           sudo chmod +x scripts/timeout_kill.sh
@@ -274,10 +285,11 @@ jobs:
           fi
 
       - name: Collect infinity release output
-        if: ${{ !cancelled() && failure() }} # always run this step even if previous steps failed
+        if: ${{ !cancelled() }} # always run this step even if previous steps failed
         # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected.
         run: |
-          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=vfs_release.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log
+          failure="${{ steps.run_tests_release_vfs_off.outcome == 'failure' || steps.stop_tests_release_vfs_off.outcome == 'failure' }}"
+          sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stderror_path=vfs_release_error.log --stdout_path=vfs_release.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
     
       - name: Unit test release version
         if: ${{ !cancelled() && !failure() }}

diff --git a/.gitignore b/.gitignore
@@ -26,6 +26,7 @@
 *.lib
 *.lo
 *.log
+*.log.*
 *.mod
 *.o
 *.obj

diff --git a/python/restart_test/infinity_runner.py b/python/restart_test/infinity_runner.py
@@ -27,6 +27,12 @@ def init(self, config_path: str | None = None):
             config_path = self.default_config_path
         cmd = f"{self.infinity_path} --config={config_path} > restart_test.log.{self.i} 2>&1"
 
+        pids = [proc.pid for proc in psutil.process_iter(['pid', 'name']) if "infinity" in proc.info['name']]
+        if len(pids) > 0:
+            ret = os.system(f"bash {self.script_path} 30 {' '.join(map(str, pids))}")
+            if ret != 0:
+                raise Exception("An error occurred.")
+
         # unset env LD_PRELOAD, ASAN_OPTIONS
         my_env = os.environ.copy()
         my_env["LD_PRELOAD"] = ""
@@ -39,6 +45,8 @@ def uninit(self):
         pids = []
         for child in psutil.Process(self.process.pid).children(recursive=True):
             pids.append(child.pid)
+        if len(pids) == 0:
+            raise Exception("Cannot find infinity process.")
         ret = os.system(f"bash {self.script_path} {timeout} {' '.join(map(str, pids))}")
         if ret != 0:
             raise Exception("An error occurred.")
@@ -63,9 +71,11 @@ def decorator(f):
         def wrapper(*args, **kwargs):
             infinity_runner.init(config_path)
             infinity_obj = InfinityRunner.connect(uri)
-            f(infinity_obj, *args, **kwargs)
-            infinity_obj.disconnect()
-            infinity_runner.uninit()
+            try :
+                f(infinity_obj, *args, **kwargs)
+            finally:
+                infinity_obj.disconnect()
+                infinity_runner.uninit()
 
         return wrapper
 

diff --git a/python/restart_test/test_cleanup.py b/python/restart_test/test_cleanup.py
@@ -63,12 +63,15 @@ def part1(infinity_obj):
 
             db_obj.drop_table(table_name, ConflictType.Error)
 
-            time.sleep(3)  # sleep 2 wait for the data to be cleaned up
+            infinity_obj.cleanup()
 
             # check
             dropped_dirs = pathlib.Path(data_dir).rglob(f"*{table_name}*")
             assert len(list(dropped_dirs)) == 0
 
+            dropped_dirs = pathlib.Path(data_dir).rglob(f"*{table_name2}*")
+            assert len(list(dropped_dirs)) == 1
+
             db_obj.drop_table(table_name2, ConflictType.Error)
 
         part1()
@@ -108,6 +111,7 @@ def test_cleanuped_index(
         infinity_runner.clear()
         table_name = "test_cleanup_index"
         index_name = "idx1"
+        index_name2 = "idx2"
 
         decorator = infinity_runner_decorator_factory(config, uri, infinity_runner)
         insert_n = 100
@@ -122,6 +126,9 @@ def part1(infinity_obj):
                 res = table_obj.create_index(index_name, idx)
                 assert res.error_code == infinity.ErrorCode.OK
 
+                res = table_obj.create_index(index_name2, idx)
+                assert res.error_code == infinity.ErrorCode.OK
+
             data_gen = data_gen_factory(insert_n)
 
             for data in data_gen:
@@ -133,16 +140,24 @@ def part1(infinity_obj):
             res = table_obj.drop_index(index_name)
             assert res.error_code == infinity.ErrorCode.OK
 
-            time.sleep(3)  # sleep 2 wait for the data to be cleaned up
+            infinity_obj.cleanup()
 
             # check
             dropped_dirs = pathlib.Path(data_dir).rglob(f"*{index_name}*")
             assert len(list(dropped_dirs)) == 0
 
+            dropped_dirs = pathlib.Path(data_dir).rglob(f"*{index_name2}*")
+            assert len(list(dropped_dirs)) == 1
+
+            res = table_obj.drop_index(index_name2)
+
         part1()
 
         @decorator
         def part2(infinity_obj):
+            dropped_dirs = pathlib.Path(data_dir).rglob(f"*{index_name2}")
+            assert len(list(dropped_dirs)) == 0
+
             db_obj = infinity_obj.get_database("default_db")
             table_obj = db_obj.get_table(table_name)
             data_dict, _ = table_obj.output(["count(*)"]).to_result()
@@ -151,9 +166,49 @@ def part2(infinity_obj):
 
             db_obj.drop_table(table_name, ConflictType.Error)
 
-            time.sleep(3)  # sleep 2 wait for the data to be cleaned up
+            infinity_obj.cleanup()
 
             dropped_dirs = pathlib.Path(data_dir).rglob(f"*{table_name}*")
             assert len(list(dropped_dirs)) == 0
 
         part2()
+
+    def test_invalidate_fulltext_cache(self, infinity_runner: InfinityRunner):
+        table_name = "test_invalid_fulltext_cache"
+        config = "test/data/config/restart_test/test_cleanup/1.toml"
+        uri = common_values.TEST_LOCAL_HOST
+        infinity_runner.clear()
+
+        decorator = infinity_runner_decorator_factory(config, uri, infinity_runner)
+        @decorator
+        def part1(infinity_obj):
+            db_obj = infinity_obj.get_database("default_db")
+            db_obj.drop_table(table_name, ConflictType.Ignore)
+            table_obj = db_obj.create_table(
+                table_name,
+                {
+                    "c1": {"type": "varchar"},
+                    "c2": {"type": "varchar"},
+                },
+            )
+            table_obj.insert([{"c1": "text1", "c2": "text2"}])
+
+            drop_index_name = "idx1_todrop"
+            table_obj.create_index(drop_index_name, index.IndexInfo("c1", index.IndexType.FullText))
+            table_obj.create_index("idx2", index.IndexInfo("c2", index.IndexType.FullText))
+
+            res = (
+                table_obj.output(["c1"])
+                .match_text(fields="c1", matching_text="text1", topn=1)
+                .to_result()
+            )
+
+            table_obj.drop_index(drop_index_name)
+
+            infinity_obj.cleanup()
+            dropped_index_dirs = pathlib.Path("/var/infinity/data").rglob(f"*{drop_index_name}*")
+            assert len(list(dropped_index_dirs)) == 0
+
+            db_obj.drop_table(table_name)
+
+        part1()
diff --git a/python/test_pysdk/test_cleanup.py b/python/test_pysdk/test_cleanup.py
-Original file line number
+Diff line change
@@ Expand Up / @@ -26,6 +26,7 @@ @@
     *.lib
     *.lo
     *.log
+    *.log.*
     *.mod
     *.o
     *.obj
@@ Expand Down @@