From d6edc273f8321f39708289d2400ee764b080e57f Mon Sep 17 00:00:00 2001
From: zhijian <djx@juicedata.io>
Date: Tue, 1 Aug 2023 12:55:18 +0800
Subject: [PATCH] CI: test list with follow link (#3953)

---
 .github/scripts/command/sync.sh | 78 +++++++++++++++++++++++++++------
 .github/workflows/command.yml   | 14 +++---
 2 files changed, 72 insertions(+), 20 deletions(-)

diff --git a/.github/scripts/command/sync.sh b/.github/scripts/command/sync.sh
index 3db245156781..efe4018dedea 100755
--- a/.github/scripts/command/sync.sh
+++ b/.github/scripts/command/sync.sh
@@ -6,6 +6,27 @@ source .github/scripts/start_meta_engine.sh
 start_meta_engine $META
 META_URL=$(get_meta_url $META)
 
+generate_source_dir(){
+    rm -rf jfs_source
+    git clone https://github.com/juicedata/juicefs.git jfs_source --depth 1
+    chmod 777 jfs_source
+    mkdir jfs_source/empty_dir
+    dd if=/dev/urandom of=jfs_source/file bs=5M count=1
+    chmod 777 jfs_source/file
+    ln -sf file jfs_source/symlink_to_file
+    ln -f jfs_source/file jfs_source/hard_link_to_file
+    id -u juicefs  && sudo userdel juicefs
+    sudo useradd -u 1101 juicefs
+    sudo -u juicefs touch jfs_source/file2
+    ln -s ../cmd jfs_source/pkg/symlink_to_cmd
+}
+
+generate_source_dir
+
+generate_fsrand(){
+    seed=$(date +%s)
+    python3 .github/scripts/fsrand.py -a -c 2000 -s $seed  fsrand
+}
 
 test_sync_with_mount_point(){
     do_sync_with_mount_point 
@@ -24,7 +45,6 @@ test_sync_without_mount_point(){
 do_sync_without_mount_point(){
     prepare_test
     options=$@
-    generate_source_dir
     ./juicefs format $META_URL myjfs
     meta_url=$META_URL ./juicefs sync jfs_source/ jfs://meta_url/jfs_source/ $options --links
 
@@ -39,7 +59,6 @@ do_sync_without_mount_point(){
 do_sync_with_mount_point(){
     prepare_test
     options=$@
-    generate_source_dir
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
     ./juicefs sync jfs_source/ /jfs/jfs_source/ $options --links
@@ -51,19 +70,50 @@ do_sync_with_mount_point(){
     diff -ur --no-dereference jfs_source/ /jfs/jfs_source/
 }
 
+test_sync_with_loop_link(){
+    prepare_test
+    options="--dirs --update --perms --check-all --list-threads 10 --list-depth 5"
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    ln -s looplink jfs_source/looplink
+    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options > err.log 2>&1 || true
+    grep "Failed to handle 1 objects" err.log
+    rm -rf jfs_source/looplink
+}
 
-generate_source_dir(){
-    [[ ! -d jfs_source ]] && git clone https://github.com/juicedata/juicefs.git jfs_source
-    [[ -d jfs_source/empty_dir ]] && rm jfs_source/empty_dir -rf
-    chmod 777 jfs_source
-    mkdir jfs_source/empty_dir
-    dd if=/dev/urandom of=jfs_source/file bs=5M count=1
-    chmod 777 jfs_source/file
-    ln -sf file jfs_source/symlink_to_file
-    ln -f jfs_source/file jfs_source/hard_link_to_file
-    id -u juicefs  && sudo userdel juicefs
-    sudo useradd -u 1101 juicefs
-    sudo -u juicefs touch jfs_source/file2
+test_sync_with_deep_link(){
+    prepare_test
+    options="--dirs --update --perms --check-all --list-threads 10 --list-depth 5"
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    touch jfs_source/symlink_1
+    for i in {1..41}; do
+        ln -s symlink_$i jfs_source/symlink_$((i+1))
+    done
+    ./juicefs sync jfs_source/ /jfs/jfs_source/ $options  2>&1 | tee err.log || true
+    grep "Failed to handle 1 objects" err.log
+    rm -rf jfs_source/symlink_*
+}
+
+skip_test_sync_fsrand_with_mount_point(){
+    generate_fsrand
+    do_test_sync_fsrand_with_mount_point 
+    do_test_sync_fsrand_with_mount_point --list-threads 10 --list-depth 5
+    do_test_sync_fsrand_with_mount_point --dirs --update --perms --check-all 
+    do_test_sync_fsrand_with_mount_point --dirs --update --perms --check-all --list-threads 10 --list-depth 5
+}
+
+do_test_sync_fsrand_with_mount_point(){
+    prepare_test
+    options=$@
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs
+    ./juicefs sync fsrand/ /jfs/fsrand/ $options --links
+
+    if [[ ! "$options" =~ "--dirs" ]]; then
+        find jfs_source -type d -empty -delete
+    fi
+    diff -ur --no-dereference fsrand/ /jfs/fsrand/
 }
 
 test_sync_randomly(){
diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml
index c611ccf38e5a..216f7cf0dc47 100644
--- a/.github/workflows/command.yml
+++ b/.github/workflows/command.yml
@@ -164,6 +164,13 @@ jobs:
 
       - name: Build 
         uses: ./.github/actions/build
+        # with:
+        #   useBeta: true
+
+      - name: Test Sync
+        timeout-minutes: 30
+        run: |
+          sudo META=${{matrix.meta}} .github/scripts/command/sync.sh 
 
       - name: Test Sync with mino
         timeout-minutes: 30
@@ -182,11 +189,6 @@ jobs:
           random_type=${types[$RANDOM % ${#types[@]}]}
           sudo CI=true META=${{matrix.meta}} KEY_TYPE=$random_type .github/scripts/command/sync_cluster.sh 
 
-      - name: Test Sync
-        timeout-minutes: 30
-        run: |
-          sudo META=${{matrix.meta}} .github/scripts/command/sync.sh 
-
       - name: Log
         if: ${{ always() }}
         run: |
@@ -222,4 +224,4 @@ jobs:
 
       - name: Success
         if: ${{ success() }}
-        run: echo "All Done"
\ No newline at end of file
+        run: echo "All Done"