From 2632b8db86399397c556b22b1e34e8025ba579bd Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 20 Jun 2024 21:01:55 -0700 Subject: [PATCH 1/5] backport ci to 1.1 --- .github/workflows/integrationtests.yml | 167 ++++++++++++------ .../workflows/resources/vdbench_big_file.conf | 1 + .../workflows/resources/vdbench_long_run.conf | 4 + .../resources/vdbench_small_file.conf | 1 + .github/workflows/vdbench.yml | 97 +++++----- 5 files changed, 174 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/resources/vdbench_long_run.conf diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index 6bf8afbf0ad9..3e8bf06a825e 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -5,18 +5,22 @@ on: branches: - 'main' - 'release-*' - paths-ignore: - - 'docs/**' - - '**.md' + paths: + - '**.c' + - '**.go' + - 'Makefile' + - '**/integrationtests.yml' pull_request: - #The branches below must be a subset of the branches above branches: - 'main' - 'release-*' - paths-ignore: - - 'docs/**' - - '**.md' - - '.github/**' + paths: + - '**.c' + - '**.go' + - 'Makefile' + - '**/integrationtests.yml' + schedule: + - cron: '0 19 * * *' workflow_dispatch: inputs: debug: @@ -26,36 +30,74 @@ on: default: false jobs: + build-matrix: + runs-on: ubuntu-20.04 + steps: + - id: set-matrix + run: | + echo "github.event_name is ${{github.event_name}}" + echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}" + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then + echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT + elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then + echo 'meta_matrix=["redis", "mysql", "tikv"]' >> $GITHUB_OUTPUT + else + echo "event_name is not supported" && exit 1 + fi + outputs: + meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} + integrationtests: - timeout-minutes: 60 + timeout-minutes: 120 runs-on: ubuntu-20.04 + needs: build-matrix + strategy: + fail-fast: false + matrix: + meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} steps: - name: Checkout uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Set Variable + id: vars + run: | + if [ "${{matrix.meta}}" == "fdb" ]; then + echo "target=juicefs.fdb" >> $GITHUB_OUTPUT + else + echo "target=juicefs" >> $GITHUB_OUTPUT + fi + - name: Build uses: ./.github/actions/build + with: + target: ${{steps.vars.outputs.target}} - - name: Copy - run: | - cp juicefs /tmp/mount.juicefs - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes + - name: Prepare meta db + run: | + chmod +x .github/scripts/start_meta_engine.sh + source .github/scripts/start_meta_engine.sh + start_meta_engine ${{matrix.meta}} + meta_url=$(get_meta_url ${{matrix.meta}}) + create_database $meta_url - name: Juicefs Format run: | - sudo ./juicefs format redis://127.0.0.1:6379/1 pics + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + sudo ./juicefs format $meta_url --trash-days 0 pics - name: Juicefs Mount run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report & + source .github/scripts/start_meta_engine.sh + meta_url=$(get_meta_url ${{matrix.meta}}) + sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr + stat /jfs/.accesslog - name: Fslock Test + timeout-minutes: 5 run: | cd /jfs/ git clone https://github.com/danjacques/gofslock.git @@ -63,52 +105,73 @@ jobs: go test -v ./fslock/... stat /jfs/ - - - name: Pyxattr Test + - name: flock test + timeout-minutes: 5 run: | - git clone https://github.com/iustin/pyxattr.git - cd pyxattr - pip3 install pytest - pip3 install pyxattr - stat /jfs/ - TEST_DIR=/jfs/ python3 -m pytest tests - + git clone https://github.com/gofrs/flock.git + mkdir /jfs/tmp + cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test . - - name: Fstests + - name: make secfs.test run: | - sudo .github/scripts/apt_install.sh libacl1-dev attr - sudo DURATION=60 make -C fstests fsx - sudo make -C fstests xattrs - sudo make -C fstests flock - - - - name: Fsracer - #https://github.com/gfx/example-github-actions-with-tty - shell: 'script -q -e -c "bash {0}"' + sudo .github/scripts/apt_install.sh libacl1-dev + git clone https://github.com/billziss-gh/secfs.test.git + make -C secfs.test + + - name: Fsx Test + timeout-minutes: 16 run: | - sudo DURATION=60 make -C fstests fsracer - - - name: MountJuiceFSTest + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then + duration=900 + else + duration=300 + fi + sudo touch /jfs/fsx.out + sudo rm -f /tmp/fsx.out + sudo ln -s /jfs/fsx.out /tmp/fsx.out + sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out + + - name: Fsracer Test + timeout-minutes: 16 + shell: 'script -q -e -c "bash {0}"' run: | - sudo /tmp/mount.juicefs redis://127.0.0.1:6379/1 /tmp/mount-jfs - [ `stat --format "%i" /tmp/mount-jfs` -eq 1 ] - - + if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then + duration=900 + else + duration=300 + fi + sudo secfs.test/tools/bin/fsracer $duration /jfs + - name: log - if: ${{ always() }} + if: always() run: | tail -300 /var/log/juicefs.log grep ":" /var/log/juicefs.log && exit 1 || true + - name: Setup upterm session + if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) + timeout-minutes: 60 + uses: lhotari/action-upterm@v1 + + success-all-test: + runs-on: ubuntu-latest + needs: [integrationtests] + if: always() + steps: + - uses: technote-space/workflow-conclusion-action@v3 + - uses: actions/checkout@v3 + + - name: Check Failure + if: env.WORKFLOW_CONCLUSION == 'failure' + run: exit 1 + - name: Send Slack Notification - if: ${{ failure() }} + if: failure() && github.event_name != 'workflow_dispatch' uses: juicedata/slack-notify-action@main with: channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" - - - name: Setup upterm session - if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }} - timeout-minutes: 60 - uses: lhotari/action-upterm@v1 + - name: Success + if: success() + run: echo "All Done" \ No newline at end of file diff --git a/.github/workflows/resources/vdbench_big_file.conf b/.github/workflows/resources/vdbench_big_file.conf index 86267c45b31d..acdefbf6457d 100644 --- a/.github/workflows/resources/vdbench_big_file.conf +++ b/.github/workflows/resources/vdbench_big_file.conf @@ -1,3 +1,4 @@ +data_errors=1 fsd=fsd1,anchor=/tmp/vdbench/vdbench-big,depth=1,width=1,files=4,size=1g,openflags=o_direct fwd=fwd1,fsd=fsd1,operation=write,xfersize=1m,fileio=sequential,fileselect=sequential,threads=4 diff --git a/.github/workflows/resources/vdbench_long_run.conf b/.github/workflows/resources/vdbench_long_run.conf new file mode 100644 index 000000000000..b34bec35e6f6 --- /dev/null +++ b/.github/workflows/resources/vdbench_long_run.conf @@ -0,0 +1,4 @@ +data_errors=1 +fsd=fsd1,anchor=/tmp/jfs,depth=1,width=2,files=2,sizes=(30m,0),shared=yes,openflags=o_direct +fwd=fwd1,fsd=fsd1,threads=4,xfersize=(512,20,4k,20,64k,20,512k,20,1024k,20),fileio=random,fileselect=random,rdpct=70 +rd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=1800,interval=1 \ No newline at end of file diff --git a/.github/workflows/resources/vdbench_small_file.conf b/.github/workflows/resources/vdbench_small_file.conf index 1f92f92555aa..9c395ad7f459 100644 --- a/.github/workflows/resources/vdbench_small_file.conf +++ b/.github/workflows/resources/vdbench_small_file.conf @@ -1,3 +1,4 @@ +data_errors=1 fsd=fsd1,anchor=/tmp/vdbench/vdbench-small,depth=3,width=10,files=10,size=128k,openflags=o_direct fwd=fwd1,fsd=fsd1,operation=write,xfersize=128k,fileio=random,fileselect=random,threads=4 diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml index f309207b3724..376314450dc9 100644 --- a/.github/workflows/vdbench.yml +++ b/.github/workflows/vdbench.yml @@ -1,16 +1,35 @@ name: "vdbench" on: + push: + branches: + - 'main' + - 'release-*' + paths: + - '**/vdbench.yml' + pull_request: + branches: + - 'main' + - 'release-*' + paths: + - '**/vdbench.yml' schedule: - cron: '0 0 * * *' workflow_dispatch: + inputs: + debug: + type: boolean + description: "Run the build with tmate debugging enabled" + required: false + default: false jobs: vdbench: + timeout-minutes: 60 strategy: fail-fast: false matrix: - meta: [ 'sqlite3', 'redis', 'mysql', 'tikv', 'tidb', 'postgres', 'mariadb', 'fdb'] + meta: [ 'redis', 'mysql', 'fdb', 'tikv'] runs-on: ubuntu-20.04 steps: @@ -28,7 +47,19 @@ jobs: echo "target=juicefs" >> $GITHUB_OUTPUT fi + - name: Remove unused software + timeout-minutes: 10 + run: | + echo "before remove unused software" + sudo df -h + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + echo "after remove unused software" + sudo df -h + - name: Build + timeout-minutes: 10 uses: ./.github/actions/build with: target: ${{steps.vars.outputs.target}} @@ -48,67 +79,45 @@ jobs: chmod +x mc wget -q https://s.juicefs.com/static/bench/vdbench50407.zip unzip vdbench50407.zip -d vdbench - - - name: Start minio - run: | - docker run -d -p 19000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - sleep 5 - - name: vdbench-big + + - name: vdbench-long-run shell: bash - run: | + run: | source .github/scripts/start_meta_engine.sh meta_url=$(get_meta_url ${{matrix.meta}}) - ./juicefs format $meta_url vdbench-big --storage minio --bucket http://127.0.0.1:19000/mypics --access-key minioadmin --secret-key minioadmin - ./juicefs mount -d $meta_url /tmp/vdbench --no-usage-report - vdbench/vdbench -f .github/workflows/resources/vdbench_big_file.conf + ./juicefs format $meta_url vdbench-long-run --trash-days 0 + ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --cache-size 1024 --max-deletes 50 + vdbench/vdbench -f .github/workflows/resources/vdbench_long_run.conf -jn - uses: actions/upload-artifact@v3 with: - name: output-big + name: output-long-run path: output - - name: Clean + - name: check vdbench log + if: always() run: | - source .github/scripts/start_meta_engine.sh - meta_url=$(get_meta_url ${{matrix.meta}}) - rm /var/jfsCache/ -rf || true - if [ -d /tmp/vdbench ]; then - ./juicefs umount /tmp/vdbench || true - fi - uuid=$(./juicefs status $meta_url | grep UUID | cut -d '"' -f 4) || true - if [ -n "$uuid" ];then - sudo ./juicefs destroy --yes $meta_url $uuid + grep -i "java.lang.RuntimeException" output/errorlog.html && exit 1 || true + if ! grep -q "Vdbench execution completed successfully" output/logfile.html; then + echo "vdbench not completed succeed" + exit 1 fi - - name: vdbench-small - run: | - source .github/scripts/start_meta_engine.sh - meta_url=$(get_meta_url ${{matrix.meta}}) - ./juicefs format $meta_url vdbench-small --storage minio --bucket http://127.0.0.1:19000/mypics --access-key minioadmin --secret-key minioadmin - ./juicefs mount -d $meta_url /tmp/vdbench --no-usage-report - vdbench/vdbench -f .github/workflows/resources/vdbench_small_file.conf - - - uses: actions/upload-artifact@v3 - with: - name: output-small - path: output - - name: log - if: ${{ always() }} - shell: bash + if: always() run: | tail -300 ~/.juicefs/juicefs.log grep ":" ~/.juicefs/juicefs.log && exit 1 || true - + - name: Send Slack Notification - if: ${{ failure() }} + if: failure() uses: juicedata/slack-notify-action@main with: channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" + - name: Setup upterm session + if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) + # if: failure() + timeout-minutes: 60 + uses: lhotari/action-upterm@v1 \ No newline at end of file From 7e4ef92a78f4dca15091532e9f4dc9ba383b81e0 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 20 Jun 2024 21:04:09 -0700 Subject: [PATCH 2/5] wip From bd77b17dba3015e6b8f01261a2da8ae2c4f45a2f Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 20 Jun 2024 21:37:42 -0700 Subject: [PATCH 3/5] enlarge disk space --- .github/workflows/integrationtests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml index 3e8bf06a825e..82383298d12c 100644 --- a/.github/workflows/integrationtests.yml +++ b/.github/workflows/integrationtests.yml @@ -61,6 +61,17 @@ jobs: with: fetch-depth: 1 + - name: Remove unused software + timeout-minutes: 10 + run: | + echo "before remove unused software" + sudo df -h + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + echo "after remove unused software" + sudo df -h + - name: Set Variable id: vars run: | From 5b1f0eef0127f404ac5f550b9ea4df4a40d1c0af Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Fri, 21 Jun 2024 10:04:31 -0700 Subject: [PATCH 4/5] check disk space --- .github/workflows/vdbench.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml index 376314450dc9..8872306e4c57 100644 --- a/.github/workflows/vdbench.yml +++ b/.github/workflows/vdbench.yml @@ -94,6 +94,11 @@ jobs: name: output-long-run path: output + - name: check free disk space + if: always() + run: | + df -h + - name: check vdbench log if: always() run: | From ea2495540a336b5f63fd3eda5093179b86bbb04a Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Sun, 23 Jun 2024 21:28:00 -0700 Subject: [PATCH 5/5] skip vdbench on tikv/fdb --- .github/workflows/vdbench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml index 8872306e4c57..3238fb66297f 100644 --- a/.github/workflows/vdbench.yml +++ b/.github/workflows/vdbench.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - meta: [ 'redis', 'mysql', 'fdb', 'tikv'] + meta: [ 'redis', 'mysql'] runs-on: ubuntu-20.04 steps: @@ -98,7 +98,7 @@ jobs: if: always() run: | df -h - + - name: check vdbench log if: always() run: |