diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
index a561195..59d0587 100644
--- a/.github/workflows/gpu_test.yml
+++ b/.github/workflows/gpu_test.yml
@@ -28,8 +28,7 @@ jobs:
           pip install -e .[test]
       - name: Running dataset tests
         run: |
-          git lfs install
-          rm -rf ~/verl-data && git clone --depth 1 https://github.com/eric-haibin-lin/verl-data ~/verl-data
+          [ ! -d "$HOME/verl-data" ] && git clone --depth 1 https://github.com/eric-haibin-lin/verl-data ~/verl-data
           pytest -s -x tests/verl
       - name: Running ray tests that need 2 GPUs
         run: |