Enabling multitenancy tests with FSDP

.azure/hpu-tests.yml

@@ -99,6 +99,13 @@ jobs:
 
     # todo: add sanity check that needed cards are visible and accessible
 
+    - bash: |
+        bash tests/run_standalone_tests.sh --hpus 2 -f \
+          tests/test_pytorch/strategies/test_fsdp.py
+      env:
+        PT_HPU_LAZY_MODE: 0
+      displayName: 'FSDP PT Multi card(2) HPU test'
+
     - bash: |
         set -ex
         pip list
@@ -128,14 +135,14 @@ jobs:
         PT_HPU_LAZY_MODE: 0
       displayName: 'HPU torch compile tests'
 
-    # - bash: |
-    #     python -m pytest -sv tests/test_fabric/test_fsdp.py \
-    #       --hpus 1 --junitxml=hpu_test_fsdp-fabric-results.xml
-    #     python -m pytest -sv tests/test_pytorch/strategies/test_fsdp.py \
-    #       --hpus 1 --junitxml=hpu_test_fsdp-results.xml
-    #   env:
-    #     PT_HPU_LAZY_MODE: 0
-    #   displayName: 'HPU FSDP tests'
+    - bash: |
+        python -m pytest -sv tests/test_fabric/test_fsdp.py \
+          --hpus 1 --junitxml=hpu_test_fsdp-fabric-results.xml
+        python -m pytest -sv tests/test_pytorch/strategies/test_fsdp.py \
+          --hpus 1 --junitxml=hpu_test_fsdp-results.xml
+      env:
+        PT_HPU_LAZY_MODE: 0
+      displayName: 'HPU FSDP tests'
 
     - bash: |
         python -m pytest -sv tests/test_pytorch/strategies/test_deepspeed.py \
@@ -165,20 +172,13 @@ jobs:
           tests/test_pytorch/test_profiler.py
       displayName: 'Multi card(2) HPU test'
 
-    # - bash: |
-    #     bash tests/run_standalone_tests.sh --hpus 2 -f \
-    #       tests/test_pytorch/test_fsdp.py
-    #   env:
-    #     PT_HPU_LAZY_MODE: 0
-    #   displayName: 'FSDP PT Multi card(2) HPU test'
-
-    # - bash: |
-    #     bash tests/run_standalone_tests.sh --hpus 2 -f \
-    #       tests/test_fabric/test_fsdp.py
-    #   env:
-    #     PT_HPU_LAZY_MODE: 0
-    #   condition: or(eq(variables['HABANA_VISIBLE_MODULES'], '4,5'), eq(variables['HABANA_VISIBLE_MODULES'], '6,7'))
-    #   displayName: 'FSDP Fabric Multi card(2) HPU test'
+    - bash: |
+        bash tests/run_standalone_tests.sh --hpus 2 -f \
+          tests/test_fabric/test_fsdp.py
+      env:
+        PT_HPU_LAZY_MODE: 0
+      condition: or(eq(variables['HABANA_VISIBLE_MODULES'], '4,5'), eq(variables['HABANA_VISIBLE_MODULES'], '6,7'))
+      displayName: 'FSDP Fabric Multi card(2) HPU test'
 
     - bash: pip install ".[examples]"
       condition: or(eq(variables['HABANA_VISIBLE_MODULES'], '4,5'), eq(variables['HABANA_VISIBLE_MODULES'], '6,7'))

src/lightning_habana/fabric/strategies/fsdp.py

@@ -95,6 +95,7 @@ def __init__(
     ) -> None:
         if not _LIGHTNING_GREATER_EQUAL_2_3_0:
             raise OSError("HPUFSDPStrategy requires `lightning>=2.3.0 or pytorch-lightning >= 2.3.0`.")
+
         super().__init__(
             accelerator=accelerator,
             parallel_devices=parallel_devices,

tests/run_standalone_tests.sh

@@ -62,7 +62,12 @@ fi
 echo "Test files: $test_files"
 
 # Get all the tests marked with standalone marker
+file_prefix="${HABANA_VISIBLE_MODULES//,/}"
+echo "$file_prefix"
+
 TEST_FILE="standalone_tests.txt"
+TEST_FILE="$file_prefix$TEST_FILE"
+echo "Test file $TEST_FILE"
 test_command="python -um pytest ${test_files} -q --collect-only --pythonwarnings ignore -m \"${marker}\""
 
 if [[ -n "$filter" ]]; then
@@ -102,5 +107,5 @@ for result in "${results[@]}"; do
 done
 echo "===== STANDALONE TEST STATUS END ====="
 
-mv tests/**/*.xml .
+mv tests/**/**/*.xml .
 rm $TEST_FILE