Merge test jobs and READMEs.

.github/workflows/test_iree.yml

@@ -148,11 +148,13 @@ jobs:
           python3 iree_tests/download_remote_files.py --root-dir pytorch/models
           python3 iree_tests/download_remote_files.py --root-dir sharktank
 
-      - name: "Running real weights PyTorch model tests"
+      - name: "Running real weight model tests"
         if: "matrix.models-config-file != '' && !cancelled()"
         run: |
           source ${VENV_DIR}/bin/activate
-          pytest iree_tests/pytorch/models \
+          pytest \
+            iree_tests/pytorch/models \
+            iree_tests/pytorch/sharktank \
             -n 4 \
             -rpfE \
             -k real_weights \
@@ -163,20 +165,6 @@ jobs:
             --durations=0 \
             --config-files=${MODELS_CONFIG_FILE_PATH}
 
-      - name: "Running SHARK Tank model tests"
-        if: "matrix.models-config-file != '' && !cancelled()"
-        run: |
-          source ${VENV_DIR}/bin/activate
-          pytest iree_tests/sharktank \
-            -n 4 \
-            -rpfE \
-            --no-skip-tests-missing-files \
-            --capture=no \
-            --log-cli-level=info \
-            --timeout=1200 \
-            --durations=0 \
-            --config-files=${MODELS_CONFIG_FILE_PATH}
-
       - name: "Running real weights SDXL prompt encoder tests"
         id: prompt_encoder
         if: ${{ !cancelled() }}

iree_tests/README.md

@@ -326,8 +326,6 @@ python -m pip install -r ./iree_tests/onnx/requirements.txt
 python ./iree_tests/onnx/import_tests.py
 ```
 
-### PyTorch models
-
 #### Generating model test cases from e2eshark
 
 > [!WARNING]
@@ -413,6 +411,27 @@ Then, run the runner with the appropriate command line args (vmfb path, device f
 You should have all the artifacts needed to add to this TestSuite at that point.
 Make sure to follow to follow appendix instructions to convert between different file types for weights and mlir.
 
+### SHARK Tank models
+
+These test cases are exported from https://github.com/nod-ai/sharktank.
+
+## Steps to add test cases
+
+* Follow instructions in https://github.com/nod-ai/sharktank/blob/main/docs/model_cookbook.md
+* Convert the exported `.mlir` to `.mlirbc`:
+
+    ```bash
+    iree-ir-tool cp file.mlir --emit-bytecode -o file.mlirbc
+    ```
+
+* Create a test_cases.json file with parameters, inputs, and outputs
+  * Parameters can come from Hugging Face by using URL from "download file"
+  * TODO: inputs and outputs should be exportable from sharktank/shortfin
+    (or a script here - need to run the tokenizer and optionally populate the
+    KV cache for some models)
+
+### PyTorch models
+
 ## Appendix
 
 ### Working with .mlirbc files