Merge branch 'main' into shortfin-system-selection

nod-ai · Oct 17, 2024 · b2ede02 · b2ede02
2 parents 214fd64 + f5fcd00
commit b2ede02
Show file tree

Hide file tree

Showing 23 changed files with 303 additions and 1,544 deletions.
diff --git a/.github/workflows/eval_test.yaml b/.github/workflows/eval_test.yaml
diff --git a/docs/model_cookbook.md b/docs/model_cookbook.md
@@ -257,16 +257,6 @@ iree-run-module \
   --parameters=model=/tmp/open_llama_3b_v2/open-llama-3b-v2-f16.gguf
 ```
 
-## Evaluation pipeline
-
-Run perplexity test:
-
-```bash
-python -m  sharktank.evaluate.perplexity \
-  --gguf-file=llama8b_f16.gguf \
-  --tokenizer-config-json=tokenizer_config.json
-```
-
 ## Generating data for llama models
 
 ```bash

diff --git a/requirements.txt b/requirements.txt
@@ -7,7 +7,6 @@ onnx==1.15.0
 huggingface-hub==0.22.2
 transformers==4.40.0
 sentencepiece==0.2.0
-datasets==3.0.0
 
 # It is expected that you have installed a PyTorch version/variant specific
 # to your needs, so we only include a minimum version spec.

diff --git a/sharktank/sharktank/evaluate/data/eval_prompts.txt b/sharktank/sharktank/evaluate/data/eval_prompts.txt