fix: set use_instructions to True in models using prompts (#79)

embeddings-benchmark · Dec 29, 2024 · 9675eaa · 9675eaa
1 parent 2a93181
commit 9675eaa
Show file tree

Hide file tree

Showing 20 changed files with 434 additions and 19 deletions.
diff --git a/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json b/results/BAAI__bge-base-en-v1.5/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model_meta.json
@@ -1 +1,25 @@
-{"name": "BAAI/bge-base-en-v1.5", "revision": "a5beb1e3e68b9ab74eb54cfd186867f64f240e1a", "release_date": "2023-09-11", "languages": ["eng_Latn"], "n_parameters": 438000000, "memory_usage": null, "max_tokens": 512, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-base-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"}
+{
+    "name": "BAAI/bge-base-en-v1.5",
+    "revision": "a5beb1e3e68b9ab74eb54cfd186867f64f240e1a",
+    "release_date": "2023-09-11",
+    "languages": [
+        "eng_Latn"
+    ],
+    "n_parameters": 438000000,
+    "memory_usage": null,
+    "max_tokens": 512,
+    "embed_dim": 768,
+    "license": "mit",
+    "open_weights": true,
+    "public_training_data": null,
+    "public_training_code": null,
+    "framework": [
+        "Sentence Transformers",
+        "PyTorch"
+    ],
+    "reference": "https://huggingface.co/BAAI/bge-base-en-v1.5",
+    "similarity_fn_name": "cosine",
+    "use_instructions": true,
+    "zero_shot_benchmarks": null,
+    "loader": "sentence_transformers_loader"
+}
diff --git a/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json b/results/BAAI__bge-large-en-v1.5/d4aa6901d3a41ba39fb536a557fa166f842b0e09/model_meta.json
@@ -19,7 +19,7 @@
   ],
   "reference": "https://huggingface.co/BAAI/bge-large-en-v1.5",
   "similarity_fn_name": "cosine",
-  "use_instructions": false,
+  "use_instructions": true,
   "zero_shot_benchmarks": null,
   "loader": "sentence_transformers_loader"
 }
diff --git a/results/BAAI__bge-small-en-v1.5/5c38ec7c405ec4b44b94cc5a9bb96e735b38267a/model_meta.json b/results/BAAI__bge-small-en-v1.5/5c38ec7c405ec4b44b94cc5a9bb96e735b38267a/model_meta.json
@@ -1 +1,25 @@
-{"name": "BAAI/bge-small-en-v1.5", "revision": "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", "release_date": "2023-09-12", "languages": ["eng_Latn"], "n_parameters": 24000000, "memory_usage": null, "max_tokens": 512, "embed_dim": 512, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-small-en-v1.5", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"}
+{
+    "name": "BAAI/bge-small-en-v1.5",
+    "revision": "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a",
+    "release_date": "2023-09-12",
+    "languages": [
+        "eng_Latn"
+    ],
+    "n_parameters": 24000000,
+    "memory_usage": null,
+    "max_tokens": 512,
+    "embed_dim": 512,
+    "license": "mit",
+    "open_weights": true,
+    "public_training_data": null,
+    "public_training_code": null,
+    "framework": [
+        "Sentence Transformers",
+        "PyTorch"
+    ],
+    "reference": "https://huggingface.co/BAAI/bge-small-en-v1.5",
+    "similarity_fn_name": "cosine",
+    "use_instructions": true,
+    "zero_shot_benchmarks": null,
+    "loader": "sentence_transformers_loader"
+}
diff --git a/results/Cohere__Cohere-embed-english-light-v3.0/1/model_meta.json b/results/Cohere__Cohere-embed-english-light-v3.0/1/model_meta.json
@@ -1 +1,26 @@
-{"name": "Cohere/Cohere-embed-english-light-v3.0", "revision": "1", "release_date": "2023-11-02", "languages": ["eng-Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": 512.0, "embed_dim": 384, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://cohere.com/blog/introducing-embed-v3", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": "CohereTextEmbeddingModel"}
+{
+    "name": "Cohere/Cohere-embed-english-light-v3.0",
+    "revision": "1",
+    "release_date": "2023-11-02",
+    "languages": [
+        "eng-Latn"
+    ],
+    "n_parameters": null,
+    "memory_usage": null,
+    "max_tokens": 512,
+    "embed_dim": 384,
+    "license": null,
+    "open_weights": false,
+    "public_training_data": null,
+    "public_training_code": null,
+    "framework": [
+        "API"
+    ],
+    "reference": "https://cohere.com/blog/introducing-embed-v3",
+    "similarity_fn_name": "cosine",
+    "use_instructions": true,
+    "zero_shot_benchmarks": null,
+    "adapted_from": null,
+    "superseded_by": null,
+    "loader": "CohereTextEmbeddingModel"
+}
diff --git a/results/Cohere__Cohere-embed-english-v3.0/1/model_meta.json b/results/Cohere__Cohere-embed-english-v3.0/1/model_meta.json
@@ -1 +1,26 @@
-{"name": "Cohere/Cohere-embed-english-v3.0", "revision": "1", "release_date": "2023-11-02", "languages": ["eng-Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": 512.0, "embed_dim": 1024, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://cohere.com/blog/introducing-embed-v3", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": "CohereTextEmbeddingModel"}
+{
+    "name": "Cohere/Cohere-embed-english-v3.0",
+    "revision": "1",
+    "release_date": "2023-11-02",
+    "languages": [
+        "eng-Latn"
+    ],
+    "n_parameters": null,
+    "memory_usage": null,
+    "max_tokens": 512,
+    "embed_dim": 1024,
+    "license": null,
+    "open_weights": false,
+    "public_training_data": null,
+    "public_training_code": null,
+    "framework": [
+        "API"
+    ],
+    "reference": "https://cohere.com/blog/introducing-embed-v3",
+    "similarity_fn_name": "cosine",
+    "use_instructions": true,
+    "zero_shot_benchmarks": null,
+    "adapted_from": null,
+    "superseded_by": null,
+    "loader": "CohereTextEmbeddingModel"
+}
diff --git a/results/Cohere__Cohere-embed-multilingual-light-v3.0/1/model_meta.json b/results/Cohere__Cohere-embed-multilingual-light-v3.0/1/model_meta.json
@@ -1 +1,134 @@
-{"name": "Cohere/Cohere-embed-multilingual-light-v3.0", "revision": "1", "release_date": "2023-11-02", "languages": ["afr-Latn", "amh-Ethi", "ara-Arab", "asm-Beng", "aze-Latn", "bel-Cyrl", "bul-Cyrl", "ben-Beng", "bod-Tibt", "bos-Latn", "cat-Latn", "ceb-Latn", "cos-Latn", "ces-Latn", "cym-Latn", "dan-Latn", "deu-Latn", "ell-Grek", "eng-Latn", "epo-Latn", "spa-Latn", "est-Latn", "eus-Latn", "fas-Arab", "fin-Latn", "fra-Latn", "fry-Latn", "gle-Latn", "gla-Latn", "glg-Latn", "guj-Gujr", "hau-Latn", "haw-Latn", "heb-Hebr", "hin-Deva", "hmn-Latn", "hrv-Latn", "hat-Latn", "hun-Latn", "hye-Armn", "ind-Latn", "ibo-Latn", "isl-Latn", "ita-Latn", "jpn-Jpan", "jav-Latn", "kat-Geor", "kaz-Cyrl", "khm-Khmr", "kan-Knda", "kor-Kore", "kur-Arab", "kir-Cyrl", "lat-Latn", "ltz-Latn", "lao-Laoo", "lit-Latn", "lav-Latn", "mlg-Latn", "mri-Latn", "mkd-Cyrl", "mal-Mlym", "mon-Cyrl", "mar-Deva", "msa-Latn", "mlt-Latn", "mya-Mymr", "nep-Deva", "nld-Latn", "nor-Latn", "nya-Latn", "ori-Orya", "pan-Guru", "pol-Latn", "por-Latn", "ron-Latn", "rus-Cyrl", "kin-Latn", "sin-Sinh", "slk-Latn", "slv-Latn", "smo-Latn", "sna-Latn", "som-Latn", "sqi-Latn", "srp-Cyrl", "sot-Latn", "sun-Latn", "swe-Latn", "swa-Latn", "tam-Taml", "tel-Telu", "tgk-Cyrl", "tha-Thai", "tuk-Latn", "tgl-Latn", "tur-Latn", "tat-Cyrl", "uig-Arab", "ukr-Cyrl", "urd-Arab", "uzb-Latn", "vie-Latn", "wol-Latn", "xho-Latn", "yid-Hebr", "yor-Latn", "zho-Hans", "zul-Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": 512.0, "embed_dim": 384, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://cohere.com/blog/introducing-embed-v3", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": "CohereTextEmbeddingModel"}
+{
+    "name": "Cohere/Cohere-embed-multilingual-light-v3.0",
+    "revision": "1",
+    "release_date": "2023-11-02",
+    "languages": [
+        "afr-Latn",
+        "amh-Ethi",
+        "ara-Arab",
+        "asm-Beng",
+        "aze-Latn",
+        "bel-Cyrl",
+        "bul-Cyrl",
+        "ben-Beng",
+        "bod-Tibt",
+        "bos-Latn",
+        "cat-Latn",
+        "ceb-Latn",
+        "cos-Latn",
+        "ces-Latn",
+        "cym-Latn",
+        "dan-Latn",
+        "deu-Latn",
+        "ell-Grek",
+        "eng-Latn",
+        "epo-Latn",
+        "spa-Latn",
+        "est-Latn",
+        "eus-Latn",
+        "fas-Arab",
+        "fin-Latn",
+        "fra-Latn",
+        "fry-Latn",
+        "gle-Latn",
+        "gla-Latn",
+        "glg-Latn",
+        "guj-Gujr",
+        "hau-Latn",
+        "haw-Latn",
+        "heb-Hebr",
+        "hin-Deva",
+        "hmn-Latn",
+        "hrv-Latn",
+        "hat-Latn",
+        "hun-Latn",
+        "hye-Armn",
+        "ind-Latn",
+        "ibo-Latn",
+        "isl-Latn",
+        "ita-Latn",
+        "jpn-Jpan",
+        "jav-Latn",
+        "kat-Geor",
+        "kaz-Cyrl",
+        "khm-Khmr",
+        "kan-Knda",
+        "kor-Kore",
+        "kur-Arab",
+        "kir-Cyrl",
+        "lat-Latn",
+        "ltz-Latn",
+        "lao-Laoo",
+        "lit-Latn",
+        "lav-Latn",
+        "mlg-Latn",
+        "mri-Latn",
+        "mkd-Cyrl",
+        "mal-Mlym",
+        "mon-Cyrl",
+        "mar-Deva",
+        "msa-Latn",
+        "mlt-Latn",
+        "mya-Mymr",
+        "nep-Deva",
+        "nld-Latn",
+        "nor-Latn",
+        "nya-Latn",
+        "ori-Orya",
+        "pan-Guru",
+        "pol-Latn",
+        "por-Latn",
+        "ron-Latn",
+        "rus-Cyrl",
+        "kin-Latn",
+        "sin-Sinh",
+        "slk-Latn",
+        "slv-Latn",
+        "smo-Latn",
+        "sna-Latn",
+        "som-Latn",
+        "sqi-Latn",
+        "srp-Cyrl",
+        "sot-Latn",
+        "sun-Latn",
+        "swe-Latn",
+        "swa-Latn",
+        "tam-Taml",
+        "tel-Telu",
+        "tgk-Cyrl",
+        "tha-Thai",
+        "tuk-Latn",
+        "tgl-Latn",
+        "tur-Latn",
+        "tat-Cyrl",
+        "uig-Arab",
+        "ukr-Cyrl",
+        "urd-Arab",
+        "uzb-Latn",
+        "vie-Latn",
+        "wol-Latn",
+        "xho-Latn",
+        "yid-Hebr",
+        "yor-Latn",
+        "zho-Hans",
+        "zul-Latn"
+    ],
+    "n_parameters": null,
+    "memory_usage": null,
+    "max_tokens": 512,
+    "embed_dim": 384,
+    "license": null,
+    "open_weights": false,
+    "public_training_data": null,
+    "public_training_code": null,
+    "framework": [
+        "API"
+    ],
+    "reference": "https://cohere.com/blog/introducing-embed-v3",
+    "similarity_fn_name": "cosine",
+    "use_instructions": true,
+    "zero_shot_benchmarks": null,
+    "adapted_from": null,
+    "superseded_by": null,
+    "loader": "CohereTextEmbeddingModel"
+}