From dba9149397aed03e7f3615d40b25f298783d0137 Mon Sep 17 00:00:00 2001
From: Almaz Dautov <64196918+thehir0@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:34:59 +0300
Subject: [PATCH 1/3] RuHumanEval format fix and RuTie format fix

---
 scripts/log_to_submission.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/log_to_submission.py b/scripts/log_to_submission.py
index 59e8028..5e4ee58 100644
--- a/scripts/log_to_submission.py
+++ b/scripts/log_to_submission.py
@@ -11,8 +11,8 @@
 from tqdm.auto import tqdm
 
 from lm_eval.loggers.evaluation_tracker import GeneralConfigTracker
-
-
+#MERA_FOLDER=/workspace/MERA/fix-mera/mera_run_030/test_mera_run
+#MERA_MODEL_STRING="pretrained=/app/superllama,dtype=auto,max_length=16384"
 BENCHMARK_STORAGE: Optional[str] = "ai-forever/MERA"
 _TASKS = {}
 GENERATIVE_SUFFIX = "_gen"
@@ -370,7 +370,7 @@ def outputs_to_submission(
                 # check that question_id was passed to LM
                 if question_id_outputs is not None:
                     new_question = {
-                        "outputs": question_id_outputs,
+                        "outputs": question_id_outputs['outputs'],
                         "meta": {
                             "dialog_id": dialog_id,
                             "question_id": question_id,
@@ -402,6 +402,15 @@ def doc_outputs_to_submission(self, doc_id, outputs):
 
 @register_task
 class ruHumanEval(TextTask):
+
+    def outputs_to_submission(self, outputs):
+        res = []
+        for doc in outputs:
+            doc_id = int(self.doc_to_id(doc["doc"]))
+            resp = doc["filtered_resps"][0]
+            res.extend([self.doc_outputs_to_submission(doc_id, resp)])
+        return {"data": {"test": res}}
+    
     def doc_outputs_to_submission(self, doc_id, outputs):
         res = {
             "outputs": outputs,

From e79ad785c51e397367b59e14c3abb25b31484204 Mon Sep 17 00:00:00 2001
From: Almaz Dautov <64196918+thehir0@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:35:39 +0300
Subject: [PATCH 2/3] Update log_to_submission.py

---
 scripts/log_to_submission.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/log_to_submission.py b/scripts/log_to_submission.py
index 5e4ee58..a6bc5c5 100644
--- a/scripts/log_to_submission.py
+++ b/scripts/log_to_submission.py
@@ -11,8 +11,7 @@
 from tqdm.auto import tqdm
 
 from lm_eval.loggers.evaluation_tracker import GeneralConfigTracker
-#MERA_FOLDER=/workspace/MERA/fix-mera/mera_run_030/test_mera_run
-#MERA_MODEL_STRING="pretrained=/app/superllama,dtype=auto,max_length=16384"
+
 BENCHMARK_STORAGE: Optional[str] = "ai-forever/MERA"
 _TASKS = {}
 GENERATIVE_SUFFIX = "_gen"

From b50891e424174f5faeea5ee4c6327bab0cfb10f3 Mon Sep 17 00:00:00 2001
From: Almaz Dautov <64196918+thehir0@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:42:24 +0300
Subject: [PATCH 3/3] Update log_to_submission.py

---
 scripts/log_to_submission.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/log_to_submission.py b/scripts/log_to_submission.py
index a6bc5c5..c918b67 100644
--- a/scripts/log_to_submission.py
+++ b/scripts/log_to_submission.py
@@ -12,6 +12,7 @@
 
 from lm_eval.loggers.evaluation_tracker import GeneralConfigTracker
 
+
 BENCHMARK_STORAGE: Optional[str] = "ai-forever/MERA"
 _TASKS = {}
 GENERATIVE_SUFFIX = "_gen"