Fix EvolQuality._apply_random_mutation not properly injecting `resp…

…onse` in template (#703) * Remove selecting final index from `response` for `EvolQuality.apply_mutation_template` * Add `_apply_random_mutation` unit test --------- Co-authored-by: Gabriel Martín Blázquez <[email protected]>
argilla-io · Jun 6, 2024 · 20aa24e · 20aa24e
1 parent 062f4fb
commit 20aa24e
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/src/distilabel/steps/tasks/evol_quality/base.py b/src/distilabel/steps/tasks/evol_quality/base.py
@@ -149,7 +149,7 @@ def _apply_random_mutation(self, instruction: str, response: str) -> str:
         return (
             self.mutation_templates[mutation]
             .replace("<PROMPT>", instruction)
-            .replace("<RESPONSE>", response[-1])
+            .replace("<RESPONSE>", response)
         )
 
     def _evolve_reponses(self, inputs: "StepInput") -> List[List[str]]:

diff --git a/tests/unit/steps/tasks/evol_quality/test_base.py b/tests/unit/steps/tasks/evol_quality/test_base.py
@@ -34,6 +34,18 @@ def test_with_errors(
         EvolQuality(name="task", llm=dummy_llm, num_evolutions=2)
         assert "Step 'task' hasn't received a pipeline" in caplog.text
 
+    def test_apply_random_mutation(self, dummy_llm: LLM) -> None:
+        pipeline = Pipeline(name="unit-test-pipeline")
+        task = EvolQuality(
+            name="task", llm=dummy_llm, num_evolutions=2, pipeline=pipeline
+        )
+        task.load()
+
+        mutated = task._apply_random_mutation("I'm an instruction", "I'm a response")
+
+        assert "I'm an instruction" in mutated
+        assert "I'm a response" in mutated
+
     def test_process(self, dummy_llm: LLM) -> None:
         pipeline = Pipeline(name="unit-test-pipeline")
         task = EvolQuality(