Merge pull request #403 from instructlab/mergify/bp/release-v0.6/pr-400

Add [End] to parser cleanup tags (backport #400)
instructlab · Nov 19, 2024 · f12823e · f12823e
2 parents 4e90549 + 38766ea
commit f12823e
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/instructlab/sdg/pipelines/full/knowledge.yaml b/src/instructlab/sdg/pipelines/full/knowledge.yaml
@@ -43,6 +43,7 @@ blocks:
         parsing_pattern: '\[(?:Question|QUESTION)\]\s*(.*?)\s*\[(?:Answer|ANSWER)\]\s*(.*?)\s*(?=\[(?:Question|QUESTION)\]|$)'
         parser_cleanup_tags:
           - "[END]"
+          - "[End]"
       gen_kwargs:
         max_tokens: 2048
     drop_duplicates: