resolve error that make_single_content_qa generate duplicate qid (#562)

Co-authored-by: jeffrey <[email protected]>
Marker-Inc-Korea · Jul 8, 2024 · 10c48b8 · 10c48b8
1 parent e018191
commit 10c48b8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/autorag/data/qacreation/base.py b/autorag/data/qacreation/base.py
@@ -46,11 +46,11 @@ def make_single_content_qa(corpus_df: pd.DataFrame,
 
     qa = qa_creation_func(contents=sampled_corpus['contents'].tolist(), **kwargs)
     qa_data = pd.DataFrame({
-        'qid': [str(uuid.uuid4()) for _ in range(len(qa))],
         'qa': qa,
         'retrieval_gt': sampled_corpus['doc_id'].tolist(),
     })
     qa_data = qa_data.explode('qa', ignore_index=True)
+    qa_data['qid'] = [str(uuid.uuid4()) for _ in range(len(qa_data))]
 
     def make_query_generation_gt(row):
         return row['qa']['query'], row['qa']['generation_gt']

diff --git a/tests/autorag/data/qacreation/test_base_qacreation.py b/tests/autorag/data/qacreation/test_base_qacreation.py
@@ -48,6 +48,7 @@ def test_single_content_qa(qa_parquet_filepath):
         upsert=True,
     )
     validate_qa_dataset(qa_df)
+    assert len(qa_df) == qa_df['qid'].nunique()
     assert len(qa_df) == 6
     assert qa_df['retrieval_gt'].tolist()[0] == qa_df['retrieval_gt'].tolist()[1]