Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove unnecessary requirement for qna.yaml in ContextAwareChunker #351

Merged
merged 1 commit into from
Nov 8, 2024
Merged

Remove unnecessary requirement for qna.yaml in ContextAwareChunker #351

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 1 addition & 22 deletions src/instructlab/sdg/utils/chunkers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from collections import defaultdict
from enum import Enum
from pathlib import Path
from typing import DefaultDict, Iterable, List, Optional, Tuple
from typing import DefaultDict, Iterable, List, Tuple
import json
import logging
import re
Expand All @@ -22,7 +22,6 @@
from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
from tabulate import tabulate
from transformers import AutoTokenizer
import yaml

logger = logging.getLogger(__name__)
_DEFAULT_CHUNK_OVERLAP = 100
Expand Down Expand Up @@ -92,7 +91,6 @@ def __new__(
)

filepaths = leaf_node[0]["filepaths"]
leaf_node_path = Path(leaf_node[0]["taxonomy_path"].replace("->", "/"))

doc_dict = cls._split_docs_by_filetype(documents, filepaths)
if len(doc_dict.keys()) > 1:
Expand All @@ -112,7 +110,6 @@ def __new__(
return ContextAwareChunker(
doc_paths,
filepaths,
taxonomy_path / leaf_node_path / "qna.yaml",
output_dir,
chunk_word_count,
tokenizer_model_name,
Expand Down Expand Up @@ -187,24 +184,19 @@ def __init__(
self,
document_paths,
filepaths,
leaf_node_path,
output_dir: Path,
chunk_word_count: int,
tokenizer_model_name="mistralai/Mixtral-8x7B-Instruct-v0.1",
):
self.document_paths = document_paths
self.filepaths = filepaths
self.leaf_node_path = leaf_node_path
self.output_dir = self._path_validator(output_dir)
self.chunk_word_count = chunk_word_count
self.tokenizer_model_name = (
tokenizer_model_name
if tokenizer_model_name is not None
else "mistralai/Mixtral-8x7B-Instruct-v0.1"
)
self.qna_yaml = self._load_qna_yaml(
self._path_validator(leaf_node_path) if leaf_node_path else None
)

self.tokenizer = self.create_tokenizer(tokenizer_model_name)

Expand Down Expand Up @@ -249,19 +241,6 @@ def _path_validator(self, path) -> Path:
raise FileNotFoundError(f"{path} does not exist.")
return path

def _load_qna_yaml(self, qna_yaml_path: Optional[Path]) -> dict:
"""
Load the qna YAML file.
Args:
qna_yaml_path (Path): Path to the knowledge qna YAML file.
Returns:
dict: Dictionary corresponding to knowledge qna YAML file.
"""
if qna_yaml_path:
with open(qna_yaml_path, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
return {}

def _process_parsed_docling_json(self, json_fp: Path) -> Dataset:
"""
Process the parsed docling json file and return a dataset.
Expand Down
Loading