-
Notifications
You must be signed in to change notification settings - Fork 0
/
__helper__.py
30 lines (21 loc) · 883 Bytes
/
__helper__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
from langchain.document_loaders import PyPDFLoader, DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
# Extract data from the PDF
def load_pdf(data):
loader = DirectoryLoader(data,
glob="*.pdf",
loader_cls=PyPDFLoader)
documents = loader.load()
return documents
# Create text chunks
def text_split(extracted_data):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=500, chunk_overlap=20)
text_chunks = text_splitter.split_documents(extracted_data)
return text_chunks
# download embedding model
def download_hugging_face_embeddings():
embeddings = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2")
return embeddings