QA_LLM_RAG_Text_to_Speech/weaviate_pdf.py

from langchain.text_splitter import  RecursiveCharacterTextSplitter
from langchain_community.document_loaders import PyPDFLoader
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain_community.vectorstores import Weaviate
import weaviate

client = weaviate.Client(
    url="http://localhost:8081", 
)

model_name = "sentence-transformers/all-mpnet-base-v2"
model_kwargs = {'device': 'cpu'}
encode_kwargs = {'normalize_embeddings': False}

hf = HuggingFaceEmbeddings(
    model_name=model_name,
    model_kwargs=model_kwargs,
    encode_kwargs=encode_kwargs
)

loader = PyPDFLoader("Insurance_Handbook_20103.pdf")
pages = loader.load_and_split()

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=50,
    length_function=len,
    is_separator_regex=False,
)

texts = text_splitter.split_documents(pages)
full_texts = [i.page_content for i in texts]

vector_db = Weaviate.from_texts(
    full_texts,hf, client=client, by_text=False, index_name='BookOfInsurance', text_key='intro'
)

# Test the similarity search query function
# print(vector_db.similarity_search("What is expense ratio?", k=3))
client.close()