# Install first: # pip install nltk sentence-transformers faiss-cpu tiktoken from sentence_transformers import SentenceTransformer import nltk import faiss import tiktoken from typing import List import numpy as np nltk.download('punkt') # Load tokenizer for token counting (OpenAI tokenizer works for most BPE encodings) tokenizer = tiktoken.get_encoding("cl100k_base") # Load embedding model model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True) MAX_TOKENS = 350 CHUNK_OVERLAP = 50 def tokenize_text(text): return tokenizer.encode(text) def count_tokens(text): return len(tokenize_text(text)) def split_into_chunks(text: str, max_tokens=MAX_TOKENS, overlap=CHUNK_OVERLAP, with_metadata=True) -> List[dict]: sentences = nltk.sent_tokenize(text) chunks = [] current_chunk = [] current_tokens = 0 chunk_id = 0 for sentence in sentences: sentence_tokens = count_tokens(sentence) if current_tokens + sentence_tokens > max_tokens: if current_chunk: chunk_text = " ".join(current_chunk) if with_metadata: chunks.append({ "chunk_id": chunk_id, "text": chunk_text, "word_count": len(chunk_text.split()), "token_count": count_tokens(chunk_text) }) else: chunks.append(chunk_text) chunk_id += 1 # Overlap handling overlap_sentences = [] overlap_tokens = 0 while current_chunk and overlap_tokens < overlap: s = current_chunk.pop() overlap_sentences.insert(0, s) overlap_tokens += count_tokens(s) current_chunk = overlap_sentences.copy() current_tokens = sum(count_tokens(s) for s in current_chunk) current_chunk.append(sentence) current_tokens += sentence_tokens if current_chunk: chunk_text = " ".join(current_chunk) if with_metadata: chunks.append({ "chunk_id": chunk_id, "text": chunk_text, "word_count": len(chunk_text.split()), "token_count": count_tokens(chunk_text) }) else: chunks.append(chunk_text) return chunks