| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677 |
- # Install first:
- # pip install nltk sentence-transformers faiss-cpu tiktoken
- from sentence_transformers import SentenceTransformer
- import nltk
- import faiss
- import tiktoken
- from typing import List
- import numpy as np
- nltk.download('punkt')
- # Load tokenizer for token counting (OpenAI tokenizer works for most BPE encodings)
- tokenizer = tiktoken.get_encoding("cl100k_base")
- # Load embedding model
- model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True)
- MAX_TOKENS = 350
- CHUNK_OVERLAP = 50
- def tokenize_text(text):
- return tokenizer.encode(text)
- def count_tokens(text):
- return len(tokenize_text(text))
- def split_into_chunks(text: str, max_tokens=MAX_TOKENS, overlap=CHUNK_OVERLAP, with_metadata=True) -> List[dict]:
- sentences = nltk.sent_tokenize(text)
- chunks = []
- current_chunk = []
- current_tokens = 0
- chunk_id = 0
- for sentence in sentences:
- sentence_tokens = count_tokens(sentence)
- if current_tokens + sentence_tokens > max_tokens:
- if current_chunk:
- chunk_text = " ".join(current_chunk)
- if with_metadata:
- chunks.append({
- "chunk_id": chunk_id,
- "text": chunk_text,
- "word_count": len(chunk_text.split()),
- "token_count": count_tokens(chunk_text)
- })
- else:
- chunks.append(chunk_text)
- chunk_id += 1
- # Overlap handling
- overlap_sentences = []
- overlap_tokens = 0
- while current_chunk and overlap_tokens < overlap:
- s = current_chunk.pop()
- overlap_sentences.insert(0, s)
- overlap_tokens += count_tokens(s)
- current_chunk = overlap_sentences.copy()
- current_tokens = sum(count_tokens(s) for s in current_chunk)
- current_chunk.append(sentence)
- current_tokens += sentence_tokens
- if current_chunk:
- chunk_text = " ".join(current_chunk)
- if with_metadata:
- chunks.append({
- "chunk_id": chunk_id,
- "text": chunk_text,
- "word_count": len(chunk_text.split()),
- "token_count": count_tokens(chunk_text)
- })
- else:
- chunks.append(chunk_text)
- return chunks
|