chunking.py 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677
  1. # Install first:
  2. # pip install nltk sentence-transformers faiss-cpu tiktoken
  3. from sentence_transformers import SentenceTransformer
  4. import nltk
  5. import faiss
  6. import tiktoken
  7. from typing import List
  8. import numpy as np
  9. nltk.download('punkt')
  10. # Load tokenizer for token counting (OpenAI tokenizer works for most BPE encodings)
  11. tokenizer = tiktoken.get_encoding("cl100k_base")
  12. # Load embedding model
  13. model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True)
  14. MAX_TOKENS = 350
  15. CHUNK_OVERLAP = 50
  16. def tokenize_text(text):
  17. return tokenizer.encode(text)
  18. def count_tokens(text):
  19. return len(tokenize_text(text))
  20. def split_into_chunks(text: str, max_tokens=MAX_TOKENS, overlap=CHUNK_OVERLAP, with_metadata=True) -> List[dict]:
  21. sentences = nltk.sent_tokenize(text)
  22. chunks = []
  23. current_chunk = []
  24. current_tokens = 0
  25. chunk_id = 0
  26. for sentence in sentences:
  27. sentence_tokens = count_tokens(sentence)
  28. if current_tokens + sentence_tokens > max_tokens:
  29. if current_chunk:
  30. chunk_text = " ".join(current_chunk)
  31. if with_metadata:
  32. chunks.append({
  33. "chunk_id": chunk_id,
  34. "text": chunk_text,
  35. "word_count": len(chunk_text.split()),
  36. "token_count": count_tokens(chunk_text)
  37. })
  38. else:
  39. chunks.append(chunk_text)
  40. chunk_id += 1
  41. # Overlap handling
  42. overlap_sentences = []
  43. overlap_tokens = 0
  44. while current_chunk and overlap_tokens < overlap:
  45. s = current_chunk.pop()
  46. overlap_sentences.insert(0, s)
  47. overlap_tokens += count_tokens(s)
  48. current_chunk = overlap_sentences.copy()
  49. current_tokens = sum(count_tokens(s) for s in current_chunk)
  50. current_chunk.append(sentence)
  51. current_tokens += sentence_tokens
  52. if current_chunk:
  53. chunk_text = " ".join(current_chunk)
  54. if with_metadata:
  55. chunks.append({
  56. "chunk_id": chunk_id,
  57. "text": chunk_text,
  58. "word_count": len(chunk_text.split()),
  59. "token_count": count_tokens(chunk_text)
  60. })
  61. else:
  62. chunks.append(chunk_text)
  63. return chunks