chunking.py 2.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. import nltk
  2. from sklearn.metrics.pairwise import cosine_similarity
  3. from sentence_transformers import SentenceTransformer
  4. import numpy as np
  5. import re
  6. from typing import List
  7. nltk.download('punkt')
  8. model = SentenceTransformer('all-MiniLM-L6-v2')
  9. MIN_WORDS_PER_CHUNK = 10
  10. def semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=150):
  11. chunks = []
  12. current_chunk = []
  13. current_embeddings = []
  14. sentence_embeddings = model.encode(sentences)
  15. for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
  16. sentence = sentence.strip()
  17. if not sentence:
  18. continue
  19. word_count = len(sentence.split())
  20. # Cas de phrase courte qui pourrait être un titre ("Gérant", "Responsable", etc.)
  21. if word_count <= 3 and i > 0:
  22. # Ajoute à la phrase précédente (cas comme "Gérante")
  23. current_chunk.append(sentence)
  24. current_embeddings.append(embedding)
  25. continue
  26. if not current_embeddings:
  27. current_chunk.append(sentence)
  28. current_embeddings.append(embedding)
  29. else:
  30. mean_embedding = np.mean(current_embeddings, axis=0)
  31. sim = cosine_similarity([embedding], [mean_embedding])[0][0]
  32. if sim > similarity_threshold and len(" ".join(current_chunk).split()) < max_words_per_chunk:
  33. current_chunk.append(sentence)
  34. current_embeddings.append(embedding)
  35. else:
  36. chunk_text = " ".join(current_chunk)
  37. if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
  38. chunks.append(chunk_text)
  39. current_chunk = [sentence]
  40. current_embeddings = [embedding]
  41. if current_chunk:
  42. chunk_text = " ".join(current_chunk)
  43. if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
  44. chunks.append(chunk_text)
  45. if not chunks:
  46. print("⚠️ Aucun chunk généré. Vérifie la structure ou le contenu des phrases.")
  47. return chunks