chunking.py 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768
  1. import nltk
  2. from sklearn.metrics.pairwise import cosine_similarity
  3. from sentence_transformers import SentenceTransformer
  4. import numpy as np
  5. import re
  6. from typing import List
  7. nltk.download('punkt')
  8. model = SentenceTransformer('all-MiniLM-L6-v2')
  9. MIN_WORDS_PER_CHUNK = 10
  10. def semantic_chunking(sentences, similarity_threshold=0.8, max_words_per_chunk=200):
  11. #initialisation des listes
  12. chunks = []
  13. current_chunk = []
  14. current_embeddings = []
  15. # encoder toutes les phrases en vecteurs avec le modèle SentenceTransformer
  16. sentence_embeddings = model.encode(sentences)
  17. # parcourir chaque phrase et son embedding associé
  18. for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
  19. sentence = sentence.strip()
  20. if not sentence:
  21. continue # ignorer les phrases vides
  22. word_count = len(sentence.split())
  23. # Cas de phrase courte qui pourrait être un titre ("Gérant", "Responsable", etc.)
  24. if word_count <= 3 and i > 0:
  25. # Ajoute à la phrase précédente (cas comme "Gérante")
  26. current_chunk.append(sentence)
  27. current_embeddings.append(embedding)
  28. continue
  29. # Si le chunk actuel est vide, on l'initialise avec la phrase actuelle
  30. if not current_embeddings:
  31. current_chunk.append(sentence)
  32. current_embeddings.append(embedding)
  33. else:
  34. # calculer la similarité entre la phrase courante et le chunk en cours (moyenne des embeddings)
  35. mean_embedding = np.mean(current_embeddings, axis=0)
  36. sim = cosine_similarity([embedding], [mean_embedding])[0][0]
  37. if sim > similarity_threshold and len(" ".join(current_chunk).split()) < max_words_per_chunk:
  38. current_chunk.append(sentence)
  39. current_embeddings.append(embedding)
  40. else:
  41. chunk_text = " ".join(current_chunk)
  42. if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
  43. chunks.append(chunk_text)
  44. current_chunk = [sentence]
  45. current_embeddings = [embedding]
  46. if current_chunk:
  47. # Sinon on ferme le chunk en cours
  48. chunk_text = " ".join(current_chunk)
  49. if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
  50. chunks.append(chunk_text)
  51. if not chunks:
  52. print("⚠️ Aucun chunk généré. Vérifie la structure ou le contenu des phrases.")
  53. return chunks