| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465 |
- import nltk
- from sklearn.metrics.pairwise import cosine_similarity
- from sentence_transformers import SentenceTransformer
- import numpy as np
- import re
- from typing import List
- nltk.download('punkt')
- model = SentenceTransformer('all-MiniLM-L6-v2')
- MIN_WORDS_PER_CHUNK = 10
- def semantic_chunking(sentences, similarity_threshold=0.8, max_words_per_chunk=200):
- #initialisation des listes
- chunks = []
- current_chunk = []
- current_embeddings = []
- # encoder toutes les phrases en vecteurs avec le modèle SentenceTransformer
- sentence_embeddings = model.encode(sentences)
- # parcourir chaque phrase et son embedding associé
- for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)):
- sentence = sentence.strip()
- if not sentence:
- continue # ignorer les phrases vides
- word_count = len(sentence.split())
- # Cas de phrase courte qui pourrait être un titre ("Gérant", "Responsable", etc.)
- if word_count <= 3 and i > 0:
- # Ajoute à la phrase précédente (cas comme "Gérante")
- current_chunk.append(sentence)
- current_embeddings.append(embedding)
- continue
- # Si le chunk actuel est vide, on l'initialise avec la phrase actuelle
- if not current_embeddings:
- current_chunk.append(sentence)
- current_embeddings.append(embedding)
- else:
- # calculer la similarité entre la phrase courante et le chunk en cours (moyenne des embeddings)
- mean_embedding = np.mean(current_embeddings, axis=0)
- sim = cosine_similarity([embedding], [mean_embedding])[0][0]
- if sim > similarity_threshold and len(" ".join(current_chunk).split()) < max_words_per_chunk:
- current_chunk.append(sentence)
- current_embeddings.append(embedding)
- else:
- chunk_text = " ".join(current_chunk)
- if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
- chunks.append(chunk_text)
- current_chunk = [sentence]
- current_embeddings = [embedding]
- if current_chunk:
- # Sinon on ferme le chunk en cours
- chunk_text = " ".join(current_chunk)
- if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK:
- chunks.append(chunk_text)
- if not chunks:
- print("⚠️ Aucun chunk généré. Vérifie la structure ou le contenu des phrases.")
- return chunks
|