import nltk from sklearn.metrics.pairwise import cosine_similarity from sentence_transformers import SentenceTransformer import numpy as np import re from typing import List nltk.download('punkt') model = SentenceTransformer('all-MiniLM-L6-v2') MIN_WORDS_PER_CHUNK = 10 def semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=150): chunks = [] current_chunk = [] current_embeddings = [] sentence_embeddings = model.encode(sentences) for i, (sentence, embedding) in enumerate(zip(sentences, sentence_embeddings)): sentence = sentence.strip() if not sentence: continue word_count = len(sentence.split()) # Cas de phrase courte qui pourrait être un titre ("Gérant", "Responsable", etc.) if word_count <= 3 and i > 0: # Ajoute à la phrase précédente (cas comme "Gérante") current_chunk.append(sentence) current_embeddings.append(embedding) continue if not current_embeddings: current_chunk.append(sentence) current_embeddings.append(embedding) else: mean_embedding = np.mean(current_embeddings, axis=0) sim = cosine_similarity([embedding], [mean_embedding])[0][0] if sim > similarity_threshold and len(" ".join(current_chunk).split()) < max_words_per_chunk: current_chunk.append(sentence) current_embeddings.append(embedding) else: chunk_text = " ".join(current_chunk) if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK: chunks.append(chunk_text) current_chunk = [sentence] current_embeddings = [embedding] if current_chunk: chunk_text = " ".join(current_chunk) if len(chunk_text.split()) >= MIN_WORDS_PER_CHUNK: chunks.append(chunk_text) if not chunks: print("⚠️ Aucun chunk généré. Vérifie la structure ou le contenu des phrases.") return chunks