|
|
@@ -16,6 +16,12 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
import json
|
|
|
from urllib.parse import urljoin, urlparse
|
|
|
from bs4 import SoupStrainer
|
|
|
+from sklearn.metrics.pairwise import cosine_similarity
|
|
|
+import getpass
|
|
|
+
|
|
|
+#username = input("Nom d'utilisateur : ")
|
|
|
+#password = getpass.getpass("Mot de passe : ")
|
|
|
+#login_to_wiki(username, password)
|
|
|
|
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
|
@@ -66,10 +72,10 @@ def extract_visible_text(html):
|
|
|
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
-def clean_text(text):
|
|
|
- doc = nlp(text)
|
|
|
- tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
|
|
|
- return ' '.join(tokens)
|
|
|
+#def clean_text(text):
|
|
|
+ # doc = nlp(text)
|
|
|
+ # tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
|
|
|
+ # return ' '.join(tokens)
|
|
|
|
|
|
def chunk_by_titles(text):
|
|
|
"""
|
|
|
@@ -97,41 +103,27 @@ def chunk_by_titles(text):
|
|
|
|
|
|
return chunks
|
|
|
|
|
|
-def semantic_chunking(raw_text, max_tokens=200):
|
|
|
- sentences = nltk.sent_tokenize(raw_text, language='french')
|
|
|
- chunks = []
|
|
|
- current_chunk = ""
|
|
|
- current_len = 0
|
|
|
-
|
|
|
- for sentence in sentences:
|
|
|
- sentence_len = len(sentence.split())
|
|
|
- if current_len + sentence_len > max_tokens:
|
|
|
- chunks.append(current_chunk.strip())
|
|
|
- current_chunk = sentence + " "
|
|
|
- current_len = sentence_len
|
|
|
- else:
|
|
|
- current_chunk += sentence + " "
|
|
|
- current_len += sentence_len
|
|
|
-
|
|
|
- if current_chunk:
|
|
|
- chunks.append(current_chunk.strip())
|
|
|
-
|
|
|
- return chunks
|
|
|
-
|
|
|
def fetch_all_pages_content(urls):
|
|
|
url_texts = {}
|
|
|
for url in urls:
|
|
|
try:
|
|
|
- response = session.get(url)
|
|
|
+ edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
|
|
|
+ response = session.get(edit_url)
|
|
|
if response.status_code == 200:
|
|
|
- page_text = extract_visible_text(response.text)
|
|
|
- url_texts[url] = page_text
|
|
|
+ soup = BeautifulSoup(response.text, "html.parser")
|
|
|
+ textarea = soup.find("textarea", {"name": "wikitext"})
|
|
|
+ if textarea:
|
|
|
+ raw_text = textarea.text
|
|
|
+ url_texts[url] = raw_text
|
|
|
+ else:
|
|
|
+ print(f"⚠️ Pas de balise <textarea> trouvée pour {edit_url}.")
|
|
|
else:
|
|
|
- print(f"⚠️ Erreur d'accès à {url} : code {response.status_code}")
|
|
|
+ print(f"⚠️ Erreur d'accès à {edit_url} : code {response.status_code}")
|
|
|
except Exception as e:
|
|
|
print(f"❌ Exception pour {url}: {e}")
|
|
|
return url_texts
|
|
|
|
|
|
+
|
|
|
def build_faiss_index(embeddings):
|
|
|
dimension = embeddings.shape[1]
|
|
|
index = faiss.IndexFlatIP(dimension)
|
|
|
@@ -144,9 +136,21 @@ def search_faiss(index, query_embedding, top_k=10):
|
|
|
distances, indices = index.search(query_embedding, top_k)
|
|
|
return distances[0], indices[0]
|
|
|
|
|
|
-def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=20, rerank_top_k=5):
|
|
|
+
|
|
|
+def get_relevance_label(score):
|
|
|
+ if score >= 2.5:
|
|
|
+ return "✅ bon"
|
|
|
+ elif score >= 0:
|
|
|
+ return "⚠️ moyen"
|
|
|
+ else:
|
|
|
+ return "❌ faible"
|
|
|
+
|
|
|
+
|
|
|
+def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
|
|
|
question_embedding = model.encode([question], convert_to_tensor=False)
|
|
|
question_embedding = np.array(question_embedding).astype('float32')
|
|
|
+ if question_embedding.ndim == 1:
|
|
|
+ question_embedding = np.expand_dims(question_embedding, axis=0)
|
|
|
|
|
|
distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
|
|
|
|
|
|
@@ -159,14 +163,12 @@ def ask_question_with_reranking(question, model, cross_encoder, index, chunks, c
|
|
|
"faiss_score": float(dist)
|
|
|
})
|
|
|
|
|
|
- # Re-ranking avec CrossEncoder
|
|
|
+ # Reranking
|
|
|
pairs = [[question, c["chunk"]] for c in candidates]
|
|
|
rerank_scores = cross_encoder.predict(pairs)
|
|
|
-
|
|
|
for i in range(len(candidates)):
|
|
|
candidates[i]["rerank_score"] = float(rerank_scores[i])
|
|
|
|
|
|
- # Tri final selon score CrossEncoder
|
|
|
reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
|
|
|
|
|
|
print("\n🔍 Résultats re-rankés :")
|
|
|
@@ -176,15 +178,72 @@ def ask_question_with_reranking(question, model, cross_encoder, index, chunks, c
|
|
|
"Score CrossEncoder": round(r["rerank_score"], 4),
|
|
|
"Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
|
|
|
} for r in reranked])
|
|
|
- print(df)
|
|
|
+ df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
|
|
|
+ pd.set_option('display.max_colwidth', None)
|
|
|
+
|
|
|
+ df_reranked = df.copy()
|
|
|
+
|
|
|
+ # Affichage formaté ligne par ligne
|
|
|
+ for i, row in df_reranked.iterrows():
|
|
|
+ print(f"{i:>3} {row['URL']:<70} {row['Score CrossEncoder']:.4f} {row['Pertinence']}")
|
|
|
+ print(f"{row['Chunk'][:150]}...\n")
|
|
|
+
|
|
|
+ # Tableau propre
|
|
|
+ colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
|
|
|
+ df_affichage = df_reranked[colonnes_a_afficher].copy()
|
|
|
+ df_affichage.reset_index(drop=True, inplace=True)
|
|
|
+
|
|
|
+ print("\n🎯 Résultats re-rankés (top 10) :")
|
|
|
+ print(df_affichage.head(10).to_string(index=True))
|
|
|
+
|
|
|
+ # Si tous les top chunks viennent d'une seule page
|
|
|
+ top_urls = [r["url"] for r in reranked]
|
|
|
+ unique_top_urls = list(set(top_urls))
|
|
|
+
|
|
|
+ if len(unique_top_urls) == 1:
|
|
|
+ print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
|
|
|
+ full_context = "\n\n".join([
|
|
|
+ chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
|
|
|
+ ])
|
|
|
+ else:
|
|
|
+ full_context = "\n\n".join([r["chunk"] for r in reranked])
|
|
|
|
|
|
- context = "\n\n".join([r["chunk"] for r in reranked])
|
|
|
- result = qa_chain.run(context=context, question=question)
|
|
|
+ result = qa_chain.run(context=full_context, question=question)
|
|
|
print(f"\n🤖 Réponse :\n{result}\n")
|
|
|
|
|
|
|
|
|
+
|
|
|
+def semantic_chunking(sentences, similarity_threshold=0.5):
|
|
|
+ chunks = []
|
|
|
+ current_chunk = []
|
|
|
+ current_embeddings = []
|
|
|
+
|
|
|
+ for sentence in sentences:
|
|
|
+ embedding = model.encode(sentence)
|
|
|
+ if not current_embeddings:
|
|
|
+ current_chunk.append(sentence)
|
|
|
+ current_embeddings.append(embedding)
|
|
|
+ else:
|
|
|
+ similarities = cosine_similarity([embedding], current_embeddings)[0]
|
|
|
+ if max(similarities) > similarity_threshold:
|
|
|
+ current_chunk.append(sentence)
|
|
|
+ current_embeddings.append(embedding)
|
|
|
+ else:
|
|
|
+ chunks.append(" ".join(current_chunk))
|
|
|
+ current_chunk = [sentence]
|
|
|
+ current_embeddings = [embedding]
|
|
|
+
|
|
|
+ if current_chunk:
|
|
|
+ chunks.append(" ".join(current_chunk))
|
|
|
+
|
|
|
+ return chunks
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
if __name__ == "__main__":
|
|
|
- if login_to_wiki("", ""):
|
|
|
+ model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')
|
|
|
+
|
|
|
+ if login_to_wiki("nadine.elleuch", "witpowgeu!75"):
|
|
|
wiki_urls = [
|
|
|
"https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
"https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
@@ -199,52 +258,50 @@ if __name__ == "__main__":
|
|
|
|
|
|
for url, text in url_texts.items():
|
|
|
print(f"\n--- Traitement de la page {url} ---")
|
|
|
- chunks = chunk_by_titles(text)
|
|
|
- print(f"Nombre de chunks par titres : {len(chunks)}")
|
|
|
-
|
|
|
- # Nettoyage + découpage en sous-chunks si trop long
|
|
|
- cleaned_chunks = []
|
|
|
- for chunk in chunks:
|
|
|
- cleaned = clean_text(chunk)
|
|
|
- if len(cleaned.split()) > 200:
|
|
|
- subchunks = semantic_chunking(cleaned, max_tokens=200)
|
|
|
- cleaned_chunks.extend(subchunks)
|
|
|
- else:
|
|
|
- cleaned_chunks.append(cleaned)
|
|
|
|
|
|
- for i, c in enumerate(cleaned_chunks):
|
|
|
- print(f"Chunk {i+1} ({len(c.split())} mots): {c[:100]}{'...' if len(c) > 100 else ''}")
|
|
|
- all_chunks.append(c)
|
|
|
+ # Split the text into sentences
|
|
|
+ sentences = nltk.sent_tokenize(text, language='french')
|
|
|
+
|
|
|
+ # Semantic chunking using similarity
|
|
|
+ chunks = semantic_chunking(sentences)
|
|
|
+
|
|
|
+ print(f"Nombre de chunks : {len(chunks)}")
|
|
|
+
|
|
|
+ for i, chunk in enumerate(chunks):
|
|
|
+ print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
|
|
|
+ all_chunks.append(chunk)
|
|
|
chunk_urls.append(url)
|
|
|
- import json
|
|
|
|
|
|
- chunk_data = [
|
|
|
- {"url": url, "chunk_index": i+1, "chunk": chunk}
|
|
|
- for i, (url, chunk) in enumerate(zip(chunk_urls, all_chunks))
|
|
|
- ]
|
|
|
+ # Prepare chunk metadata
|
|
|
+ chunk_data = [
|
|
|
+ {"url": url_, "chunk_index": i + 1, "chunk": chunk}
|
|
|
+ for i, (url_, chunk) in enumerate(zip(chunk_urls, all_chunks))
|
|
|
+ ]
|
|
|
|
|
|
- with open("chunks_pretty.json", "w", encoding="utf-8") as f:
|
|
|
- json.dump(chunk_data, f, ensure_ascii=False, indent=2)
|
|
|
+ with open("chunks_pretty.json", "w", encoding="utf-8") as f:
|
|
|
+ json.dump(chunk_data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
- print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
|
|
|
+ print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
|
|
|
|
|
|
- model = SentenceTransformer('all-mpnet-base-v2')
|
|
|
+ # Load reranker
|
|
|
cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
|
|
|
+ # Encode the chunks
|
|
|
print("\n✅ Génération des embeddings des chunks...")
|
|
|
chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
|
|
|
chunk_embeddings = np.array(chunk_embeddings).astype('float32')
|
|
|
-
|
|
|
+
|
|
|
+ # Build FAISS index
|
|
|
print("\n✅ Construction de l'index FAISS...")
|
|
|
index = build_faiss_index(chunk_embeddings)
|
|
|
print("✅ Index FAISS construit avec succès.")
|
|
|
-
|
|
|
+
|
|
|
+ # Define prompt
|
|
|
prompt_template = PromptTemplate(
|
|
|
input_variables=["context", "question"],
|
|
|
template="""
|
|
|
-Tu es un assistant expert du contenu du wiki interne d'Intersel. Ta tâche est de répondre à la question en utilisant uniquement les informations pertinentes du contexte fourni.
|
|
|
-
|
|
|
-Si l'information existe, donne une réponse claire, concise et précise.
|
|
|
+Tu es un assistant expert du wiki interne d'Intersel. Réponds uniquement à partir du CONTEXTE donné.
|
|
|
+Si aucune information pertinente n'est disponible, dis "Je ne sais pas." sans inventer.
|
|
|
|
|
|
CONTEXTE :
|
|
|
{context}
|
|
|
@@ -263,5 +320,7 @@ RÉPONSE :
|
|
|
question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
|
|
|
if question.lower() in ["exit", "quit"]:
|
|
|
break
|
|
|
- ask_question_with_reranking(question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain)
|
|
|
-
|
|
|
+ ask_question_with_reranking(
|
|
|
+ question, model, cross_encoder, index,
|
|
|
+ all_chunks, chunk_urls, qa_chain
|
|
|
+ )
|