|
@@ -0,0 +1,135 @@
|
|
|
|
|
+"""
|
|
|
|
|
+Programme principal
|
|
|
|
|
+"""
|
|
|
|
|
+
|
|
|
|
|
+import nltk
|
|
|
|
|
+import numpy as np
|
|
|
|
|
+import pandas as pd
|
|
|
|
|
+from sentence_transformers import SentenceTransformer, CrossEncoder
|
|
|
|
|
+from auth import login_to_wiki
|
|
|
|
|
+from scraping import fetch_all_pages_content
|
|
|
|
|
+from chunking import semantic_chunking, model
|
|
|
|
|
+from embedding_index import build_faiss_index, search_faiss
|
|
|
|
|
+from qa_chain import get_qa_chain
|
|
|
|
|
+import getpass
|
|
|
|
|
+import warnings
|
|
|
|
|
+warnings.filterwarnings("ignore")
|
|
|
|
|
+
|
|
|
|
|
+def get_relevance_label(score):
|
|
|
|
|
+ if score >= 2.5:
|
|
|
|
|
+ return "✅ bon"
|
|
|
|
|
+ elif score >= 0:
|
|
|
|
|
+ return "⚠️ moyen"
|
|
|
|
|
+ else:
|
|
|
|
|
+ return "❌ faible"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
|
|
|
|
|
+ question_embedding = model.encode([question], convert_to_tensor=False)
|
|
|
|
|
+ question_embedding = np.array(question_embedding).astype('float32')
|
|
|
|
|
+ if question_embedding.ndim == 1:
|
|
|
|
|
+ question_embedding = np.expand_dims(question_embedding, axis=0)
|
|
|
|
|
+
|
|
|
|
|
+ distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
|
|
|
|
|
+
|
|
|
|
|
+ candidates = []
|
|
|
|
|
+ for idx, dist in zip(indices, distances):
|
|
|
|
|
+ candidates.append({
|
|
|
|
|
+ "chunk": chunks[idx],
|
|
|
|
|
+ "url": chunk_urls[idx],
|
|
|
|
|
+ "index": idx,
|
|
|
|
|
+ "faiss_score": float(dist)
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ # Reranking
|
|
|
|
|
+ pairs = [[question, c["chunk"]] for c in candidates]
|
|
|
|
|
+ rerank_scores = cross_encoder.predict(pairs)
|
|
|
|
|
+ for i in range(len(candidates)):
|
|
|
|
|
+ candidates[i]["rerank_score"] = float(rerank_scores[i])
|
|
|
|
|
+
|
|
|
|
|
+ reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
|
|
|
|
|
+
|
|
|
|
|
+ print("\n🔍 Résultats re-rankés :")
|
|
|
|
|
+ df = pd.DataFrame([{
|
|
|
|
|
+ "Index": r["index"],
|
|
|
|
|
+ "URL": r["url"],
|
|
|
|
|
+ "Score CrossEncoder": round(r["rerank_score"], 4),
|
|
|
|
|
+ "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
|
|
|
|
|
+ } for r in reranked])
|
|
|
|
|
+ df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
|
|
|
|
|
+ pd.set_option('display.max_colwidth', None)
|
|
|
|
|
+
|
|
|
|
|
+ df_reranked = df.copy()
|
|
|
|
|
+
|
|
|
|
|
+ # Tableau propre
|
|
|
|
|
+ colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
|
|
|
|
|
+ df_affichage = df_reranked[colonnes_a_afficher].copy()
|
|
|
|
|
+ df_affichage.reset_index(drop=True, inplace=True)
|
|
|
|
|
+
|
|
|
|
|
+ print("\n🎯 Résultats re-rankés (top 10) :")
|
|
|
|
|
+ print(df_affichage.head(10).to_string(index=True))
|
|
|
|
|
+
|
|
|
|
|
+ # Si tous les top chunks viennent d'une seule page
|
|
|
|
|
+ top_urls = [r["url"] for r in reranked]
|
|
|
|
|
+ unique_top_urls = list(set(top_urls))
|
|
|
|
|
+
|
|
|
|
|
+ if len(unique_top_urls) == 1:
|
|
|
|
|
+ print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
|
|
|
|
|
+ full_context = "\n\n".join([
|
|
|
|
|
+ chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
|
|
|
|
|
+ ])
|
|
|
|
|
+ else:
|
|
|
|
|
+ full_context = "\n\n".join([r["chunk"] for r in reranked])
|
|
|
|
|
+
|
|
|
|
|
+ result = qa_chain.run(context=full_context, question=question)
|
|
|
|
|
+ print(f"\n🤖 Réponse :\n{result}\n")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+if __name__ == "__main__":
|
|
|
|
|
+ nltk.download('punkt')
|
|
|
|
|
+ user = input("Entrez votre nom d'utilisateur : ")
|
|
|
|
|
+ pwd = getpass.getpass("Entrez votre mot de passe (entrée masquée) : ")
|
|
|
|
|
+ success = login_to_wiki(user, pwd)
|
|
|
|
|
+ if success:
|
|
|
|
|
+ wiki_urls = [
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ url_texts = fetch_all_pages_content(wiki_urls)
|
|
|
|
|
+
|
|
|
|
|
+ all_chunks = []
|
|
|
|
|
+ chunk_urls = []
|
|
|
|
|
+
|
|
|
|
|
+ for url, text in url_texts.items():
|
|
|
|
|
+ print(f"\n--- Traitement de la page {url} ---")
|
|
|
|
|
+ sentences = nltk.sent_tokenize(text, language='french')
|
|
|
|
|
+ chunks = semantic_chunking(sentences)
|
|
|
|
|
+ print(f"Nombre de chunks : {len(chunks)}")
|
|
|
|
|
+ for i, chunk in enumerate(chunks):
|
|
|
|
|
+ print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
|
|
|
|
|
+ all_chunks.append(chunk)
|
|
|
|
|
+ chunk_urls.append(url)
|
|
|
|
|
+
|
|
|
|
|
+ print("\n✅ Génération des embeddings des chunks...")
|
|
|
|
|
+ chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
|
|
|
|
|
+ chunk_embeddings = np.array(chunk_embeddings).astype('float32')
|
|
|
|
|
+
|
|
|
|
|
+ print("\n✅ Construction de l'index FAISS...")
|
|
|
|
|
+ index = build_faiss_index(chunk_embeddings)
|
|
|
|
|
+ print("✅ Index FAISS construit avec succès.")
|
|
|
|
|
+
|
|
|
|
|
+ cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
|
|
+ qa_chain = get_qa_chain()
|
|
|
|
|
+
|
|
|
|
|
+ while True:
|
|
|
|
|
+ question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
|
|
|
|
|
+ if question.lower() in ["exit", "quit"]:
|
|
|
|
|
+ break
|
|
|
|
|
+ ask_question_with_reranking(
|
|
|
|
|
+ question, model, cross_encoder, index,
|
|
|
|
|
+ all_chunks, chunk_urls, qa_chain
|
|
|
|
|
+ )
|