""" Programme principal """ import nltk import numpy as np import pandas as pd from sentence_transformers import SentenceTransformer, CrossEncoder from auth import login_to_wiki from scraping import fetch_all_pages_content from chunking import semantic_chunking, model from embedding_index import build_faiss_index, search_faiss from qa_chain import get_qa_chain import getpass import warnings warnings.filterwarnings("ignore") def get_relevance_label(score): if score >= 2.5: return "✅ bon" elif score >= 0: return "⚠️ moyen" else: return "❌ faible" def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10): question_embedding = model.encode([question], convert_to_tensor=False) question_embedding = np.array(question_embedding).astype('float32') if question_embedding.ndim == 1: question_embedding = np.expand_dims(question_embedding, axis=0) distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k) candidates = [] for idx, dist in zip(indices, distances): candidates.append({ "chunk": chunks[idx], "url": chunk_urls[idx], "index": idx, "faiss_score": float(dist) }) # Reranking pairs = [[question, c["chunk"]] for c in candidates] rerank_scores = cross_encoder.predict(pairs) for i in range(len(candidates)): candidates[i]["rerank_score"] = float(rerank_scores[i]) reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k] print("\n🔍 Résultats re-rankés :") df = pd.DataFrame([{ "Index": r["index"], "URL": r["url"], "Score CrossEncoder": round(r["rerank_score"], 4), "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"] } for r in reranked]) df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label) pd.set_option('display.max_colwidth', None) df_reranked = df.copy() # Tableau propre colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk'] df_affichage = df_reranked[colonnes_a_afficher].copy() df_affichage.reset_index(drop=True, inplace=True) print("\n🎯 Résultats re-rankés (top 10) :") print(df_affichage.head(10).to_string(index=True)) # Si tous les top chunks viennent d'une seule page top_urls = [r["url"] for r in reranked] unique_top_urls = list(set(top_urls)) if len(unique_top_urls) == 1: print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.") full_context = "\n\n".join([ chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0] ]) else: full_context = "\n\n".join([r["chunk"] for r in reranked]) result = qa_chain.run(context=full_context, question=question) print(f"\n🤖 Réponse :\n{result}\n") if __name__ == "__main__": nltk.download('punkt') user = input("Entrez votre nom d'utilisateur : ") pwd = getpass.getpass("Entrez votre mot de passe (entrée masquée) : ") success = login_to_wiki(user, pwd) if success: wiki_urls = [ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe", "https://wiki.intersel.fr/doku.php?id=intersel:presentation", "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires", "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start" ] url_texts = fetch_all_pages_content(wiki_urls) all_chunks = [] chunk_urls = [] for url, text in url_texts.items(): print(f"\n--- Traitement de la page {url} ---") sentences = nltk.sent_tokenize(text, language='french') chunks = semantic_chunking(sentences) print(f"Nombre de chunks : {len(chunks)}") for i, chunk in enumerate(chunks): print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}") all_chunks.append(chunk) chunk_urls.append(url) print("\n✅ Génération des embeddings des chunks...") chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True) chunk_embeddings = np.array(chunk_embeddings).astype('float32') print("\n✅ Construction de l'index FAISS...") index = build_faiss_index(chunk_embeddings) print("✅ Index FAISS construit avec succès.") cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') qa_chain = get_qa_chain() while True: question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ") if question.lower() in ["exit", "quit"]: break ask_question_with_reranking( question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain )