| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135 |
- """
- Programme principal
- """
- import nltk
- import numpy as np
- import pandas as pd
- from sentence_transformers import SentenceTransformer, CrossEncoder
- from auth import login_to_wiki
- from scraping import fetch_all_pages_content
- from chunking import semantic_chunking, model
- from embedding_index import build_faiss_index, search_faiss
- from qa_chain import get_qa_chain
- import getpass
- import warnings
- warnings.filterwarnings("ignore")
- def get_relevance_label(score):
- if score >= 2.5:
- return "✅ bon"
- elif score >= 0:
- return "⚠️ moyen"
- else:
- return "❌ faible"
- def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
- question_embedding = model.encode([question], convert_to_tensor=False)
- question_embedding = np.array(question_embedding).astype('float32')
- if question_embedding.ndim == 1:
- question_embedding = np.expand_dims(question_embedding, axis=0)
- distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
- candidates = []
- for idx, dist in zip(indices, distances):
- candidates.append({
- "chunk": chunks[idx],
- "url": chunk_urls[idx],
- "index": idx,
- "faiss_score": float(dist)
- })
- # Reranking
- pairs = [[question, c["chunk"]] for c in candidates]
- rerank_scores = cross_encoder.predict(pairs)
- for i in range(len(candidates)):
- candidates[i]["rerank_score"] = float(rerank_scores[i])
- reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
- print("\n🔍 Résultats re-rankés :")
- df = pd.DataFrame([{
- "Index": r["index"],
- "URL": r["url"],
- "Score CrossEncoder": round(r["rerank_score"], 4),
- "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
- } for r in reranked])
- df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
- pd.set_option('display.max_colwidth', None)
- df_reranked = df.copy()
- # Tableau propre
- colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
- df_affichage = df_reranked[colonnes_a_afficher].copy()
- df_affichage.reset_index(drop=True, inplace=True)
- print("\n🎯 Résultats re-rankés (top 10) :")
- print(df_affichage.head(10).to_string(index=True))
- # Si tous les top chunks viennent d'une seule page
- top_urls = [r["url"] for r in reranked]
- unique_top_urls = list(set(top_urls))
- if len(unique_top_urls) == 1:
- print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
- full_context = "\n\n".join([
- chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
- ])
- else:
- full_context = "\n\n".join([r["chunk"] for r in reranked])
- result = qa_chain.run(context=full_context, question=question)
- print(f"\n🤖 Réponse :\n{result}\n")
- if __name__ == "__main__":
- nltk.download('punkt')
- user = input("Entrez votre nom d'utilisateur : ")
- pwd = getpass.getpass("Entrez votre mot de passe (entrée masquée) : ")
- success = login_to_wiki(user, pwd)
- if success:
- wiki_urls = [
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
- "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
- "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
- ]
- url_texts = fetch_all_pages_content(wiki_urls)
- all_chunks = []
- chunk_urls = []
- for url, text in url_texts.items():
- print(f"\n--- Traitement de la page {url} ---")
- sentences = nltk.sent_tokenize(text, language='french')
- chunks = semantic_chunking(sentences)
- print(f"Nombre de chunks : {len(chunks)}")
- for i, chunk in enumerate(chunks):
- print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
- all_chunks.append(chunk)
- chunk_urls.append(url)
- print("\n✅ Génération des embeddings des chunks...")
- chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
- chunk_embeddings = np.array(chunk_embeddings).astype('float32')
- print("\n✅ Construction de l'index FAISS...")
- index = build_faiss_index(chunk_embeddings)
- print("✅ Index FAISS construit avec succès.")
- cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
- qa_chain = get_qa_chain()
- while True:
- question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
- if question.lower() in ["exit", "quit"]:
- break
- ask_question_with_reranking(
- question, model, cross_encoder, index,
- all_chunks, chunk_urls, qa_chain
- )
|