main.py 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135
  1. """
  2. Programme principal
  3. """
  4. import nltk
  5. import numpy as np
  6. import pandas as pd
  7. from sentence_transformers import SentenceTransformer, CrossEncoder
  8. from auth import login_to_wiki
  9. from scraping import fetch_all_pages_content
  10. from chunking import semantic_chunking, model
  11. from embedding_index import build_faiss_index, search_faiss
  12. from qa_chain import get_qa_chain
  13. import getpass
  14. import warnings
  15. warnings.filterwarnings("ignore")
  16. def get_relevance_label(score):
  17. if score >= 2.5:
  18. return "✅ bon"
  19. elif score >= 0:
  20. return "⚠️ moyen"
  21. else:
  22. return "❌ faible"
  23. def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
  24. question_embedding = model.encode([question], convert_to_tensor=False)
  25. question_embedding = np.array(question_embedding).astype('float32')
  26. if question_embedding.ndim == 1:
  27. question_embedding = np.expand_dims(question_embedding, axis=0)
  28. distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
  29. candidates = []
  30. for idx, dist in zip(indices, distances):
  31. candidates.append({
  32. "chunk": chunks[idx],
  33. "url": chunk_urls[idx],
  34. "index": idx,
  35. "faiss_score": float(dist)
  36. })
  37. # Reranking
  38. pairs = [[question, c["chunk"]] for c in candidates]
  39. rerank_scores = cross_encoder.predict(pairs)
  40. for i in range(len(candidates)):
  41. candidates[i]["rerank_score"] = float(rerank_scores[i])
  42. reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
  43. print("\n🔍 Résultats re-rankés :")
  44. df = pd.DataFrame([{
  45. "Index": r["index"],
  46. "URL": r["url"],
  47. "Score CrossEncoder": round(r["rerank_score"], 4),
  48. "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
  49. } for r in reranked])
  50. df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
  51. pd.set_option('display.max_colwidth', None)
  52. df_reranked = df.copy()
  53. # Tableau propre
  54. colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
  55. df_affichage = df_reranked[colonnes_a_afficher].copy()
  56. df_affichage.reset_index(drop=True, inplace=True)
  57. print("\n🎯 Résultats re-rankés (top 10) :")
  58. print(df_affichage.head(10).to_string(index=True))
  59. # Si tous les top chunks viennent d'une seule page
  60. top_urls = [r["url"] for r in reranked]
  61. unique_top_urls = list(set(top_urls))
  62. if len(unique_top_urls) == 1:
  63. print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
  64. full_context = "\n\n".join([
  65. chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
  66. ])
  67. else:
  68. full_context = "\n\n".join([r["chunk"] for r in reranked])
  69. result = qa_chain.run(context=full_context, question=question)
  70. print(f"\n🤖 Réponse :\n{result}\n")
  71. if __name__ == "__main__":
  72. nltk.download('punkt')
  73. user = input("Entrez votre nom d'utilisateur : ")
  74. pwd = getpass.getpass("Entrez votre mot de passe (entrée masquée) : ")
  75. success = login_to_wiki(user, pwd)
  76. if success:
  77. wiki_urls = [
  78. "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
  79. "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
  80. "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
  81. "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
  82. ]
  83. url_texts = fetch_all_pages_content(wiki_urls)
  84. all_chunks = []
  85. chunk_urls = []
  86. for url, text in url_texts.items():
  87. print(f"\n--- Traitement de la page {url} ---")
  88. sentences = nltk.sent_tokenize(text, language='french')
  89. chunks = semantic_chunking(sentences)
  90. print(f"Nombre de chunks : {len(chunks)}")
  91. for i, chunk in enumerate(chunks):
  92. print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
  93. all_chunks.append(chunk)
  94. chunk_urls.append(url)
  95. print("\n✅ Génération des embeddings des chunks...")
  96. chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
  97. chunk_embeddings = np.array(chunk_embeddings).astype('float32')
  98. print("\n✅ Construction de l'index FAISS...")
  99. index = build_faiss_index(chunk_embeddings)
  100. print("✅ Index FAISS construit avec succès.")
  101. cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
  102. qa_chain = get_qa_chain()
  103. while True:
  104. question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
  105. if question.lower() in ["exit", "quit"]:
  106. break
  107. ask_question_with_reranking(
  108. question, model, cross_encoder, index,
  109. all_chunks, chunk_urls, qa_chain
  110. )