main.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221
  1. import streamlit as st
  2. import nltk
  3. import numpy as np
  4. import pandas as pd
  5. from sentence_transformers import SentenceTransformer, CrossEncoder
  6. from auth import login_to_wiki
  7. from scraping import fetch_all_pages_content
  8. from chunking import semantic_chunking, model
  9. from embedding_index import build_faiss_index, search_faiss
  10. from qa_chain import get_qa_chain
  11. import warnings
  12. import os
  13. import json
  14. warnings.filterwarnings("ignore")
  15. os.environ["STREAMLIT_SERVER_RUN_ON_SAVE"] = "false"
  16. # Pour l'affichage du niveau de pertinence
  17. def get_relevance_label(score):
  18. if score >= 2.5:
  19. return "✅ bon"
  20. elif score >= -2:
  21. return "⚠️ moyen"
  22. else:
  23. return "❌ faible"
  24. # Fonction principale de QA avec reranking et affichage complet
  25. def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
  26. question_embedding = model.encode([question], convert_to_tensor=False)
  27. question_embedding = np.array(question_embedding).astype('float32')
  28. if question_embedding.ndim == 1:
  29. question_embedding = np.expand_dims(question_embedding, axis=0)
  30. distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
  31. candidates = []
  32. for idx, dist in zip(indices, distances):
  33. candidates.append({
  34. "chunk": chunks[idx],
  35. "url": chunk_urls[idx],
  36. "index": idx,
  37. "faiss_score": float(dist)
  38. })
  39. # Reranking
  40. pairs = [[question, c["chunk"]] for c in candidates]
  41. rerank_scores = cross_encoder.predict(pairs)
  42. for i in range(len(candidates)):
  43. candidates[i]["rerank_score"] = float(rerank_scores[i])
  44. reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
  45. # Console output
  46. print("\n🔍 Résultats re-rankés :")
  47. df = pd.DataFrame([{
  48. "Index": r["index"],
  49. "URL": r["url"],
  50. "Score CrossEncoder": round(r["rerank_score"], 4),
  51. "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
  52. } for r in reranked])
  53. df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
  54. pd.set_option('display.max_colwidth', None)
  55. print("\n🎯 Résultats re-rankés (top 10) :")
  56. print(df.head(10).to_string(index=True))
  57. # Sélection du contexte
  58. top_urls = [r["url"] for r in reranked]
  59. unique_top_urls = list(set(top_urls))
  60. if len(unique_top_urls) == 1:
  61. st.info("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
  62. full_context = "\n\n".join([chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]])
  63. else:
  64. full_context = "\n\n".join([r["chunk"] for r in reranked])
  65. # Template
  66. template = """
  67. Tu es un assistant intelligent qui répond uniquement à partir du texte ci-dessous.
  68. Ne réponds que si la réponse est clairement présente. Si tu ne sais pas, dis "Je ne sais pas".
  69. CONTEXTE :
  70. {context}
  71. QUESTION :
  72. {question}
  73. RÉPONSE :
  74. """
  75. # Génération de la réponse
  76. result = qa_chain.run(context=full_context, question=question)
  77. # Affichage Streamlit
  78. st.write("### 🤖 Réponse")
  79. st.write(result)
  80. # Affichage terminal : prompt complet
  81. print("\n====================== PROMPT UTILISÉ ======================")
  82. print(template.format(context=full_context, question=question))
  83. print(result)
  84. print("============================================================\n")
  85. def main():
  86. st.title("🔎 Recherche Wiki")
  87. nltk.download('punkt')
  88. user = st.text_input("Entrez votre nom d'utilisateur")
  89. pwd = st.text_input("Entrez votre mot de passe", type="password")
  90. if st.button("Se connecter et charger données"):
  91. if login_to_wiki(user, pwd):
  92. st.success("Connexion réussie.")
  93. wiki_urls = [
  94. "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
  95. "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
  96. "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
  97. "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start",
  98. "https://wiki.intersel.fr/doku.php?id=intersel:php",
  99. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start",
  100. "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire",
  101. "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start",
  102. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start",
  103. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start"
  104. ]
  105. with st.spinner("Récupération et traitement des pages Wiki..."):
  106. url_texts = fetch_all_pages_content(wiki_urls)
  107. # Sauvegarde des contenus bruts de chaque page dans un fichier JSON lisible
  108. raw_pages = [{"url": url, "text": text.strip()} for url, text in url_texts.items()]
  109. with open("raw_pages_pretty.json", "w", encoding="utf-8") as f:
  110. json.dump(raw_pages, f, ensure_ascii=False, indent=2)
  111. print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'")
  112. all_chunks = []
  113. chunk_urls = []
  114. for url, text in url_texts.items():
  115. sentences = nltk.sent_tokenize(text, language='french')
  116. print("\n" + "="*80)
  117. print(f"📄 Traitement de la page : {url}")
  118. print("="*80)
  119. print(f"🔢 Nombre de phrases détectées : {len(sentences)}")
  120. print(sentences)
  121. chunks = semantic_chunking(sentences)
  122. for chunk in chunks:
  123. all_chunks.append(chunk)
  124. chunk_urls.append(url)
  125. print("Nombre de mots total :", sum(len(s.split()) for s in sentences))
  126. print("Chunks générés :", len(chunks))
  127. print("\n🧩 Détail des chunks générés :")
  128. for i, chunk in enumerate(chunks):
  129. print("=" * 60)
  130. print(f"📌 Chunk {i+1}/{len(chunks)} | 📝 {len(chunk.split())} mots")
  131. print("-" * 60)
  132. print(chunk.strip())
  133. print("=" * 60 + "\n")
  134. print("\n✅ Génération des embeddings des chunks...")
  135. chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
  136. chunk_embeddings = np.array(chunk_embeddings).astype('float32')
  137. print("\n✅ Construction de l'index FAISS...")
  138. index = build_faiss_index(chunk_embeddings)
  139. print("✅ Index FAISS construit avec succès")
  140. chunk_data = [
  141. {
  142. "url": chunk_urls[i],
  143. "chunk_index": i + 1,
  144. "nb_mots": len(all_chunks[i].split()),
  145. "chunk": all_chunks[i]
  146. }
  147. for i in range(len(all_chunks))
  148. ]
  149. with open("chunks_pretty.json", "w", encoding="utf-8") as f:
  150. json.dump(chunk_data, f, ensure_ascii=False, indent=2)
  151. print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
  152. cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
  153. qa_chain = get_qa_chain()
  154. st.session_state['model'] = model
  155. st.session_state['cross_encoder'] = cross_encoder
  156. st.session_state['index'] = index
  157. st.session_state['all_chunks'] = all_chunks
  158. st.session_state['chunk_urls'] = chunk_urls
  159. st.session_state['qa_chain'] = qa_chain
  160. else:
  161. st.error("Échec de la connexion, veuillez vérifier vos identifiants.")
  162. if 'model' in st.session_state:
  163. question = st.text_input("❓ Posez votre question sur le Wiki")
  164. if st.button("Chercher"):
  165. ask_question_with_reranking(
  166. question,
  167. st.session_state['model'],
  168. st.session_state['cross_encoder'],
  169. st.session_state['index'],
  170. st.session_state['all_chunks'],
  171. st.session_state['chunk_urls'],
  172. st.session_state['qa_chain']
  173. )
  174. if __name__ == "__main__":
  175. main()