import streamlit as st import nltk import numpy as np import pandas as pd from sentence_transformers import SentenceTransformer, CrossEncoder from auth import login_to_wiki from scraping import fetch_all_pages_content from chunking import semantic_chunking, model from embedding_index import build_faiss_index, search_faiss from qa_chain import get_qa_chain import warnings import os import json warnings.filterwarnings("ignore") os.environ["STREAMLIT_SERVER_RUN_ON_SAVE"] = "false" # Pour l'affichage du niveau de pertinence def get_relevance_label(score): if score >= 2.5: return "✅ bon" elif score >= -2: return "⚠️ moyen" else: return "❌ faible" # Fonction principale de QA avec reranking et affichage complet def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10): question_embedding = model.encode([question], convert_to_tensor=False) question_embedding = np.array(question_embedding).astype('float32') if question_embedding.ndim == 1: question_embedding = np.expand_dims(question_embedding, axis=0) distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k) candidates = [] for idx, dist in zip(indices, distances): candidates.append({ "chunk": chunks[idx], "url": chunk_urls[idx], "index": idx, "faiss_score": float(dist) }) # Reranking pairs = [[question, c["chunk"]] for c in candidates] rerank_scores = cross_encoder.predict(pairs) for i in range(len(candidates)): candidates[i]["rerank_score"] = float(rerank_scores[i]) reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k] # Console output print("\n🔍 Résultats re-rankés :") df = pd.DataFrame([{ "Index": r["index"], "URL": r["url"], "Score CrossEncoder": round(r["rerank_score"], 4), "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"] } for r in reranked]) df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label) pd.set_option('display.max_colwidth', None) print("\n🎯 Résultats re-rankés (top 10) :") print(df.head(10).to_string(index=True)) # Sélection du contexte top_urls = [r["url"] for r in reranked] unique_top_urls = list(set(top_urls)) if len(unique_top_urls) == 1: st.info("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.") full_context = "\n\n".join([chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]]) else: full_context = "\n\n".join([r["chunk"] for r in reranked]) # Template template = """ Tu es un assistant intelligent qui répond uniquement à partir du texte ci-dessous. Ne réponds que si la réponse est clairement présente. Si tu ne sais pas, dis "Je ne sais pas". CONTEXTE : {context} QUESTION : {question} RÉPONSE : """ # Génération de la réponse result = qa_chain.run(context=full_context, question=question) # Affichage Streamlit st.write("### 🤖 Réponse") st.write(result) # Affichage terminal : prompt complet print("\n====================== PROMPT UTILISÉ ======================") print(template.format(context=full_context, question=question)) print(result) print("============================================================\n") def main(): st.title("🔎 Recherche Wiki") nltk.download('punkt') user = st.text_input("Entrez votre nom d'utilisateur") pwd = st.text_input("Entrez votre mot de passe", type="password") if st.button("Se connecter et charger données"): if login_to_wiki(user, pwd): st.success("Connexion réussie.") wiki_urls = [ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe", "https://wiki.intersel.fr/doku.php?id=intersel:presentation", "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires", "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start", "https://wiki.intersel.fr/doku.php?id=intersel:php", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start", "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire", "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start" ] with st.spinner("Récupération et traitement des pages Wiki..."): url_texts = fetch_all_pages_content(wiki_urls) # Sauvegarde des contenus bruts de chaque page dans un fichier JSON lisible raw_pages = [{"url": url, "text": text.strip()} for url, text in url_texts.items()] with open("raw_pages_pretty.json", "w", encoding="utf-8") as f: json.dump(raw_pages, f, ensure_ascii=False, indent=2) print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'") all_chunks = [] chunk_urls = [] for url, text in url_texts.items(): sentences = nltk.sent_tokenize(text, language='french') print("\n" + "="*80) print(f"📄 Traitement de la page : {url}") print("="*80) print(f"🔢 Nombre de phrases détectées : {len(sentences)}") print(sentences) chunks = semantic_chunking(sentences) for chunk in chunks: all_chunks.append(chunk) chunk_urls.append(url) print("Nombre de mots total :", sum(len(s.split()) for s in sentences)) print("Chunks générés :", len(chunks)) print("\n🧩 Détail des chunks générés :") for i, chunk in enumerate(chunks): print("=" * 60) print(f"📌 Chunk {i+1}/{len(chunks)} | 📝 {len(chunk.split())} mots") print("-" * 60) print(chunk.strip()) print("=" * 60 + "\n") print("\n✅ Génération des embeddings des chunks...") chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True) chunk_embeddings = np.array(chunk_embeddings).astype('float32') print("\n✅ Construction de l'index FAISS...") index = build_faiss_index(chunk_embeddings) print("✅ Index FAISS construit avec succès") chunk_data = [ { "url": chunk_urls[i], "chunk_index": i + 1, "nb_mots": len(all_chunks[i].split()), "chunk": all_chunks[i] } for i in range(len(all_chunks)) ] with open("chunks_pretty.json", "w", encoding="utf-8") as f: json.dump(chunk_data, f, ensure_ascii=False, indent=2) print("📁 Chunks enregistrés dans 'chunks_pretty.json'") cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') qa_chain = get_qa_chain() st.session_state['model'] = model st.session_state['cross_encoder'] = cross_encoder st.session_state['index'] = index st.session_state['all_chunks'] = all_chunks st.session_state['chunk_urls'] = chunk_urls st.session_state['qa_chain'] = qa_chain else: st.error("Échec de la connexion, veuillez vérifier vos identifiants.") if 'model' in st.session_state: question = st.text_input("❓ Posez votre question sur le Wiki") if st.button("Chercher"): ask_question_with_reranking( question, st.session_state['model'], st.session_state['cross_encoder'], st.session_state['index'], st.session_state['all_chunks'], st.session_state['chunk_urls'], st.session_state['qa_chain'] ) if __name__ == "__main__": main()