|
@@ -1,7 +1,4 @@
|
|
|
-"""
|
|
|
|
|
-Programme principal
|
|
|
|
|
-"""
|
|
|
|
|
-
|
|
|
|
|
|
|
+import streamlit as st
|
|
|
import nltk
|
|
import nltk
|
|
|
import numpy as np
|
|
import numpy as np
|
|
|
import pandas as pd
|
|
import pandas as pd
|
|
@@ -11,19 +8,23 @@ from scraping import fetch_all_pages_content
|
|
|
from chunking import semantic_chunking, model
|
|
from chunking import semantic_chunking, model
|
|
|
from embedding_index import build_faiss_index, search_faiss
|
|
from embedding_index import build_faiss_index, search_faiss
|
|
|
from qa_chain import get_qa_chain
|
|
from qa_chain import get_qa_chain
|
|
|
-import getpass
|
|
|
|
|
-import warnings
|
|
|
|
|
|
|
+import warnings
|
|
|
|
|
+import os
|
|
|
|
|
+import json
|
|
|
|
|
+
|
|
|
warnings.filterwarnings("ignore")
|
|
warnings.filterwarnings("ignore")
|
|
|
|
|
+os.environ["STREAMLIT_SERVER_RUN_ON_SAVE"] = "false"
|
|
|
|
|
|
|
|
|
|
+# Pour l'affichage du niveau de pertinence
|
|
|
def get_relevance_label(score):
|
|
def get_relevance_label(score):
|
|
|
if score >= 2.5:
|
|
if score >= 2.5:
|
|
|
return "✅ bon"
|
|
return "✅ bon"
|
|
|
- elif score >= 0:
|
|
|
|
|
|
|
+ elif score >= -2:
|
|
|
return "⚠️ moyen"
|
|
return "⚠️ moyen"
|
|
|
else:
|
|
else:
|
|
|
return "❌ faible"
|
|
return "❌ faible"
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+# Fonction principale de QA avec reranking et affichage complet
|
|
|
def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
|
|
def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
|
|
|
question_embedding = model.encode([question], convert_to_tensor=False)
|
|
question_embedding = model.encode([question], convert_to_tensor=False)
|
|
|
question_embedding = np.array(question_embedding).astype('float32')
|
|
question_embedding = np.array(question_embedding).astype('float32')
|
|
@@ -49,6 +50,7 @@ def ask_question_with_reranking(question, model, cross_encoder, index, chunks, c
|
|
|
|
|
|
|
|
reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
|
|
reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
|
|
|
|
|
|
|
|
|
|
+ # Console output
|
|
|
print("\n🔍 Résultats re-rankés :")
|
|
print("\n🔍 Résultats re-rankés :")
|
|
|
df = pd.DataFrame([{
|
|
df = pd.DataFrame([{
|
|
|
"Index": r["index"],
|
|
"Index": r["index"],
|
|
@@ -59,77 +61,161 @@ def ask_question_with_reranking(question, model, cross_encoder, index, chunks, c
|
|
|
df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
|
|
df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
|
|
|
pd.set_option('display.max_colwidth', None)
|
|
pd.set_option('display.max_colwidth', None)
|
|
|
|
|
|
|
|
- df_reranked = df.copy()
|
|
|
|
|
-
|
|
|
|
|
- # Tableau propre
|
|
|
|
|
- colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
|
|
|
|
|
- df_affichage = df_reranked[colonnes_a_afficher].copy()
|
|
|
|
|
- df_affichage.reset_index(drop=True, inplace=True)
|
|
|
|
|
-
|
|
|
|
|
print("\n🎯 Résultats re-rankés (top 10) :")
|
|
print("\n🎯 Résultats re-rankés (top 10) :")
|
|
|
- print(df_affichage.head(10).to_string(index=True))
|
|
|
|
|
|
|
+ print(df.head(10).to_string(index=True))
|
|
|
|
|
|
|
|
- # Si tous les top chunks viennent d'une seule page
|
|
|
|
|
|
|
+ # Sélection du contexte
|
|
|
top_urls = [r["url"] for r in reranked]
|
|
top_urls = [r["url"] for r in reranked]
|
|
|
unique_top_urls = list(set(top_urls))
|
|
unique_top_urls = list(set(top_urls))
|
|
|
|
|
|
|
|
if len(unique_top_urls) == 1:
|
|
if len(unique_top_urls) == 1:
|
|
|
- print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
|
|
|
|
|
- full_context = "\n\n".join([
|
|
|
|
|
- chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
|
|
|
|
|
- ])
|
|
|
|
|
|
|
+ st.info("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
|
|
|
|
|
+ full_context = "\n\n".join([chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]])
|
|
|
else:
|
|
else:
|
|
|
full_context = "\n\n".join([r["chunk"] for r in reranked])
|
|
full_context = "\n\n".join([r["chunk"] for r in reranked])
|
|
|
|
|
|
|
|
|
|
+ # Template
|
|
|
|
|
+ template = """
|
|
|
|
|
+Tu es un assistant intelligent qui répond uniquement à partir du texte ci-dessous.
|
|
|
|
|
+Ne réponds que si la réponse est clairement présente. Si tu ne sais pas, dis "Je ne sais pas".
|
|
|
|
|
+
|
|
|
|
|
+CONTEXTE :
|
|
|
|
|
+{context}
|
|
|
|
|
+
|
|
|
|
|
+QUESTION :
|
|
|
|
|
+{question}
|
|
|
|
|
+
|
|
|
|
|
+RÉPONSE :
|
|
|
|
|
+"""
|
|
|
|
|
+
|
|
|
|
|
+ # Génération de la réponse
|
|
|
result = qa_chain.run(context=full_context, question=question)
|
|
result = qa_chain.run(context=full_context, question=question)
|
|
|
- print(f"\n🤖 Réponse :\n{result}\n")
|
|
|
|
|
|
|
|
|
|
|
|
+ # Affichage Streamlit
|
|
|
|
|
+ st.write("### 🤖 Réponse")
|
|
|
|
|
+ st.write(result)
|
|
|
|
|
|
|
|
|
|
+ # Affichage terminal : prompt complet
|
|
|
|
|
+ print("\n====================== PROMPT UTILISÉ ======================")
|
|
|
|
|
+ print(template.format(context=full_context, question=question))
|
|
|
|
|
+ print(result)
|
|
|
|
|
+ print("============================================================\n")
|
|
|
|
|
|
|
|
-if __name__ == "__main__":
|
|
|
|
|
|
|
+def main():
|
|
|
|
|
+ st.title("🔎 Recherche Wiki")
|
|
|
nltk.download('punkt')
|
|
nltk.download('punkt')
|
|
|
- user = input("Entrez votre nom d'utilisateur : ")
|
|
|
|
|
- pwd = getpass.getpass("Entrez votre mot de passe (entrée masquée) : ")
|
|
|
|
|
- success = login_to_wiki(user, pwd)
|
|
|
|
|
- if success:
|
|
|
|
|
- wiki_urls = [
|
|
|
|
|
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
|
|
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
|
|
- "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
|
|
|
|
|
- "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
|
|
|
|
|
- ]
|
|
|
|
|
-
|
|
|
|
|
- url_texts = fetch_all_pages_content(wiki_urls)
|
|
|
|
|
-
|
|
|
|
|
- all_chunks = []
|
|
|
|
|
- chunk_urls = []
|
|
|
|
|
-
|
|
|
|
|
- for url, text in url_texts.items():
|
|
|
|
|
- print(f"\n--- Traitement de la page {url} ---")
|
|
|
|
|
- sentences = nltk.sent_tokenize(text, language='french')
|
|
|
|
|
- chunks = semantic_chunking(sentences)
|
|
|
|
|
- print(f"Nombre de chunks : {len(chunks)}")
|
|
|
|
|
- for i, chunk in enumerate(chunks):
|
|
|
|
|
- print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
|
|
|
|
|
- all_chunks.append(chunk)
|
|
|
|
|
- chunk_urls.append(url)
|
|
|
|
|
-
|
|
|
|
|
- print("\n✅ Génération des embeddings des chunks...")
|
|
|
|
|
- chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
|
|
|
|
|
- chunk_embeddings = np.array(chunk_embeddings).astype('float32')
|
|
|
|
|
-
|
|
|
|
|
- print("\n✅ Construction de l'index FAISS...")
|
|
|
|
|
- index = build_faiss_index(chunk_embeddings)
|
|
|
|
|
- print("✅ Index FAISS construit avec succès.")
|
|
|
|
|
-
|
|
|
|
|
- cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
|
|
- qa_chain = get_qa_chain()
|
|
|
|
|
-
|
|
|
|
|
- while True:
|
|
|
|
|
- question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
|
|
|
|
|
- if question.lower() in ["exit", "quit"]:
|
|
|
|
|
- break
|
|
|
|
|
|
|
+
|
|
|
|
|
+ user = st.text_input("Entrez votre nom d'utilisateur")
|
|
|
|
|
+ pwd = st.text_input("Entrez votre mot de passe", type="password")
|
|
|
|
|
+
|
|
|
|
|
+ if st.button("Se connecter et charger données"):
|
|
|
|
|
+ if login_to_wiki(user, pwd):
|
|
|
|
|
+ st.success("Connexion réussie.")
|
|
|
|
|
+ wiki_urls = [
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:php",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start"
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ with st.spinner("Récupération et traitement des pages Wiki..."):
|
|
|
|
|
+ url_texts = fetch_all_pages_content(wiki_urls)
|
|
|
|
|
+
|
|
|
|
|
+ # Sauvegarde des contenus bruts de chaque page dans un fichier JSON lisible
|
|
|
|
|
+ raw_pages = [{"url": url, "text": text.strip()} for url, text in url_texts.items()]
|
|
|
|
|
+
|
|
|
|
|
+ with open("raw_pages_pretty.json", "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump(raw_pages, f, ensure_ascii=False, indent=2)
|
|
|
|
|
+
|
|
|
|
|
+ print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ all_chunks = []
|
|
|
|
|
+ chunk_urls = []
|
|
|
|
|
+
|
|
|
|
|
+ for url, text in url_texts.items():
|
|
|
|
|
+ sentences = nltk.sent_tokenize(text, language='french')
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ print("\n" + "="*80)
|
|
|
|
|
+ print(f"📄 Traitement de la page : {url}")
|
|
|
|
|
+ print("="*80)
|
|
|
|
|
+ print(f"🔢 Nombre de phrases détectées : {len(sentences)}")
|
|
|
|
|
+ print(sentences)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ chunks = semantic_chunking(sentences)
|
|
|
|
|
+ for chunk in chunks:
|
|
|
|
|
+ all_chunks.append(chunk)
|
|
|
|
|
+ chunk_urls.append(url)
|
|
|
|
|
+
|
|
|
|
|
+ print("Nombre de mots total :", sum(len(s.split()) for s in sentences))
|
|
|
|
|
+ print("Chunks générés :", len(chunks))
|
|
|
|
|
+
|
|
|
|
|
+ print("\n🧩 Détail des chunks générés :")
|
|
|
|
|
+ for i, chunk in enumerate(chunks):
|
|
|
|
|
+ print("=" * 60)
|
|
|
|
|
+ print(f"📌 Chunk {i+1}/{len(chunks)} | 📝 {len(chunk.split())} mots")
|
|
|
|
|
+ print("-" * 60)
|
|
|
|
|
+ print(chunk.strip())
|
|
|
|
|
+ print("=" * 60 + "\n")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ print("\n✅ Génération des embeddings des chunks...")
|
|
|
|
|
+ chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
|
|
|
|
|
+ chunk_embeddings = np.array(chunk_embeddings).astype('float32')
|
|
|
|
|
+
|
|
|
|
|
+ print("\n✅ Construction de l'index FAISS...")
|
|
|
|
|
+ index = build_faiss_index(chunk_embeddings)
|
|
|
|
|
+ print("✅ Index FAISS construit avec succès")
|
|
|
|
|
+
|
|
|
|
|
+ chunk_data = [
|
|
|
|
|
+ {
|
|
|
|
|
+ "url": chunk_urls[i],
|
|
|
|
|
+ "chunk_index": i + 1,
|
|
|
|
|
+ "nb_mots": len(all_chunks[i].split()),
|
|
|
|
|
+ "chunk": all_chunks[i]
|
|
|
|
|
+ }
|
|
|
|
|
+ for i in range(len(all_chunks))
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ with open("chunks_pretty.json", "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump(chunk_data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
+ print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
|
|
|
|
|
+
|
|
|
|
|
+ cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
|
|
+ qa_chain = get_qa_chain()
|
|
|
|
|
+
|
|
|
|
|
+ st.session_state['model'] = model
|
|
|
|
|
+ st.session_state['cross_encoder'] = cross_encoder
|
|
|
|
|
+ st.session_state['index'] = index
|
|
|
|
|
+ st.session_state['all_chunks'] = all_chunks
|
|
|
|
|
+ st.session_state['chunk_urls'] = chunk_urls
|
|
|
|
|
+ st.session_state['qa_chain'] = qa_chain
|
|
|
|
|
+
|
|
|
|
|
+ else:
|
|
|
|
|
+ st.error("Échec de la connexion, veuillez vérifier vos identifiants.")
|
|
|
|
|
+
|
|
|
|
|
+ if 'model' in st.session_state:
|
|
|
|
|
+ question = st.text_input("❓ Posez votre question sur le Wiki")
|
|
|
|
|
+ if st.button("Chercher"):
|
|
|
ask_question_with_reranking(
|
|
ask_question_with_reranking(
|
|
|
- question, model, cross_encoder, index,
|
|
|
|
|
- all_chunks, chunk_urls, qa_chain
|
|
|
|
|
|
|
+ question,
|
|
|
|
|
+ st.session_state['model'],
|
|
|
|
|
+ st.session_state['cross_encoder'],
|
|
|
|
|
+ st.session_state['index'],
|
|
|
|
|
+ st.session_state['all_chunks'],
|
|
|
|
|
+ st.session_state['chunk_urls'],
|
|
|
|
|
+ st.session_state['qa_chain']
|
|
|
)
|
|
)
|
|
|
|
|
+
|
|
|
|
|
+if __name__ == "__main__":
|
|
|
|
|
+ main()
|