| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163 |
- from flask import Flask, request, jsonify, send_from_directory
- from apscheduler.schedulers.background import BackgroundScheduler
- from sentence_transformers import CrossEncoder, SentenceTransformer
- from langchain_ollama import OllamaLLM
- from agent_ai.qa_chain import get_qa_chain
- from processing import full_update_pipeline, load_index_from_disk
- from agent_ai.auth import login_to_wiki
- import faiss
- import json
- import numpy as np
- import textwrap
- from sklearn.metrics.pairwise import cosine_similarity
- from tabulate import tabulate
- import os
- import threading
- import getpass
- app = Flask(__name__)
- state = {
- "index": None,
- "all_chunks": [],
- "chunk_urls": [],
- "all_embeddings": None,
- "embedding_model": SentenceTransformer('all-MiniLM-L6-v2'),
- "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2"),
- "is_updating": False
- }
- llm = OllamaLLM(model="llama3", temperature=0.3, top_p=0.9, max_tokens=200)
- def load_or_update_index():
- print("🚀 App starting up...")
- index, texts, urls = load_index_from_disk()
- if index:
- state["index"] = index
- state["all_chunks"] = texts
- state["chunk_urls"] = urls
- print("✅ Index loaded from disk.")
- else:
- print("❌ No index found. Launching full update in background...")
- threading.Thread(target=run_full_update, daemon=True).start()
- def run_full_update():
- state["is_updating"] = True
- try:
- index, texts, urls = full_update_pipeline()
- if index:
- state["index"] = index
- state["all_chunks"] = texts
- state["chunk_urls"] = urls
- state["all_embeddings"] = state["embedding_model"].encode(texts)
- print("✅ Background update finished.")
- except Exception as e:
- print(f"❌ Failed to update index: {e}")
- finally:
- state["is_updating"] = False
- @app.route('/index.html')
- def index():
- return send_from_directory(os.getcwd(), 'index.html')
- @app.route("/login", methods=["POST"])
- def login():
- data = request.json
- username = data.get("username")
- password = data.get("password")
- if not username or not password:
- return jsonify({"error": "username and password required"}), 400
- if not login_to_wiki(username, password):
- return jsonify({"error": "Authentication failed"}), 401
- return jsonify({"message": "Login successful"})
- @app.route("/qa_chain", methods=["POST"])
- def qa_chain_route():
- if state["index"] is None or not state["all_chunks"]:
- if state["is_updating"]:
- return jsonify({"error": "Le système est encore en train d’indexer les documents. Réessayez dans quelques instants."}), 503
- else:
- return jsonify({"error": "L’index est vide. Veuillez attendre la mise à jour ou réinitialiser le système."}), 500
- data = request.get_json()
- question = data.get("question", "")
- # 1. Encode la question
- query_embedding = state["embedding_model"].encode([question])[0]
- # 2. Encode les chunks si non encodés
- if state["all_embeddings"] is None:
- state["all_embeddings"] = state["embedding_model"].encode(state["all_chunks"])
- # 3. Similarité cosinus
- similarities = cosine_similarity([query_embedding], state["all_embeddings"])[0]
- top_indices = np.argsort(similarities)[::-1][:100]
- # 4. Rerank avec CrossEncoder
- rerank_data = [(question, state["all_chunks"][i]) for i in top_indices]
- rerank_scores = state["cross_encoder"].predict(rerank_data)
- reranked = sorted(zip([state["all_chunks"][i] for i in top_indices], top_indices, rerank_scores),
- key=lambda x: x[2], reverse=True)
- # 5. Affichage en terminal
- table = []
- top_chunks = []
- for rank, (chunk, idx, score) in enumerate(reranked, start=1):
- url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
- wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..."
- wrapped_url = "\n".join(textwrap.wrap(url, width=80))
- table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
- top_chunks.append(chunk)
- print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
- print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
- print("=======================================================================\n")
- top_context = "\n\n".join(top_chunks)
- if len(top_context) > 20000:
- top_context = top_context[:20000]
- prompt = (
- "Fournis une réponse factuelle et précise en français en te basant uniquement sur le contexte fourni. "
- "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. "
- "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n"
- f"{top_context}\n\n"
- f"Question : {question}"
- )
- result = llm(prompt)
- print("\n====================== PROMPT UTILISÉ ======================")
- print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}")
- print("============================================================\n")
- source_urls = [state["chunk_urls"][idx] for _, idx, _ in reranked[:5]]
- response = {
- "answer": result,
- "sources": list(set(source_urls))
- }
- return jsonify(response)
- # Scheduler
- scheduler = BackgroundScheduler()
- scheduler.add_job(run_full_update, 'interval', minutes=30)
- print("🔁 Scheduler activé pour mises à jour toutes les 30 min")
- def prompt_for_login():
- print("🔐 Please log in to the Wiki before starting the app.")
- while True:
- username = input("Username: ")
- password = getpass.getpass("Password: ")
- if login_to_wiki(username, password):
- print("Login successful, starting the app...")
- break
- else:
- print("Login failed, please try again.\n")
-
- if __name__ == "__main__":
- prompt_for_login()
- load_or_update_index()
- scheduler.start()
- app.run(debug=True, use_reloader=False)
|