from flask import Flask, request, jsonify, send_from_directory from apscheduler.schedulers.background import BackgroundScheduler from sentence_transformers import CrossEncoder, SentenceTransformer from langchain_ollama import OllamaLLM from agent_ai.qa_chain import get_qa_chain from processing import full_update_pipeline, load_index_from_disk from agent_ai.auth import login_to_wiki import faiss import json import numpy as np import textwrap from sklearn.metrics.pairwise import cosine_similarity from tabulate import tabulate import os import threading import getpass app = Flask(__name__) state = { "index": None, "all_chunks": [], "chunk_urls": [], "all_embeddings": None, "embedding_model": SentenceTransformer('all-MiniLM-L6-v2'), "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2"), "is_updating": False } llm = OllamaLLM(model="llama3", temperature=0.3, top_p=0.9, max_tokens=200) def load_or_update_index(): print("🚀 App starting up...") index, texts, urls = load_index_from_disk() if index: state["index"] = index state["all_chunks"] = texts state["chunk_urls"] = urls print("✅ Index loaded from disk.") else: print("❌ No index found. Launching full update in background...") threading.Thread(target=run_full_update, daemon=True).start() def run_full_update(): state["is_updating"] = True try: index, texts, urls = full_update_pipeline() if index: state["index"] = index state["all_chunks"] = texts state["chunk_urls"] = urls state["all_embeddings"] = state["embedding_model"].encode(texts) print("✅ Background update finished.") except Exception as e: print(f"❌ Failed to update index: {e}") finally: state["is_updating"] = False @app.route('/index.html') def index(): return send_from_directory(os.getcwd(), 'index.html') @app.route("/login", methods=["POST"]) def login(): data = request.json username = data.get("username") password = data.get("password") if not username or not password: return jsonify({"error": "username and password required"}), 400 if not login_to_wiki(username, password): return jsonify({"error": "Authentication failed"}), 401 return jsonify({"message": "Login successful"}) @app.route("/qa_chain", methods=["POST"]) def qa_chain_route(): if state["index"] is None or not state["all_chunks"]: if state["is_updating"]: return jsonify({"error": "Le systĂšme est encore en train d’indexer les documents. RĂ©essayez dans quelques instants."}), 503 else: return jsonify({"error": "L’index est vide. Veuillez attendre la mise Ă  jour ou rĂ©initialiser le systĂšme."}), 500 data = request.get_json() question = data.get("question", "") # 1. Encode la question query_embedding = state["embedding_model"].encode([question])[0] # 2. Encode les chunks si non encodĂ©s if state["all_embeddings"] is None: state["all_embeddings"] = state["embedding_model"].encode(state["all_chunks"]) # 3. SimilaritĂ© cosinus similarities = cosine_similarity([query_embedding], state["all_embeddings"])[0] top_indices = np.argsort(similarities)[::-1][:100] # 4. Rerank avec CrossEncoder rerank_data = [(question, state["all_chunks"][i]) for i in top_indices] rerank_scores = state["cross_encoder"].predict(rerank_data) reranked = sorted(zip([state["all_chunks"][i] for i in top_indices], top_indices, rerank_scores), key=lambda x: x[2], reverse=True) # 5. Affichage en terminal table = [] top_chunks = [] for rank, (chunk, idx, score) in enumerate(reranked, start=1): url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue" wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..." wrapped_url = "\n".join(textwrap.wrap(url, width=80)) table.append([rank, round(score, 4), wrapped_chunk, wrapped_url]) top_chunks.append(chunk) print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================") print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid")) print("=======================================================================\n") top_context = "\n\n".join(top_chunks) if len(top_context) > 20000: top_context = top_context[:20000] prompt = ( "Fournis une rĂ©ponse factuelle et prĂ©cise en français en te basant uniquement sur le contexte fourni. " "Si tu n'es pas certain de la rĂ©ponse ou si les donnĂ©es sont ambiguĂ«s, indique clairement que l'information n'est pas disponible ou nĂ©cessite une vĂ©rification. " "Ne fais aucune supposition ou infĂ©rence qui pourrait ĂȘtre incorrecte, et prĂ©cise les sources utilisĂ©es lorsque possible.\n\n" f"{top_context}\n\n" f"Question : {question}" ) result = llm(prompt) print("\n====================== PROMPT UTILISÉ ======================") print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}") print("============================================================\n") source_urls = [state["chunk_urls"][idx] for _, idx, _ in reranked[:5]] response = { "answer": result, "sources": list(set(source_urls)) } return jsonify(response) # Scheduler scheduler = BackgroundScheduler() scheduler.add_job(run_full_update, 'interval', minutes=30) print("🔁 Scheduler activĂ© pour mises Ă  jour toutes les 30 min") def prompt_for_login(): print("🔐 Please log in to the Wiki before starting the app.") while True: username = input("Username: ") password = getpass.getpass("Password: ") if login_to_wiki(username, password): print("Login successful, starting the app...") break else: print("Login failed, please try again.\n") if __name__ == "__main__": prompt_for_login() load_or_update_index() scheduler.start() app.run(debug=True, use_reloader=False)