app.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. from flask import Flask, request, jsonify, send_from_directory
  2. from apscheduler.schedulers.background import BackgroundScheduler
  3. from sentence_transformers import CrossEncoder, SentenceTransformer
  4. from langchain_ollama import OllamaLLM
  5. from agent_ai.qa_chain import get_qa_chain
  6. from processing import full_update_pipeline, load_index_from_disk
  7. from agent_ai.auth import login_to_wiki
  8. import faiss
  9. import json
  10. import numpy as np
  11. import textwrap
  12. from sklearn.metrics.pairwise import cosine_similarity
  13. from tabulate import tabulate
  14. import os
  15. import threading
  16. import getpass
  17. app = Flask(__name__)
  18. state = {
  19. "index": None,
  20. "all_chunks": [],
  21. "chunk_urls": [],
  22. "all_embeddings": None,
  23. "embedding_model": SentenceTransformer('all-MiniLM-L6-v2'),
  24. "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2"),
  25. "is_updating": False
  26. }
  27. llm = OllamaLLM(model="llama3", temperature=0.3, top_p=0.9, max_tokens=200)
  28. def load_or_update_index():
  29. print("🚀 App starting up...")
  30. index, texts, urls = load_index_from_disk()
  31. if index:
  32. state["index"] = index
  33. state["all_chunks"] = texts
  34. state["chunk_urls"] = urls
  35. print("✅ Index loaded from disk.")
  36. else:
  37. print("❌ No index found. Launching full update in background...")
  38. threading.Thread(target=run_full_update, daemon=True).start()
  39. def run_full_update():
  40. state["is_updating"] = True
  41. try:
  42. index, texts, urls = full_update_pipeline()
  43. if index:
  44. state["index"] = index
  45. state["all_chunks"] = texts
  46. state["chunk_urls"] = urls
  47. state["all_embeddings"] = state["embedding_model"].encode(texts)
  48. print("✅ Background update finished.")
  49. except Exception as e:
  50. print(f"❌ Failed to update index: {e}")
  51. finally:
  52. state["is_updating"] = False
  53. @app.route('/index.html')
  54. def index():
  55. return send_from_directory(os.getcwd(), 'index.html')
  56. @app.route("/login", methods=["POST"])
  57. def login():
  58. data = request.json
  59. username = data.get("username")
  60. password = data.get("password")
  61. if not username or not password:
  62. return jsonify({"error": "username and password required"}), 400
  63. if not login_to_wiki(username, password):
  64. return jsonify({"error": "Authentication failed"}), 401
  65. return jsonify({"message": "Login successful"})
  66. @app.route("/qa_chain", methods=["POST"])
  67. def qa_chain_route():
  68. if state["index"] is None or not state["all_chunks"]:
  69. if state["is_updating"]:
  70. return jsonify({"error": "Le système est encore en train d’indexer les documents. Réessayez dans quelques instants."}), 503
  71. else:
  72. return jsonify({"error": "L’index est vide. Veuillez attendre la mise à jour ou réinitialiser le système."}), 500
  73. data = request.get_json()
  74. question = data.get("question", "")
  75. # 1. Encode la question
  76. query_embedding = state["embedding_model"].encode([question])[0]
  77. # 2. Encode les chunks si non encodés
  78. if state["all_embeddings"] is None:
  79. state["all_embeddings"] = state["embedding_model"].encode(state["all_chunks"])
  80. # 3. Similarité cosinus
  81. similarities = cosine_similarity([query_embedding], state["all_embeddings"])[0]
  82. top_indices = np.argsort(similarities)[::-1][:100]
  83. # 4. Rerank avec CrossEncoder
  84. rerank_data = [(question, state["all_chunks"][i]) for i in top_indices]
  85. rerank_scores = state["cross_encoder"].predict(rerank_data)
  86. reranked = sorted(zip([state["all_chunks"][i] for i in top_indices], top_indices, rerank_scores),
  87. key=lambda x: x[2], reverse=True)
  88. # 5. Affichage en terminal
  89. table = []
  90. top_chunks = []
  91. for rank, (chunk, idx, score) in enumerate(reranked, start=1):
  92. url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
  93. wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..."
  94. wrapped_url = "\n".join(textwrap.wrap(url, width=80))
  95. table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
  96. top_chunks.append(chunk)
  97. print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
  98. print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
  99. print("=======================================================================\n")
  100. top_context = "\n\n".join(top_chunks)
  101. if len(top_context) > 20000:
  102. top_context = top_context[:20000]
  103. prompt = (
  104. "Fournis une réponse factuelle et précise en français en te basant uniquement sur le contexte fourni. "
  105. "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. "
  106. "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n"
  107. f"{top_context}\n\n"
  108. f"Question : {question}"
  109. )
  110. result = llm(prompt)
  111. print("\n====================== PROMPT UTILISÉ ======================")
  112. print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}")
  113. print("============================================================\n")
  114. source_urls = [state["chunk_urls"][idx] for _, idx, _ in reranked[:5]]
  115. response = {
  116. "answer": result,
  117. "sources": list(set(source_urls))
  118. }
  119. return jsonify(response)
  120. # Scheduler
  121. scheduler = BackgroundScheduler()
  122. scheduler.add_job(run_full_update, 'interval', minutes=30)
  123. print("🔁 Scheduler activé pour mises à jour toutes les 30 min")
  124. def prompt_for_login():
  125. print("🔐 Please log in to the Wiki before starting the app.")
  126. while True:
  127. username = input("Username: ")
  128. password = getpass.getpass("Password: ")
  129. if login_to_wiki(username, password):
  130. print("Login successful, starting the app...")
  131. break
  132. else:
  133. print("Login failed, please try again.\n")
  134. if __name__ == "__main__":
  135. prompt_for_login()
  136. load_or_update_index()
  137. scheduler.start()
  138. app.run(debug=True, use_reloader=False)