app.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. from flask import Flask, request, jsonify, send_from_directory, Response
  2. import nltk
  3. import numpy as np
  4. import json
  5. import os
  6. from sentence_transformers import CrossEncoder, SentenceTransformer
  7. from langchain_ollama import OllamaLLM
  8. from agent_ai.auth import login_to_wiki
  9. from agent_ai.scraping import fetch_all_pages_content
  10. from agent_ai.chunking import semantic_chunking, model
  11. from agent_ai.embedding_index import build_faiss_index, search_faiss
  12. from agent_ai.qa_chain import get_qa_chain
  13. from tabulate import tabulate
  14. import textwrap
  15. app = Flask(__name__)
  16. nltk.download('punkt')
  17. # Global state
  18. state = {
  19. "model": model,
  20. "cross_encoder": CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'),
  21. "index": None,
  22. "all_chunks": [],
  23. "chunk_urls": [],
  24. "qa_chain": get_qa_chain()
  25. }
  26. @app.route('/index.html')
  27. def index():
  28. return send_from_directory(os.getcwd(), 'index.html')
  29. @app.route("/login", methods=["POST"])
  30. def login():
  31. data = request.json
  32. username = data.get("username")
  33. password = data.get("password")
  34. if not username or not password:
  35. return jsonify({"error": "username and password required"}), 400
  36. if not login_to_wiki(username, password):
  37. return jsonify({"error": "Authentication failed"}), 401
  38. return jsonify({"message": "Login successful"})
  39. @app.route("/fetch_pages", methods=["POST"])
  40. def fetch_pages():
  41. urls = [
  42. "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
  43. "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
  44. "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
  45. "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start",
  46. "https://wiki.intersel.fr/doku.php?id=intersel:php",
  47. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start",
  48. "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire",
  49. "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start",
  50. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start",
  51. "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start"
  52. ]
  53. pages = fetch_all_pages_content(urls)
  54. raw_pages = [{"url": url, "text": text.strip()} for url, text in pages.items()]
  55. with open("raw_pages_pretty.json", "w", encoding="utf-8") as f:
  56. json.dump(raw_pages, f, ensure_ascii=False, indent=2)
  57. print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'")
  58. return jsonify(raw_pages)
  59. state = {
  60. "chunk_urls": [],
  61. "model": SentenceTransformer("all-MiniLM-L6-v2"),
  62. "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
  63. }
  64. @app.route("/chunk_all", methods=["POST"])
  65. def chunk_all():
  66. data = request.get_json()
  67. text = data.get("text", "")
  68. url = data.get("url", "url inconnue")
  69. if not text.strip():
  70. return jsonify({"error": "Texte vide"}), 400
  71. sentences = nltk.sent_tokenize(text, language='french')
  72. chunks = semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=1000)
  73. # chunks = chunk_text_generic(text, max_words=500)
  74. chunk_data = []
  75. response_chunks = []
  76. for i, chunk in enumerate(chunks, start=1):
  77. clean_chunk = chunk.strip()
  78. word_count = len(clean_chunk.split())
  79. chunk_data.append({
  80. "chunk_id": i,
  81. "text": clean_chunk,
  82. "url": url,
  83. "word_count": word_count
  84. })
  85. response_chunks.append(clean_chunk)
  86. # Sécuriser l'état global
  87. if "chunk_urls" not in state:
  88. state["chunk_urls"] = []
  89. state["chunk_urls"].append(url)
  90. # Enregistrement JSON
  91. with open("chunks_pretty.json", "a", encoding="utf-8") as f:
  92. for chunk in chunk_data:
  93. f.write(json.dumps(chunk, ensure_ascii=False) + ",\n")
  94. print(f"✅ {len(chunk_data)} chunks enregistrés pour {url}")
  95. return jsonify({"chunks": response_chunks})
  96. @app.route("/build_index", methods=["POST"])
  97. def build_index():
  98. data = request.json
  99. texts = data.get("texts")
  100. urls = data.get("urls")
  101. if not texts or not isinstance(texts, list):
  102. return jsonify({"error": "A list of texts is required"}), 400
  103. embeddings = state["model"].encode(texts, convert_to_tensor=False)
  104. embeddings = np.array(embeddings).astype('float32')
  105. index = build_faiss_index(embeddings)
  106. state["index"] = index
  107. state["all_chunks"] = texts
  108. # Important: also assign the urls to the global state here
  109. if urls and isinstance(urls, list) and len(urls) == len(texts):
  110. state["chunk_urls"] = urls
  111. print("✅ Index FAISS construit avec succès")
  112. return jsonify({"message": f"Index built with {len(texts)} vectors"})
  113. @app.route("/search_index", methods=["POST"])
  114. def search_index():
  115. data = request.json
  116. question = data.get("question")
  117. if not question:
  118. return jsonify({"error": "Question is required"}), 400
  119. if state["index"] is None:
  120. return jsonify({"error": "Index not built yet"}), 400
  121. question_embedding = state["model"].encode([question], convert_to_tensor=False)
  122. question_embedding = np.array(question_embedding).astype('float32')
  123. if question_embedding.ndim == 1:
  124. question_embedding = np.expand_dims(question_embedding, axis=0)
  125. distances, indices = search_faiss(state["index"], question_embedding, top_k=20)
  126. faiss_chunks = [state["all_chunks"][i] for i in indices[0]]
  127. # Re-rank with CrossEncoder
  128. cross_encoder = state["cross_encoder"]
  129. cross_scores = cross_encoder.predict([(question, chunk) for chunk in faiss_chunks])
  130. reranked = sorted(zip(faiss_chunks, indices[0], cross_scores), key=lambda x: x[2], reverse=True)
  131. top_chunks = [chunk for chunk, _, _ in reranked]
  132. indices[0] = [idx for _, idx, _ in reranked]
  133. state["last_indices"] = indices[0]
  134. # Display ranked results in a table
  135. table = []
  136. results = []
  137. for rank, (chunk, idx, score) in enumerate(reranked, start=1):
  138. url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
  139. # Wrap chunk and URL text to keep it readable in the terminal
  140. wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..."
  141. wrapped_url = "\n".join(textwrap.wrap(url, width=80))
  142. table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
  143. results.append({"text": chunk, "score": float(score), "url": url})
  144. print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
  145. print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
  146. print("=======================================================================\n")
  147. top_context = " ".join(top_chunks)
  148. return jsonify({"results": results, "context": top_context})
  149. llm = OllamaLLM(
  150. model="llama3",
  151. temperature=0.3,
  152. top_p=0.9,
  153. max_tokens=200
  154. )
  155. @app.route("/qa_chain", methods=["POST"])
  156. def qa_chain_route():
  157. data = request.get_json()
  158. question = data.get("question", "")
  159. top_chunks = [
  160. # Pour chaque indice 'i' dans la liste 'last_indices' (indices des meilleurs chunks)
  161. f"[Source: {state['chunk_urls'][i] if i < len(state['chunk_urls']) else 'URL inconnue'}]\n{state['all_chunks'][i]}" # On récupère l'URL correspondante à cet indice si elle existe, sinon on met 'URL inconnue' Puis on ajoute le texte complet du chunk correspondant
  162. for i in state.get("last_indices", [])
  163. if i < len(state["all_chunks"]) # On s'assure que l'indice est valide pour éviter les erreurs d'index hors limites
  164. ]
  165. top_context = "\n\n".join(top_chunks)
  166. # Limit context length to avoid overload
  167. max_context_length = 5000
  168. if len(top_context) > max_context_length:
  169. top_context = top_context[:max_context_length]
  170. prompt = (
  171. "Fournis une réponse factuelle et précise en te basant uniquement sur des informations vérifiables. "
  172. "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. "
  173. "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n"
  174. f"{top_context}\n\n"
  175. f"Question : {question}"
  176. )
  177. result = llm(prompt)
  178. print("\n====================== PROMPT UTILISÉ ======================")
  179. print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}")
  180. print("============================================================\n")
  181. return jsonify({"answer": result})
  182. if __name__ == "__main__":
  183. app.run(debug=True)