|
|
@@ -0,0 +1,239 @@
|
|
|
+from flask import Flask, request, jsonify, send_from_directory, Response
|
|
|
+import nltk
|
|
|
+import numpy as np
|
|
|
+import json
|
|
|
+import os
|
|
|
+from sentence_transformers import CrossEncoder, SentenceTransformer
|
|
|
+from langchain_ollama import OllamaLLM
|
|
|
+from agent_ai.auth import login_to_wiki
|
|
|
+from agent_ai.scraping import fetch_all_pages_content
|
|
|
+from agent_ai.chunking import semantic_chunking, model, chunk_text, chunk_text_generic
|
|
|
+from agent_ai.embedding_index import build_faiss_index, search_faiss
|
|
|
+from agent_ai.qa_chain import get_qa_chain
|
|
|
+from tabulate import tabulate
|
|
|
+import textwrap
|
|
|
+
|
|
|
+app = Flask(__name__)
|
|
|
+
|
|
|
+nltk.download('punkt')
|
|
|
+
|
|
|
+# Global state
|
|
|
+state = {
|
|
|
+ "model": model,
|
|
|
+ "cross_encoder": CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'),
|
|
|
+ "index": None,
|
|
|
+ "all_chunks": [],
|
|
|
+ "chunk_urls": [],
|
|
|
+ "qa_chain": get_qa_chain()
|
|
|
+}
|
|
|
+
|
|
|
+@app.route('/index.html')
|
|
|
+def index():
|
|
|
+ return send_from_directory(os.getcwd(), 'index.html')
|
|
|
+
|
|
|
+@app.route("/login", methods=["POST"])
|
|
|
+def login():
|
|
|
+ data = request.json
|
|
|
+ username = data.get("username")
|
|
|
+ password = data.get("password")
|
|
|
+
|
|
|
+ if not username or not password:
|
|
|
+ return jsonify({"error": "username and password required"}), 400
|
|
|
+
|
|
|
+ if not login_to_wiki(username, password):
|
|
|
+ return jsonify({"error": "Authentication failed"}), 401
|
|
|
+
|
|
|
+ return jsonify({"message": "Login successful"})
|
|
|
+
|
|
|
+@app.route("/fetch_pages", methods=["POST"])
|
|
|
+def fetch_pages():
|
|
|
+ urls = [
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:php",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start"
|
|
|
+ ]
|
|
|
+ pages = fetch_all_pages_content(urls)
|
|
|
+ raw_pages = [{"url": url, "text": text.strip()} for url, text in pages.items()]
|
|
|
+
|
|
|
+ with open("raw_pages_pretty.json", "w", encoding="utf-8") as f:
|
|
|
+ json.dump(raw_pages, f, ensure_ascii=False, indent=2)
|
|
|
+
|
|
|
+ print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'")
|
|
|
+ return jsonify(raw_pages)
|
|
|
+
|
|
|
+
|
|
|
+state = {
|
|
|
+ "chunk_urls": [],
|
|
|
+ "model": SentenceTransformer("all-MiniLM-L6-v2"),
|
|
|
+ "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+@app.route("/chunk_all", methods=["POST"])
|
|
|
+def chunk_all():
|
|
|
+ data = request.get_json()
|
|
|
+ text = data.get("text", "")
|
|
|
+ url = data.get("url", "url inconnue")
|
|
|
+
|
|
|
+ if not text.strip():
|
|
|
+ return jsonify({"error": "Texte vide"}), 400
|
|
|
+
|
|
|
+ sentences = nltk.sent_tokenize(text, language='french')
|
|
|
+ chunks = semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=1000)
|
|
|
+ # chunks = chunk_text_generic(text, max_words=500)
|
|
|
+
|
|
|
+ chunk_data = []
|
|
|
+ response_chunks = []
|
|
|
+
|
|
|
+ for i, chunk in enumerate(chunks, start=1):
|
|
|
+ clean_chunk = chunk.strip()
|
|
|
+ word_count = len(clean_chunk.split())
|
|
|
+
|
|
|
+ chunk_data.append({
|
|
|
+ "chunk_id": i,
|
|
|
+ "text": clean_chunk,
|
|
|
+ "url": url,
|
|
|
+ "word_count": word_count
|
|
|
+ })
|
|
|
+ response_chunks.append(clean_chunk)
|
|
|
+
|
|
|
+ # Sécuriser l'état global
|
|
|
+ if "chunk_urls" not in state:
|
|
|
+ state["chunk_urls"] = []
|
|
|
+ state["chunk_urls"].append(url)
|
|
|
+
|
|
|
+ # Enregistrement JSON
|
|
|
+ with open("chunks_pretty.json", "a", encoding="utf-8") as f:
|
|
|
+ for chunk in chunk_data:
|
|
|
+ f.write(json.dumps(chunk, ensure_ascii=False) + ",\n")
|
|
|
+
|
|
|
+ print(f"✅ {len(chunk_data)} chunks enregistrés pour {url}")
|
|
|
+ return jsonify({"chunks": response_chunks})
|
|
|
+
|
|
|
+@app.route("/build_index", methods=["POST"])
|
|
|
+def build_index():
|
|
|
+ data = request.json
|
|
|
+ texts = data.get("texts")
|
|
|
+ urls = data.get("urls")
|
|
|
+
|
|
|
+ if not texts or not isinstance(texts, list):
|
|
|
+ return jsonify({"error": "A list of texts is required"}), 400
|
|
|
+
|
|
|
+ embeddings = state["model"].encode(texts, convert_to_tensor=False)
|
|
|
+ embeddings = np.array(embeddings).astype('float32')
|
|
|
+
|
|
|
+ index = build_faiss_index(embeddings)
|
|
|
+ state["index"] = index
|
|
|
+ state["all_chunks"] = texts
|
|
|
+
|
|
|
+ # Important: also assign the urls to the global state here
|
|
|
+ if urls and isinstance(urls, list) and len(urls) == len(texts):
|
|
|
+ state["chunk_urls"] = urls
|
|
|
+
|
|
|
+
|
|
|
+ print("✅ Index FAISS construit avec succès")
|
|
|
+ return jsonify({"message": f"Index built with {len(texts)} vectors"})
|
|
|
+
|
|
|
+@app.route("/search_index", methods=["POST"])
|
|
|
+def search_index():
|
|
|
+ data = request.json
|
|
|
+ question = data.get("question")
|
|
|
+
|
|
|
+ if not question:
|
|
|
+ return jsonify({"error": "Question is required"}), 400
|
|
|
+
|
|
|
+ if state["index"] is None:
|
|
|
+ return jsonify({"error": "Index not built yet"}), 400
|
|
|
+
|
|
|
+ question_embedding = state["model"].encode([question], convert_to_tensor=False)
|
|
|
+ question_embedding = np.array(question_embedding).astype('float32')
|
|
|
+
|
|
|
+ if question_embedding.ndim == 1:
|
|
|
+ question_embedding = np.expand_dims(question_embedding, axis=0)
|
|
|
+
|
|
|
+ distances, indices = search_faiss(state["index"], question_embedding, top_k=20)
|
|
|
+
|
|
|
+ faiss_chunks = [state["all_chunks"][i] for i in indices[0]]
|
|
|
+
|
|
|
+ # Re-rank with CrossEncoder
|
|
|
+ cross_encoder = state["cross_encoder"]
|
|
|
+ cross_scores = cross_encoder.predict([(question, chunk) for chunk in faiss_chunks])
|
|
|
+ reranked = sorted(zip(faiss_chunks, indices[0], cross_scores), key=lambda x: x[2], reverse=True)
|
|
|
+
|
|
|
+ top_chunks = [chunk for chunk, _, _ in reranked]
|
|
|
+ indices[0] = [idx for _, idx, _ in reranked]
|
|
|
+
|
|
|
+ state["last_indices"] = indices[0]
|
|
|
+
|
|
|
+ # Display ranked results in a table
|
|
|
+ table = []
|
|
|
+ results = []
|
|
|
+
|
|
|
+ for rank, (chunk, idx, score) in enumerate(reranked, start=1):
|
|
|
+ url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
|
|
|
+
|
|
|
+ # Wrap chunk and URL text to keep it readable in the terminal
|
|
|
+ wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..."
|
|
|
+ wrapped_url = "\n".join(textwrap.wrap(url, width=80))
|
|
|
+
|
|
|
+ table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
|
|
|
+ results.append({"text": chunk, "score": float(score), "url": url})
|
|
|
+
|
|
|
+ print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
|
|
|
+ print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
|
|
|
+ print("=======================================================================\n")
|
|
|
+
|
|
|
+
|
|
|
+ top_context = " ".join(top_chunks)
|
|
|
+
|
|
|
+ return jsonify({"results": results, "context": top_context})
|
|
|
+
|
|
|
+llm = OllamaLLM(
|
|
|
+ model="llama3",
|
|
|
+ temperature=0.3,
|
|
|
+ top_p=0.9,
|
|
|
+ max_tokens=200
|
|
|
+)
|
|
|
+
|
|
|
+@app.route("/qa_chain", methods=["POST"])
|
|
|
+def qa_chain_route():
|
|
|
+ data = request.get_json()
|
|
|
+ question = data.get("question", "")
|
|
|
+
|
|
|
+ top_chunks = [
|
|
|
+ f"[Source: {state['chunk_urls'][i] if i < len(state['chunk_urls']) else 'URL inconnue'}]\n{state['all_chunks'][i]}"
|
|
|
+ for i in state.get("last_indices", [])
|
|
|
+ if i < len(state["all_chunks"])
|
|
|
+ ]
|
|
|
+ top_context = "\n\n".join(top_chunks)
|
|
|
+
|
|
|
+ # Limit context length to avoid overload
|
|
|
+ max_context_length = 5000
|
|
|
+ if len(top_context) > max_context_length:
|
|
|
+ top_context = top_context[:max_context_length]
|
|
|
+
|
|
|
+ prompt = (
|
|
|
+ "Fournis une réponse factuelle et précise en te basant uniquement sur des informations vérifiables. "
|
|
|
+ "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. "
|
|
|
+ "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n"
|
|
|
+ f"{top_context}\n\n"
|
|
|
+ f"Question : {question}"
|
|
|
+ )
|
|
|
+
|
|
|
+ result = llm(prompt)
|
|
|
+
|
|
|
+ print("\n====================== PROMPT UTILISÉ ======================")
|
|
|
+ print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}")
|
|
|
+ print("============================================================\n")
|
|
|
+
|
|
|
+ return jsonify({"answer": result})
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ app.run(debug=True)
|