|
@@ -0,0 +1,404 @@
|
|
|
|
|
+from flask import Flask, request, jsonify, send_from_directory
|
|
|
|
|
+from apscheduler.schedulers.background import BackgroundScheduler
|
|
|
|
|
+from sentence_transformers import CrossEncoder, SentenceTransformer
|
|
|
|
|
+from langchain_ollama import OllamaLLM
|
|
|
|
|
+from .utils.processing import full_update_pipeline, load_index_from_disk
|
|
|
|
|
+from .agent_ai.auth import login_to_wiki
|
|
|
|
|
+import faiss
|
|
|
|
|
+import json
|
|
|
|
|
+import numpy as np
|
|
|
|
|
+import textwrap
|
|
|
|
|
+from sklearn.metrics.pairwise import cosine_similarity
|
|
|
|
|
+from tabulate import tabulate
|
|
|
|
|
+import os
|
|
|
|
|
+import threading
|
|
|
|
|
+from serpapi import GoogleSearch
|
|
|
|
|
+from werkzeug.utils import secure_filename
|
|
|
|
|
+import PyPDF2
|
|
|
|
|
+from PIL import Image
|
|
|
|
|
+from transformers import AutoTokenizer
|
|
|
|
|
+from .utils.ocr import run_ocr, generate_html , generate_word_level_html
|
|
|
|
|
+import tiktoken
|
|
|
|
|
+from .models.VLM import ask_blip
|
|
|
|
|
+
|
|
|
|
|
+app = Flask(__name__)
|
|
|
|
|
+
|
|
|
|
|
+# Configuration pour la production
|
|
|
|
|
+app.config['MAX_CONTENT_LENGTH'] = int(os.environ.get('MAX_CONTENT_LENGTH', 16 * 1024 * 1024)) # 16MB par défaut
|
|
|
|
|
+
|
|
|
|
|
+state = {
|
|
|
|
|
+ "index": None,
|
|
|
|
|
+ "all_chunks": [],
|
|
|
|
|
+ "chunk_urls": [],
|
|
|
|
|
+ "all_embeddings": None,
|
|
|
|
|
+ "embedding_model": SentenceTransformer('all-MiniLM-L6-v2'),
|
|
|
|
|
+ "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2"),
|
|
|
|
|
+ "is_updating": False,
|
|
|
|
|
+ "llm": OllamaLLM(
|
|
|
|
|
+ model="llama3",
|
|
|
|
|
+ temperature=0.3,
|
|
|
|
|
+ top_p=0.9,
|
|
|
|
|
+ max_tokens=600,
|
|
|
|
|
+ base_url=os.environ.get('OLLAMA_HOST', 'http://localhost:11434')
|
|
|
|
|
+ ),
|
|
|
|
|
+ "chat_history": []
|
|
|
|
|
+
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
|
|
|
|
+
|
|
|
|
|
+def chunk_text(text, chunk_size=400, overlap=100):
|
|
|
|
|
+ tokens = tokenizer.encode(text)
|
|
|
|
|
+ chunks = []
|
|
|
|
|
+ for i in range(0, len(tokens), chunk_size - overlap):
|
|
|
|
|
+ chunk = tokens[i:i + chunk_size]
|
|
|
|
|
+ chunks.append(tokenizer.decode(chunk))
|
|
|
|
|
+ return chunks
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+UPLOAD_FOLDER = os.environ.get("UPLOAD_FOLDER", "uploads")
|
|
|
|
|
+os.makedirs(UPLOAD_FOLDER, exist_ok=True) # Crée le dossier s'il n'existe pas
|
|
|
|
|
+app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
|
|
|
|
|
+
|
|
|
|
|
+ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg'}
|
|
|
|
|
+
|
|
|
|
|
+def allowed_file(filename):
|
|
|
|
|
+ return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
|
|
|
|
|
+
|
|
|
|
|
+@app.route("/upload_file", methods=["POST"])
|
|
|
|
|
+def upload_file():
|
|
|
|
|
+ if 'file' not in request.files:
|
|
|
|
|
+ return jsonify({"error": "Aucun fichier détecté."}), 400
|
|
|
|
|
+
|
|
|
|
|
+ file = request.files['file']
|
|
|
|
|
+ if file.filename == '':
|
|
|
|
|
+ return jsonify({"error": "Nom de fichier vide."}), 400
|
|
|
|
|
+
|
|
|
|
|
+ if file and allowed_file(file.filename):
|
|
|
|
|
+ filename = secure_filename(file.filename)
|
|
|
|
|
+ filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
|
|
|
|
|
+ file.save(filepath)
|
|
|
|
|
+
|
|
|
|
|
+ question = request.form.get("question", "")
|
|
|
|
|
+ if not question:
|
|
|
|
|
+ return jsonify({"error": "Veuillez poser une question liée au fichier."}), 400
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ # ✅ 1. Exécuter l'OCR
|
|
|
|
|
+ document, ocr_output = run_ocr(filepath)
|
|
|
|
|
+
|
|
|
|
|
+ # ✅ 2. Sauvegarder le JSON OCR
|
|
|
|
|
+ json_path = os.path.join(app.config['UPLOAD_FOLDER'], "ocr_output.json")
|
|
|
|
|
+ with open(json_path, "w", encoding="utf-8") as fjson:
|
|
|
|
|
+ json.dump(ocr_output, fjson, ensure_ascii=False, indent=2)
|
|
|
|
|
+
|
|
|
|
|
+ # ✅ 3. Générer le HTML avec bounding boxes par mot
|
|
|
|
|
+ html_path = os.path.join(app.config['UPLOAD_FOLDER'], "ocr_output_words.html")
|
|
|
|
|
+ generate_word_level_html(ocr_output, output_html_path=html_path)
|
|
|
|
|
+
|
|
|
|
|
+ # ✅ 4. Extraire le texte brut pour le traitement RAG
|
|
|
|
|
+ text = ""
|
|
|
|
|
+ for page in ocr_output['pages']:
|
|
|
|
|
+ for block in page['blocks']:
|
|
|
|
|
+ for line in block['lines']:
|
|
|
|
|
+ for word in line['words']:
|
|
|
|
|
+ text += word['value'] + " "
|
|
|
|
|
+ text += "\n"
|
|
|
|
|
+ text += "\n"
|
|
|
|
|
+ text += "\n"
|
|
|
|
|
+
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ return jsonify({"error": f"OCR échoué : {str(e)}"}), 500
|
|
|
|
|
+
|
|
|
|
|
+ if text.strip():
|
|
|
|
|
+ chunks = chunk_text(text)
|
|
|
|
|
+ embeddings = state["embedding_model"].encode(chunks)
|
|
|
|
|
+ question_embedding = state["embedding_model"].encode([question])[0]
|
|
|
|
|
+ similarities = cosine_similarity([question_embedding], embeddings)[0]
|
|
|
|
|
+ top_indices = np.argsort(similarities)[::-1][:5]
|
|
|
|
|
+ top_chunks = [chunks[i] for i in top_indices]
|
|
|
|
|
+ top_context = "\n\n".join(top_chunks)
|
|
|
|
|
+
|
|
|
|
|
+ prompt = (
|
|
|
|
|
+ "Réponds en français à la question suivante en utilisant uniquement le contenu fourni. "
|
|
|
|
|
+ "Si la réponse est absente, dis-le clairement.\n\n"
|
|
|
|
|
+ f"{top_context}\n\nQuestion : {question}"
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ answer = state["llm"](prompt).strip()
|
|
|
|
|
+ return jsonify({
|
|
|
|
|
+ "answer": answer,
|
|
|
|
|
+ "html_file": f"uploads/ocr_output_words.html"
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ else:
|
|
|
|
|
+ try:
|
|
|
|
|
+ answer = ask_blip(filepath, question)
|
|
|
|
|
+ return jsonify({"answer": answer})
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ return jsonify({"error": f"LLaVA échoué : {str(e)}"}), 500
|
|
|
|
|
+
|
|
|
|
|
+ return jsonify({"error": "Format de fichier non supporté."}), 400
|
|
|
|
|
+
|
|
|
|
|
+@app.route("/delete_file", methods=["POST"])
|
|
|
|
|
+def delete_file():
|
|
|
|
|
+ data = request.get_json()
|
|
|
|
|
+ filename = data.get("filename")
|
|
|
|
|
+
|
|
|
|
|
+ if not filename:
|
|
|
|
|
+ return jsonify({"error": "Nom de fichier manquant."}), 400
|
|
|
|
|
+
|
|
|
|
|
+ filepath = os.path.join(app.config["UPLOAD_FOLDER"], secure_filename(filename))
|
|
|
|
|
+
|
|
|
|
|
+ if os.path.exists(filepath):
|
|
|
|
|
+ os.remove(filepath)
|
|
|
|
|
+ return jsonify({"message": "Fichier supprimé avec succès."})
|
|
|
|
|
+ else:
|
|
|
|
|
+ return jsonify({"error": "Fichier introuvable."}), 404
|
|
|
|
|
+
|
|
|
|
|
+def load_or_update_index():
|
|
|
|
|
+ print("🚀 App starting up...")
|
|
|
|
|
+ index, texts, urls = load_index_from_disk()
|
|
|
|
|
+ if index:
|
|
|
|
|
+ state["index"] = index
|
|
|
|
|
+ state["all_chunks"] = texts
|
|
|
|
|
+ state["chunk_urls"] = urls
|
|
|
|
|
+ print("✅ Index loaded from disk.")
|
|
|
|
|
+ else:
|
|
|
|
|
+ print("❌ No index found. Launching full update in background...")
|
|
|
|
|
+ threading.Thread(target=run_full_update, daemon=True).start()
|
|
|
|
|
+
|
|
|
|
|
+def run_full_update():
|
|
|
|
|
+ state["is_updating"] = True
|
|
|
|
|
+ try:
|
|
|
|
|
+ index, texts, urls = full_update_pipeline()
|
|
|
|
|
+ if index:
|
|
|
|
|
+ state["index"] = index
|
|
|
|
|
+ state["all_chunks"] = texts
|
|
|
|
|
+ state["chunk_urls"] = urls
|
|
|
|
|
+ state["all_embeddings"] = state["embedding_model"].encode(texts)
|
|
|
|
|
+ print("✅ Background update finished.")
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ print(f"❌ Failed to update index: {e}")
|
|
|
|
|
+ finally:
|
|
|
|
|
+ state["is_updating"] = False
|
|
|
|
|
+
|
|
|
|
|
+@app.route('/')
|
|
|
|
|
+def index():
|
|
|
|
|
+ return send_from_directory('templates', 'index.html')
|
|
|
|
|
+
|
|
|
|
|
+@app.route("/login", methods=["POST"])
|
|
|
|
|
+def login():
|
|
|
|
|
+ data = request.json
|
|
|
|
|
+ username = data.get("username")
|
|
|
|
|
+ password = data.get("password")
|
|
|
|
|
+
|
|
|
|
|
+ if not username or not password:
|
|
|
|
|
+ return jsonify({"error": "username and password required"}), 400
|
|
|
|
|
+
|
|
|
|
|
+ if not login_to_wiki(username, password):
|
|
|
|
|
+ return jsonify({"error": "Authentication failed"}), 401
|
|
|
|
|
+
|
|
|
|
|
+ return jsonify({"message": "Login successful"})
|
|
|
|
|
+
|
|
|
|
|
+@app.route("/chat_history", methods=["GET"])
|
|
|
|
|
+def chat_history():
|
|
|
|
|
+ return jsonify(state["chat_history"])
|
|
|
|
|
+
|
|
|
|
|
+@app.route("/qa_chain", methods=["POST"])
|
|
|
|
|
+def qa_chain_route():
|
|
|
|
|
+ if state.get("llm") is None:
|
|
|
|
|
+ return jsonify({"error": "Le modèle LLM n'est pas initialisé."}), 500
|
|
|
|
|
+
|
|
|
|
|
+ if state["index"] is None or not state["all_chunks"]:
|
|
|
|
|
+ if state["is_updating"]:
|
|
|
|
|
+ return jsonify({"error": "Le système est encore en train d’indexer les documents. Réessayez dans quelques instants."}), 503
|
|
|
|
|
+ else:
|
|
|
|
|
+ return jsonify({"error": "L’index est vide. Veuillez attendre la mise à jour ou réinitialiser le système."}), 500
|
|
|
|
|
+
|
|
|
|
|
+ data = request.get_json()
|
|
|
|
|
+ question = data.get("question", "")
|
|
|
|
|
+
|
|
|
|
|
+ if not question:
|
|
|
|
|
+ return jsonify({"error": "Question vide."}), 400
|
|
|
|
|
+
|
|
|
|
|
+ # ÉTAPE 1 : Tentative de réponse avec le Wiki local
|
|
|
|
|
+ try:
|
|
|
|
|
+ query_embedding = state["embedding_model"].encode([question])[0]
|
|
|
|
|
+
|
|
|
|
|
+ if state["all_embeddings"] is None:
|
|
|
|
|
+ state["all_embeddings"] = state["embedding_model"].encode(state["all_chunks"])
|
|
|
|
|
+
|
|
|
|
|
+ similarities = cosine_similarity([query_embedding], state["all_embeddings"])[0]
|
|
|
|
|
+ top_indices = np.argsort(similarities)[::-1][:1500]
|
|
|
|
|
+
|
|
|
|
|
+ rerank_data = [(question, state["all_chunks"][i]) for i in top_indices]
|
|
|
|
|
+ rerank_scores = state["cross_encoder"].predict(rerank_data)
|
|
|
|
|
+ reranked = sorted(
|
|
|
|
|
+ zip([state["all_chunks"][i] for i in top_indices], top_indices, rerank_scores),
|
|
|
|
|
+ key=lambda x: x[2], reverse=True
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ # Sauvegarde du contexte dans un fichier de cache
|
|
|
|
|
+ enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
|
|
|
|
+ cache_data = []
|
|
|
|
|
+
|
|
|
|
|
+ for rank, (chunk, idx, score) in enumerate(reranked[:1500], start=1):
|
|
|
|
|
+ url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
|
|
|
|
|
+ token_count = len(enc.encode(chunk))
|
|
|
|
|
+
|
|
|
|
|
+ cache_data.append({
|
|
|
|
|
+ "rank": rank,
|
|
|
|
|
+ "url": url,
|
|
|
|
|
+ "chunk": chunk,
|
|
|
|
|
+ "tokens": token_count
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ with open("cache_context.json", "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump(cache_data, f, indent=2, ensure_ascii=False)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ # Affichage en terminal
|
|
|
|
|
+ table = []
|
|
|
|
|
+ top_chunks = []
|
|
|
|
|
+ for rank, (chunk, idx, score) in enumerate(reranked, start=1):
|
|
|
|
|
+ url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
|
|
|
|
|
+ wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:1000] + "..."
|
|
|
|
|
+ wrapped_url = "\n".join(textwrap.wrap(url, width=80))
|
|
|
|
|
+ table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
|
|
|
|
|
+ top_chunks.append(chunk)
|
|
|
|
|
+
|
|
|
|
|
+ print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
|
|
|
|
|
+ print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
|
|
|
|
|
+ print("=======================================================================\n")
|
|
|
|
|
+
|
|
|
|
|
+ max_chars = 15000
|
|
|
|
|
+ top_chunks = []
|
|
|
|
|
+ current_length = 0
|
|
|
|
|
+ for chunk, _, _ in reranked:
|
|
|
|
|
+ if current_length + len(chunk) > max_chars:
|
|
|
|
|
+ break
|
|
|
|
|
+ top_chunks.append(chunk)
|
|
|
|
|
+ current_length += len(chunk)
|
|
|
|
|
+
|
|
|
|
|
+ top_context = "\n\n".join(top_chunks)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ prompt = (
|
|
|
|
|
+ "Réponds uniquement à la question en utilisant les informations contenues dans le contexte ci-dessous. "
|
|
|
|
|
+ "Ta réponse doit être factuelle, concise et précise. "
|
|
|
|
|
+ "N’ajoute aucune information extérieure. "
|
|
|
|
|
+ "Si la réponse n’est pas présente dans le contexte, dis simplement : 'Information non disponible.'\n\n"
|
|
|
|
|
+ f"Contexte :\n{top_context}\n\n"
|
|
|
|
|
+ f"Question : {question}"
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ local_answer = state["llm"](prompt).strip()
|
|
|
|
|
+ print("\n====================== PROMPT UTILISÉ ======================")
|
|
|
|
|
+ print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{local_answer}")
|
|
|
|
|
+ print("============================================================\n")
|
|
|
|
|
+
|
|
|
|
|
+ # Vérifie si la réponse est valable
|
|
|
|
|
+ phrases_non_reponse = [
|
|
|
|
|
+ "n'est pas disponible",
|
|
|
|
|
+ "je ne sais pas",
|
|
|
|
|
+ "je ne trouve pas",
|
|
|
|
|
+ "aucune information",
|
|
|
|
|
+ "information non disponible",
|
|
|
|
|
+ "pas d'information",
|
|
|
|
|
+ "je ne suis pas en mesure",
|
|
|
|
|
+ "le contexte ne contient pas",
|
|
|
|
|
+ "https://localhost:8080"
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ if local_answer and not any(phrase in local_answer.lower() for phrase in phrases_non_reponse):
|
|
|
|
|
+ source_urls = [state["chunk_urls"][idx] for _, idx, _ in reranked[:5]]
|
|
|
|
|
+ state["chat_history"].append({
|
|
|
|
|
+ "question": question,
|
|
|
|
|
+ "answer": local_answer,
|
|
|
|
|
+ "sources": list(set(source_urls))
|
|
|
|
|
+ })
|
|
|
|
|
+ return jsonify({
|
|
|
|
|
+ "answer": local_answer,
|
|
|
|
|
+ "sources": list(set(source_urls))
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ print(f"[Erreur Wiki] : {e}")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ # ÉTAPE 2 : Fallback vers recherche web (SerpAPI)
|
|
|
|
|
+ try:
|
|
|
|
|
+ serpapi_key = os.environ.get('SERPAPI_KEY')
|
|
|
|
|
+ if not serpapi_key:
|
|
|
|
|
+ return jsonify({"error": "Clé SerpAPI non configurée"}), 500
|
|
|
|
|
+
|
|
|
|
|
+ params = {
|
|
|
|
|
+ "engine": "google",
|
|
|
|
|
+ "q": question,
|
|
|
|
|
+ "api_key": serpapi_key,
|
|
|
|
|
+ "gl": "fr",
|
|
|
|
|
+ "hl": "fr"
|
|
|
|
|
+ }
|
|
|
|
|
+ search = GoogleSearch(params)
|
|
|
|
|
+ results = search.get_dict()
|
|
|
|
|
+
|
|
|
|
|
+ answer = "Je n'ai pas trouvé de réponse précise."
|
|
|
|
|
+ if "answer_box" in results and "answer" in results["answer_box"]:
|
|
|
|
|
+ answer = results["answer_box"]["answer"]
|
|
|
|
|
+ elif "organic_results" in results and results["organic_results"]:
|
|
|
|
|
+ answer = results["organic_results"][0].get("snippet", answer)
|
|
|
|
|
+
|
|
|
|
|
+ # Résumer ou filtrer la réponse via le LLM
|
|
|
|
|
+ if state.get("llm"):
|
|
|
|
|
+ prompt = (
|
|
|
|
|
+ "Réponds uniquement à la question de manière directe et concise.\n"
|
|
|
|
|
+ "Sans reformuler la question, sans phrase d'introduction, sans politesse et sans contenu promotionnel. "
|
|
|
|
|
+ "Utilise uniquement les faits du texte ci-dessous. "
|
|
|
|
|
+ "Si l'information n'est pas présente, réponds : 'Information non disponible.'\n\n"
|
|
|
|
|
+ f"Texte : {answer}\n\n"
|
|
|
|
|
+ f"Question : {question}"
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ answer = state["llm"](prompt).strip()
|
|
|
|
|
+ except Exception as llm_error:
|
|
|
|
|
+ print("[Erreur LLM sur reformulation] :", llm_error)
|
|
|
|
|
+
|
|
|
|
|
+ # Historique
|
|
|
|
|
+ state["chat_history"].append({
|
|
|
|
|
+ "question": question,
|
|
|
|
|
+ "answer": answer,
|
|
|
|
|
+ "sources": ["Google Search"]
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ return jsonify({
|
|
|
|
|
+ "answer": answer,
|
|
|
|
|
+ "sources": ["Google Search"]
|
|
|
|
|
+ })
|
|
|
|
|
+
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ return jsonify({"error": f"Erreur SerpAPI : {str(e)}"}), 500
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+# Scheduler
|
|
|
|
|
+scheduler = BackgroundScheduler()
|
|
|
|
|
+scheduler.add_job(run_full_update, 'interval', minutes=30)
|
|
|
|
|
+print("🔁 Scheduler activé pour mises à jour toutes les 30 min")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+if __name__ == "__main__":
|
|
|
|
|
+
|
|
|
|
|
+ load_or_update_index()
|
|
|
|
|
+ scheduler.start()
|
|
|
|
|
+
|
|
|
|
|
+ # Configuration pour la production
|
|
|
|
|
+ debug_mode = os.environ.get('FLASK_DEBUG', 'False').lower() == 'true'
|
|
|
|
|
+ host = os.environ.get('FLASK_HOST', '0.0.0.0')
|
|
|
|
|
+ port = int(os.environ.get('FLASK_PORT', 5000))
|
|
|
|
|
+
|
|
|
|
|
+ app.run(debug=debug_mode, host=host, port=port, use_reloader=False)
|