from flask import Flask, request, jsonify, send_from_directory, Response import nltk import numpy as np import json import os from sentence_transformers import CrossEncoder, SentenceTransformer from langchain_ollama import OllamaLLM from agent_ai.auth import login_to_wiki from agent_ai.scraping import fetch_all_pages_content from agent_ai.chunking import semantic_chunking, model, chunk_text, chunk_text_generic from agent_ai.embedding_index import build_faiss_index, search_faiss from agent_ai.qa_chain import get_qa_chain from tabulate import tabulate import textwrap app = Flask(__name__) nltk.download('punkt') # Global state state = { "model": model, "cross_encoder": CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'), "index": None, "all_chunks": [], "chunk_urls": [], "qa_chain": get_qa_chain() } @app.route('/index.html') def index(): return send_from_directory(os.getcwd(), 'index.html') @app.route("/login", methods=["POST"]) def login(): data = request.json username = data.get("username") password = data.get("password") if not username or not password: return jsonify({"error": "username and password required"}), 400 if not login_to_wiki(username, password): return jsonify({"error": "Authentication failed"}), 401 return jsonify({"message": "Login successful"}) @app.route("/fetch_pages", methods=["POST"]) def fetch_pages(): urls = [ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe", "https://wiki.intersel.fr/doku.php?id=intersel:presentation", "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires", "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start", "https://wiki.intersel.fr/doku.php?id=intersel:php", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start", "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire", "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start", "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start" ] pages = fetch_all_pages_content(urls) raw_pages = [{"url": url, "text": text.strip()} for url, text in pages.items()] with open("raw_pages_pretty.json", "w", encoding="utf-8") as f: json.dump(raw_pages, f, ensure_ascii=False, indent=2) print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'") return jsonify(raw_pages) state = { "chunk_urls": [], "model": SentenceTransformer("all-MiniLM-L6-v2"), "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2") } @app.route("/chunk_all", methods=["POST"]) def chunk_all(): data = request.get_json() text = data.get("text", "") url = data.get("url", "url inconnue") if not text.strip(): return jsonify({"error": "Texte vide"}), 400 sentences = nltk.sent_tokenize(text, language='french') chunks = semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=1000) # chunks = chunk_text_generic(text, max_words=500) chunk_data = [] response_chunks = [] for i, chunk in enumerate(chunks, start=1): clean_chunk = chunk.strip() word_count = len(clean_chunk.split()) chunk_data.append({ "chunk_id": i, "text": clean_chunk, "url": url, "word_count": word_count }) response_chunks.append(clean_chunk) # Sécuriser l'état global if "chunk_urls" not in state: state["chunk_urls"] = [] state["chunk_urls"].append(url) # Enregistrement JSON with open("chunks_pretty.json", "a", encoding="utf-8") as f: for chunk in chunk_data: f.write(json.dumps(chunk, ensure_ascii=False) + ",\n") print(f"✅ {len(chunk_data)} chunks enregistrés pour {url}") return jsonify({"chunks": response_chunks}) @app.route("/build_index", methods=["POST"]) def build_index(): data = request.json texts = data.get("texts") urls = data.get("urls") if not texts or not isinstance(texts, list): return jsonify({"error": "A list of texts is required"}), 400 embeddings = state["model"].encode(texts, convert_to_tensor=False) embeddings = np.array(embeddings).astype('float32') index = build_faiss_index(embeddings) state["index"] = index state["all_chunks"] = texts # Important: also assign the urls to the global state here if urls and isinstance(urls, list) and len(urls) == len(texts): state["chunk_urls"] = urls print("✅ Index FAISS construit avec succès") return jsonify({"message": f"Index built with {len(texts)} vectors"}) @app.route("/search_index", methods=["POST"]) def search_index(): data = request.json question = data.get("question") if not question: return jsonify({"error": "Question is required"}), 400 if state["index"] is None: return jsonify({"error": "Index not built yet"}), 400 question_embedding = state["model"].encode([question], convert_to_tensor=False) question_embedding = np.array(question_embedding).astype('float32') if question_embedding.ndim == 1: question_embedding = np.expand_dims(question_embedding, axis=0) distances, indices = search_faiss(state["index"], question_embedding, top_k=20) faiss_chunks = [state["all_chunks"][i] for i in indices[0]] # Re-rank with CrossEncoder cross_encoder = state["cross_encoder"] cross_scores = cross_encoder.predict([(question, chunk) for chunk in faiss_chunks]) reranked = sorted(zip(faiss_chunks, indices[0], cross_scores), key=lambda x: x[2], reverse=True) top_chunks = [chunk for chunk, _, _ in reranked] indices[0] = [idx for _, idx, _ in reranked] state["last_indices"] = indices[0] # Display ranked results in a table table = [] results = [] for rank, (chunk, idx, score) in enumerate(reranked, start=1): url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue" # Wrap chunk and URL text to keep it readable in the terminal wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..." wrapped_url = "\n".join(textwrap.wrap(url, width=80)) table.append([rank, round(score, 4), wrapped_chunk, wrapped_url]) results.append({"text": chunk, "score": float(score), "url": url}) print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================") print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid")) print("=======================================================================\n") top_context = " ".join(top_chunks) return jsonify({"results": results, "context": top_context}) llm = OllamaLLM( model="llama3", temperature=0.3, top_p=0.9, max_tokens=200 ) @app.route("/qa_chain", methods=["POST"]) def qa_chain_route(): data = request.get_json() question = data.get("question", "") top_chunks = [ f"[Source: {state['chunk_urls'][i] if i < len(state['chunk_urls']) else 'URL inconnue'}]\n{state['all_chunks'][i]}" for i in state.get("last_indices", []) if i < len(state["all_chunks"]) ] top_context = "\n\n".join(top_chunks) # Limit context length to avoid overload max_context_length = 5000 if len(top_context) > max_context_length: top_context = top_context[:max_context_length] prompt = ( "Fournis une réponse factuelle et précise en te basant uniquement sur des informations vérifiables. " "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. " "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n" f"{top_context}\n\n" f"Question : {question}" ) result = llm(prompt) print("\n====================== PROMPT UTILISÉ ======================") print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}") print("============================================================\n") return jsonify({"answer": result}) if __name__ == "__main__": app.run(debug=True)