| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240 |
- from flask import Flask, request, jsonify, send_from_directory, Response
- import nltk
- import numpy as np
- import json
- import os
- from sentence_transformers import CrossEncoder, SentenceTransformer
- from langchain_ollama import OllamaLLM
- from agent_ai.auth import login_to_wiki
- from agent_ai.scraping import fetch_all_pages_content
- from agent_ai.chunking import semantic_chunking, model
- from agent_ai.embedding_index import build_faiss_index, search_faiss
- from agent_ai.qa_chain import get_qa_chain
- from tabulate import tabulate
- import textwrap
- app = Flask(__name__)
- nltk.download('punkt')
- # Global state
- state = {
- "model": model,
- "cross_encoder": CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2'),
- "index": None,
- "all_chunks": [],
- "chunk_urls": [],
- "qa_chain": get_qa_chain()
- }
- @app.route('/index.html')
- def index():
- return send_from_directory(os.getcwd(), 'index.html')
- @app.route("/login", methods=["POST"])
- def login():
- data = request.json
- username = data.get("username")
- password = data.get("password")
- if not username or not password:
- return jsonify({"error": "username and password required"}), 400
- if not login_to_wiki(username, password):
- return jsonify({"error": "Authentication failed"}), 401
- return jsonify({"message": "Login successful"})
- @app.route("/fetch_pages", methods=["POST"])
- def fetch_pages():
- urls = [
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
- "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
- "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start",
- "https://wiki.intersel.fr/doku.php?id=intersel:php",
- "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:admin_manual:start",
- "https://wiki.intersel.fr/doku.php?id=intersel:resume_annuaire",
- "https://wiki.intersel.fr/doku.php?id=technique:developpement:developpement_phplist:start",
- "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:projets:integration_metronic:start",
- "https://wiki.intersel.fr/doku.php?id=produit:developpement:livinweb:documentations:dev_manual:start"
- ]
- pages = fetch_all_pages_content(urls)
- raw_pages = [{"url": url, "text": text.strip()} for url, text in pages.items()]
-
- with open("raw_pages_pretty.json", "w", encoding="utf-8") as f:
- json.dump(raw_pages, f, ensure_ascii=False, indent=2)
-
- print("📄 Contenu brut des pages enregistré dans 'raw_pages_pretty.json'")
- return jsonify(raw_pages)
- state = {
- "chunk_urls": [],
- "model": SentenceTransformer("all-MiniLM-L6-v2"),
- "cross_encoder": CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
- }
- @app.route("/chunk_all", methods=["POST"])
- def chunk_all():
- data = request.get_json()
- text = data.get("text", "")
- url = data.get("url", "url inconnue")
- if not text.strip():
- return jsonify({"error": "Texte vide"}), 400
- sentences = nltk.sent_tokenize(text, language='french')
- chunks = semantic_chunking(sentences, similarity_threshold=0.10, max_words_per_chunk=1000)
- # chunks = chunk_text_generic(text, max_words=500)
- chunk_data = []
- response_chunks = []
- for i, chunk in enumerate(chunks, start=1):
- clean_chunk = chunk.strip()
- word_count = len(clean_chunk.split())
- chunk_data.append({
- "chunk_id": i,
- "text": clean_chunk,
- "url": url,
- "word_count": word_count
- })
- response_chunks.append(clean_chunk)
- # Sécuriser l'état global
- if "chunk_urls" not in state:
- state["chunk_urls"] = []
- state["chunk_urls"].append(url)
- # Enregistrement JSON
- with open("chunks_pretty.json", "a", encoding="utf-8") as f:
- for chunk in chunk_data:
- f.write(json.dumps(chunk, ensure_ascii=False) + ",\n")
- print(f"✅ {len(chunk_data)} chunks enregistrés pour {url}")
- return jsonify({"chunks": response_chunks})
- @app.route("/build_index", methods=["POST"])
- def build_index():
- data = request.json
- texts = data.get("texts")
- urls = data.get("urls")
- if not texts or not isinstance(texts, list):
- return jsonify({"error": "A list of texts is required"}), 400
- embeddings = state["model"].encode(texts, convert_to_tensor=False)
- embeddings = np.array(embeddings).astype('float32')
- index = build_faiss_index(embeddings)
- state["index"] = index
- state["all_chunks"] = texts
- # Important: also assign the urls to the global state here
- if urls and isinstance(urls, list) and len(urls) == len(texts):
- state["chunk_urls"] = urls
-
- print("✅ Index FAISS construit avec succès")
- return jsonify({"message": f"Index built with {len(texts)} vectors"})
- @app.route("/search_index", methods=["POST"])
- def search_index():
- data = request.json
- question = data.get("question")
- if not question:
- return jsonify({"error": "Question is required"}), 400
- if state["index"] is None:
- return jsonify({"error": "Index not built yet"}), 400
- question_embedding = state["model"].encode([question], convert_to_tensor=False)
- question_embedding = np.array(question_embedding).astype('float32')
- if question_embedding.ndim == 1:
- question_embedding = np.expand_dims(question_embedding, axis=0)
- distances, indices = search_faiss(state["index"], question_embedding, top_k=20)
- faiss_chunks = [state["all_chunks"][i] for i in indices[0]]
- # Re-rank with CrossEncoder
- cross_encoder = state["cross_encoder"]
- cross_scores = cross_encoder.predict([(question, chunk) for chunk in faiss_chunks])
- reranked = sorted(zip(faiss_chunks, indices[0], cross_scores), key=lambda x: x[2], reverse=True)
- top_chunks = [chunk for chunk, _, _ in reranked]
- indices[0] = [idx for _, idx, _ in reranked]
- state["last_indices"] = indices[0]
- # Display ranked results in a table
- table = []
- results = []
- for rank, (chunk, idx, score) in enumerate(reranked, start=1):
- url = state["chunk_urls"][idx] if idx < len(state["chunk_urls"]) else "URL inconnue"
- # Wrap chunk and URL text to keep it readable in the terminal
- wrapped_chunk = "\n".join(textwrap.wrap(chunk.strip(), width=80))[:500] + "..."
- wrapped_url = "\n".join(textwrap.wrap(url, width=80))
- table.append([rank, round(score, 4), wrapped_chunk, wrapped_url])
- results.append({"text": chunk, "score": float(score), "url": url})
- print("\n================== RÉSULTATS CLASSÉS PAR PERTINENCE ==================")
- print(tabulate(table, headers=["Rank", "Score", "Chunk", "URL"], tablefmt="fancy_grid"))
- print("=======================================================================\n")
- top_context = " ".join(top_chunks)
- return jsonify({"results": results, "context": top_context})
- llm = OllamaLLM(
- model="llama3",
- temperature=0.3,
- top_p=0.9,
- max_tokens=200
- )
- @app.route("/qa_chain", methods=["POST"])
- def qa_chain_route():
- data = request.get_json()
- question = data.get("question", "")
- top_chunks = [
- # Pour chaque indice 'i' dans la liste 'last_indices' (indices des meilleurs chunks)
- f"[Source: {state['chunk_urls'][i] if i < len(state['chunk_urls']) else 'URL inconnue'}]\n{state['all_chunks'][i]}" # On récupère l'URL correspondante à cet indice si elle existe, sinon on met 'URL inconnue' Puis on ajoute le texte complet du chunk correspondant
- for i in state.get("last_indices", [])
- if i < len(state["all_chunks"]) # On s'assure que l'indice est valide pour éviter les erreurs d'index hors limites
- ]
- top_context = "\n\n".join(top_chunks)
- # Limit context length to avoid overload
- max_context_length = 5000
- if len(top_context) > max_context_length:
- top_context = top_context[:max_context_length]
- prompt = (
- "Fournis une réponse factuelle et précise en te basant uniquement sur des informations vérifiables. "
- "Si tu n'es pas certain de la réponse ou si les données sont ambiguës, indique clairement que l'information n'est pas disponible ou nécessite une vérification. "
- "Ne fais aucune supposition ou inférence qui pourrait être incorrecte, et précise les sources utilisées lorsque possible.\n\n"
- f"{top_context}\n\n"
- f"Question : {question}"
- )
- result = llm(prompt)
- print("\n====================== PROMPT UTILISÉ ======================")
- print(f"CONTEXTE :\n{top_context}\nQUESTION :\n{question}\nRÉPONSE :\n{result}")
- print("============================================================\n")
- return jsonify({"answer": result})
- if __name__ == "__main__":
- app.run(debug=True)
|