|
|
@@ -1,6 +1,7 @@
|
|
|
import os
|
|
|
import json
|
|
|
import re
|
|
|
+import time
|
|
|
import faiss
|
|
|
import numpy as np
|
|
|
from sentence_transformers import SentenceTransformer
|
|
|
@@ -8,6 +9,7 @@ from src.agent_ai.chunking import semantic_chunking
|
|
|
from src.agent_ai.scraping import scrape_pages
|
|
|
from threading import Lock
|
|
|
import logging
|
|
|
+from datetime import datetime
|
|
|
|
|
|
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
|
logger = logging.getLogger(__name__)
|
|
|
@@ -23,6 +25,7 @@ os.makedirs(data_dir, exist_ok=True)
|
|
|
CHUNKS_FILE = os.path.join(data_dir, "All_chunks.json")
|
|
|
INDEX_FILE = os.path.join(data_dir, "faiss_index.index")
|
|
|
RAW_PAGES_FILE = os.path.join(data_dir, "raw_pages.json")
|
|
|
+PIPELINE_STATUS_FILE = os.path.join(data_dir, "pipeline_status.json")
|
|
|
|
|
|
# === Embedding Model ===
|
|
|
#model = SentenceTransformer('sentence-transformers/all-roberta-large-v1')
|
|
|
@@ -121,43 +124,172 @@ def build_and_save_index(chunks):
|
|
|
return index
|
|
|
|
|
|
|
|
|
-# === Full Pipeline ===
|
|
|
-
|
|
|
-def full_update_pipeline():
|
|
|
- print("🔄 Pipeline complet déclenché...")
|
|
|
-
|
|
|
- if os.path.exists(CHUNKS_FILE):
|
|
|
- print("📄 Fichier de chunks trouvé. Chargement en cours...")
|
|
|
+# === Pipeline status (logs & monitoring) ===
|
|
|
+
|
|
|
+def _load_pipeline_status():
|
|
|
+ """Charge le statut du dernier pipeline depuis le fichier."""
|
|
|
+ if not os.path.exists(PIPELINE_STATUS_FILE):
|
|
|
+ return {
|
|
|
+ "last_run": None,
|
|
|
+ "duration_seconds": None,
|
|
|
+ "chunk_count": 0,
|
|
|
+ "mode": None,
|
|
|
+ "error": None,
|
|
|
+ "scraping_stats": None,
|
|
|
+ "ready": False,
|
|
|
+ }
|
|
|
+ try:
|
|
|
+ with open(PIPELINE_STATUS_FILE, "r", encoding="utf-8") as f:
|
|
|
+ return json.load(f)
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning("Impossible de charger pipeline_status.json: %s", e)
|
|
|
+ return {"last_run": None, "duration_seconds": None, "chunk_count": 0, "error": str(e), "ready": False}
|
|
|
+
|
|
|
+
|
|
|
+def _save_pipeline_status(status: dict):
|
|
|
+ """Sauvegarde le statut du pipeline (durée, chunks, erreurs)."""
|
|
|
+ try:
|
|
|
+ with open(PIPELINE_STATUS_FILE, "w", encoding="utf-8") as f:
|
|
|
+ json.dump(status, f, ensure_ascii=False, indent=2)
|
|
|
+ except Exception as e:
|
|
|
+ logger.warning("Impossible de sauvegarder pipeline_status.json: %s", e)
|
|
|
+
|
|
|
+
|
|
|
+def get_pipeline_status():
|
|
|
+ """Retourne le statut du pipeline (pour /api/status). Met à jour 'ready' selon l'état actuel."""
|
|
|
+ status = _load_pipeline_status()
|
|
|
+ chunks_ok = os.path.exists(CHUNKS_FILE) and os.path.getsize(CHUNKS_FILE) > 2
|
|
|
+ index_ok = os.path.exists(INDEX_FILE) and os.path.getsize(INDEX_FILE) > 0
|
|
|
+ status["ready"] = bool(chunks_ok and index_ok)
|
|
|
+ return status
|
|
|
+
|
|
|
+
|
|
|
+# === Première indexation (aucun chunk/index existant) ===
|
|
|
+
|
|
|
+def _run_first_indexation() -> dict:
|
|
|
+ """
|
|
|
+ Première indexation complète : scrape tout le wiki, chunk, build index.
|
|
|
+ Appelée quand All_chunks.json et faiss_index.index n'existent pas.
|
|
|
+ """
|
|
|
+ logger.info("🔄 Première indexation : scraping complet du wiki...")
|
|
|
+ pages_list, scraping_stats = scrape_pages()
|
|
|
+ page_dict = {p["url"]: p["text"] for p in pages_list if "url" in p and "text" in p}
|
|
|
+
|
|
|
+ if not page_dict:
|
|
|
+ return {
|
|
|
+ "mode": "first_indexation",
|
|
|
+ "error": "Aucune page récupérée lors du scraping.",
|
|
|
+ "scraping_stats": scraping_stats,
|
|
|
+ "chunk_count": 0,
|
|
|
+ }
|
|
|
+
|
|
|
+ chunks = update_chunks(page_dict)
|
|
|
+ build_and_save_index(chunks)
|
|
|
+ return {
|
|
|
+ "mode": "first_indexation",
|
|
|
+ "error": None,
|
|
|
+ "scraping_stats": scraping_stats,
|
|
|
+ "chunk_count": len(chunks),
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# === Mise à jour incrémentale (chunks/index déjà présents) ===
|
|
|
+
|
|
|
+def _run_incremental_update() -> dict:
|
|
|
+ """
|
|
|
+ Mise à jour incrémentale : scrape, détecte les pages modifiées (hash),
|
|
|
+ met à jour uniquement les chunks concernés, reconstruit l'index.
|
|
|
+ """
|
|
|
+ logger.info("🔄 Mise à jour incrémentale : détection des pages modifiées...")
|
|
|
+ pages_list, scraping_stats = scrape_pages()
|
|
|
+ page_dict = {p["url"]: p["text"] for p in pages_list if "url" in p and "text" in p}
|
|
|
+
|
|
|
+ # Même si aucune page modifiée, on garde les chunks existants
|
|
|
+ if not page_dict:
|
|
|
chunks = load_chunks_from_file()
|
|
|
+ return {
|
|
|
+ "mode": "incremental_update",
|
|
|
+ "error": None,
|
|
|
+ "scraping_stats": scraping_stats,
|
|
|
+ "chunk_count": len(chunks),
|
|
|
+ "pages_updated": 0,
|
|
|
+ }
|
|
|
+
|
|
|
+ chunks = update_chunks(page_dict)
|
|
|
+ build_and_save_index(chunks)
|
|
|
+ return {
|
|
|
+ "mode": "incremental_update",
|
|
|
+ "error": None,
|
|
|
+ "scraping_stats": scraping_stats,
|
|
|
+ "chunk_count": len(chunks),
|
|
|
+ "pages_updated": len(page_dict),
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+# === Pipeline unifié ===
|
|
|
+
|
|
|
+def full_update_pipeline() -> dict:
|
|
|
+ """
|
|
|
+ Pipeline unifié : distingue clairement première indexation vs mise à jour incrémentale.
|
|
|
+ Retourne un dict avec mode, durée, chunk_count, erreurs, scraping_stats.
|
|
|
+ """
|
|
|
+ start = time.time()
|
|
|
+ chunks_exist = os.path.exists(CHUNKS_FILE)
|
|
|
+ index_exists = os.path.exists(INDEX_FILE)
|
|
|
+
|
|
|
+ if not chunks_exist or not index_exists:
|
|
|
+ result = _run_first_indexation()
|
|
|
else:
|
|
|
- print("🕸️ Aucun fichier de chunks trouvé. Scraping des pages...")
|
|
|
- pages = scrape_pages()
|
|
|
- page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p}
|
|
|
-
|
|
|
- if not page_dict:
|
|
|
- print("⚠️ Aucune page récupérée.")
|
|
|
- return
|
|
|
+ result = _run_incremental_update()
|
|
|
+
|
|
|
+ result["duration_seconds"] = round(time.time() - start, 2)
|
|
|
+ result["last_run"] = datetime.utcnow().isoformat() + "Z"
|
|
|
+
|
|
|
+ # Persister le statut pour monitoring
|
|
|
+ status = {
|
|
|
+ "last_run": result["last_run"],
|
|
|
+ "duration_seconds": result["duration_seconds"],
|
|
|
+ "chunk_count": result["chunk_count"],
|
|
|
+ "mode": result["mode"],
|
|
|
+ "error": result.get("error"),
|
|
|
+ "scraping_stats": result.get("scraping_stats"),
|
|
|
+ "ready": result.get("error") is None and result["chunk_count"] > 0,
|
|
|
+ }
|
|
|
+ _save_pipeline_status(status)
|
|
|
+
|
|
|
+ logger.info(
|
|
|
+ "✅ Pipeline terminé en %.2fs | mode=%s | chunks=%d",
|
|
|
+ result["duration_seconds"],
|
|
|
+ result["mode"],
|
|
|
+ result["chunk_count"],
|
|
|
+ )
|
|
|
+ return result
|
|
|
|
|
|
- chunks = update_chunks(page_dict)
|
|
|
- #save_chunks_to_file(chunks)
|
|
|
-
|
|
|
- if os.path.exists(INDEX_FILE):
|
|
|
- print("📦 Fichier d'index trouvé. Chargement non nécessaire.")
|
|
|
- index = faiss.read_index(INDEX_FILE)
|
|
|
- else:
|
|
|
- print("📦 Aucun fichier d'index trouvé. Construction de l'index...")
|
|
|
- build_and_save_index(chunks)
|
|
|
|
|
|
# ----------------- LOCK POUR MISE À JOUR -----------------------
|
|
|
update_lock = Lock()
|
|
|
|
|
|
+
|
|
|
def run_full_update():
|
|
|
+ """Appelé par le scheduler. Exécute le pipeline et recharge le state."""
|
|
|
if update_lock.locked():
|
|
|
logger.info("Mise à jour déjà en cours, nouvelle exécution ignorée.")
|
|
|
return
|
|
|
with update_lock:
|
|
|
logger.info("🔄 Lancement du pipeline via le scheduler...")
|
|
|
try:
|
|
|
- full_update_pipeline()
|
|
|
+ result = full_update_pipeline()
|
|
|
+ from src.service.state import state, load_chunks_and_index
|
|
|
+ state["all_chunks"], state["index"] = load_chunks_and_index()
|
|
|
+ logger.info("State rechargé après mise à jour (%d chunks).", len(state.get("all_chunks", [])))
|
|
|
except Exception as e:
|
|
|
- logger.error(f"Erreur pendant la mise à jour complète : {e}")
|
|
|
+ logger.error("Erreur pendant la mise à jour complète : %s", e)
|
|
|
+ _save_pipeline_status({
|
|
|
+ "last_run": datetime.utcnow().isoformat() + "Z",
|
|
|
+ "duration_seconds": None,
|
|
|
+ "chunk_count": 0,
|
|
|
+ "mode": None,
|
|
|
+ "error": str(e),
|
|
|
+ "scraping_stats": None,
|
|
|
+ "ready": False,
|
|
|
+ })
|