import os import json import faiss import numpy as np from sentence_transformers import SentenceTransformer from agent_ai.chunking import semantic_chunking from agent_ai.scraping import scrape_pages from nltk.tokenize import sent_tokenize import re import requests CHUNKS_FILE = "data/All_chunks.json" INDEX_FILE = "data/faiss_index.index" RAW_PAGES_FILE = "data/raw_pages.json" URLS_FILE = "data/urls.json" embedding_model = SentenceTransformer("all-MiniLM-L6-v2") def clean_text(text): # Supprimer les balises DokuWiki comme ~~ODT~~ ou ~~TOC~~ text = re.sub(r"~~.*?~~", " ", text) # Supprimer les balises HTML restantes éventuelles text = re.sub(r"<[^>]+>", " ", text) # supprimer les caractères non imprimables sauf accentués text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text) # supprimer les espaces multiples text = re.sub(r"\s+", " ", text) return text.strip() def update_raw_pages(new_pages_dict): # Charger les anciennes pages old_pages = {} if os.path.exists(RAW_PAGES_FILE): with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f: old_pages = json.load(f) # Comparer et détecter les modifications updated = {} for url, new_text in new_pages_dict.items(): if url not in old_pages or old_pages[url] != new_text: updated[url] = new_text # Ajouté ou modifié # Supprimer les anciens contenus modifiés for url in updated.keys(): old_pages.pop(url, None) # Ajouter les nouveaux contenus old_pages.update(updated) with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f: json.dump(old_pages, f, indent=2, ensure_ascii=False) # 🔁 Mettre à jour la liste des URLs if os.path.exists(URLS_FILE): with open(URLS_FILE, "r", encoding="utf-8") as f: all_urls = set(json.load(f)) else: all_urls = set() all_urls.update(new_pages_dict.keys()) with open(URLS_FILE, "w", encoding="utf-8") as f: json.dump(list(all_urls), f, indent=2) return updated def update_chunks_for_modified_pages(updated_pages): if os.path.exists(CHUNKS_FILE): with open(CHUNKS_FILE, "r", encoding="utf-8") as f: all_chunks = json.load(f) else: all_chunks = [] # Remove old chunks for updated URLs updated_urls = set(updated_pages.keys()) all_chunks = [c for c in all_chunks if c["url"] not in updated_urls] # Add new chunks chunk_index_start = len(all_chunks) new_chunks = [] for url, text in updated_pages.items(): text = clean_text(text) sentences = sent_tokenize(text) chunks = semantic_chunking(sentences, similarity_threshold=0.6) for ch in chunks: new_chunks.append({ "chunk_id": chunk_index_start + len(new_chunks), "text": ch, "url": url, "word_count": len(ch.split()) }) all_chunks.extend(new_chunks) with open(CHUNKS_FILE, "w", encoding="utf-8") as f: json.dump(all_chunks, f, indent=2, ensure_ascii=False) return all_chunks def build_and_save_index(chunks): texts = [c["text"] for c in chunks] urls = [c["url"] for c in chunks] embeddings = embedding_model.encode(texts, convert_to_tensor=False) index = faiss.IndexFlatL2(len(embeddings[0])) index.add(np.array(embeddings).astype("float32")) # ✅ Save index to disk faiss.write_index(index, INDEX_FILE) return index, texts, urls def full_update_pipeline(): print("🔄 Running full update pipeline...") pages = scrape_pages() page_dict = {p["url"]: p["text"] for p in pages} updated = update_raw_pages(page_dict) if updated: all_chunks = update_chunks_for_modified_pages(updated) else: if os.path.exists(CHUNKS_FILE): with open(CHUNKS_FILE, "r", encoding="utf-8") as f: all_chunks = json.load(f) else: print("❌ Aucune donnée pour construire l'index.") return None, None, None index, texts, urls = build_and_save_index(all_chunks) print("✅ Index reconstruit (même sans mises à jour).") return index, texts, urls def load_index_from_disk(): if not os.path.exists(INDEX_FILE) or not os.path.exists(CHUNKS_FILE): return None, [], [] index = faiss.read_index(INDEX_FILE) with open(CHUNKS_FILE, "r", encoding="utf-8") as f: chunks = json.load(f) texts = [chunk["text"] for chunk in chunks] urls = [chunk["url"] for chunk in chunks] return index, texts, urls