import os import time import json import re import faiss import numpy as np from sentence_transformers import SentenceTransformer from agent_ai.chunking import split_into_chunks, count_tokens from agent_ai.scraping import scrape_pages # === Paths === os.makedirs("data", exist_ok=True) CHUNKS_FILE = "data/All_chunks.json" INDEX_FILE = "data/faiss_index.index" RAW_PAGES_FILE = "data/raw_pages.json" URLS_FILE = "data/urls.json" # === Embedding Model === model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True) # === Cleaning Utility === def clean_text(text): text = re.sub(r"~~.*?~~", " ", text) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text) text = re.sub(r"\s+", " ", text) return text.strip() # === Step 1: Update Pages === def update_raw_pages(new_pages_dict): old_pages = {} try: with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f: old_pages = {p["url"]: p["text"] for p in json.load(f)} except Exception as e: print(f"⚠️ Could not load old raw pages: {e}") updated = {url: text for url, text in new_pages_dict.items() if url not in old_pages or old_pages[url] != text} # No need to pop, just update with new/modified pages old_pages.update(updated) with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f: json.dump([{"url": url, "text": text} for url, text in old_pages.items()], f, indent=2, ensure_ascii=False) # Update URL list all_urls = set(old_pages.keys()) with open(URLS_FILE, "w", encoding="utf-8") as f: json.dump(list(all_urls), f, indent=2) print(f"📄 Raw pages updated: {len(updated)} pages modified.") return updated # === Step 2: Update Chunks === def update_chunks_for_modified_pages(updated_pages): # Load old chunks all_chunks = [] if os.path.exists(CHUNKS_FILE): try: with open(CHUNKS_FILE, "r", encoding="utf-8") as f: all_chunks = json.load(f) except Exception as e: print(f"⚠️ Could not load old chunks: {e}") all_chunks = [] # Remove outdated chunks for updated URLs updated_urls = set(updated_pages.keys()) all_chunks = [c for c in all_chunks if c.get("url") not in updated_urls] new_chunks = [] chunk_id_start = len(all_chunks) for url, text in updated_pages.items(): cleaned = clean_text(text) if not cleaned: continue chunks = split_into_chunks(cleaned, with_metadata=True) for ch in chunks: # safety check: ch must be dict with string 'text' if not isinstance(ch, dict) or not isinstance(ch.get("text"), str): print(f"⚠️ Ignored malformed chunk: {ch}") continue new_chunks.append({ "chunk_id": chunk_id_start + len(new_chunks), "text": ch["text"], "url": url, "word_count": ch["word_count"], "token_count": ch["token_count"] }) all_chunks.extend(new_chunks) with open(CHUNKS_FILE, "w", encoding="utf-8") as f: json.dump(all_chunks, f, indent=2, ensure_ascii=False) print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.") return all_chunks # === Step 3: Build Index === def build_and_save_index(chunks): if not chunks: print("⚠️ No chunks to index.") return None texts = [c["text"] for c in chunks] embeddings = model.encode(texts, convert_to_tensor=False) if len(embeddings) == 0: print("⚠️ Embeddings generation failed or empty.") return None dim = len(embeddings[0]) index = faiss.IndexFlatL2(dim) index.add(np.array(embeddings).astype("float32")) faiss.write_index(index, INDEX_FILE) print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.") return index # === Full Pipeline === def full_update_pipeline(): print("🔄 Pipeline complet déclenché...") pages = scrape_pages() page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p} if not page_dict: print("⚠️ Aucune page récupérée.") return index_exists = os.path.exists(INDEX_FILE) # First run: build everything from scratch if not index_exists: print("🚀 Première exécution : génération complète des chunks et de l'index.") with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f: json.dump([{"url": url, "text": text} for url, text in page_dict.items()], f, indent=2, ensure_ascii=False) with open(URLS_FILE, "w", encoding="utf-8") as f: json.dump(list(page_dict.keys()), f, indent=2) chunks = update_chunks_for_modified_pages(page_dict) build_and_save_index(chunks) else: updated = update_raw_pages(page_dict) if updated: chunks = update_chunks_for_modified_pages(updated) build_and_save_index(chunks) else: print("✅ Aucun changement détecté. Pas d'update nécessaire.") # === Init (if needed) === def initialize_if_needed(): if not os.path.exists(URLS_FILE) or not os.path.exists(INDEX_FILE): print("🚨 Initializing system...") full_update_pipeline() # === Auto-refresh every 30 mins === def start_auto_refresh(interval_minutes=30): initialize_if_needed() while True: print(f"\n⏰ Checking for updates every {interval_minutes} minutes...") full_update_pipeline() time.sleep(interval_minutes * 60)