import os import json import re import faiss import numpy as np from sentence_transformers import SentenceTransformer from src.agent_ai.chunking import semantic_chunking from src.agent_ai.scraping import scrape_pages # === Paths === os.makedirs("data", exist_ok=True) CHUNKS_FILE = "data/All_chunks.json" INDEX_FILE = "data/faiss_index.index" RAW_PAGES_FILE = "data/raw_pages.json" # === Embedding Model === '''model = SentenceTransformer( 'nomic-ai/nomic-embed-text-v1', device='cuda', trust_remote_code=True )''' model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', device='cpu') # === Cleaning Utility === def clean_text(text): text = re.sub(r"~~.*?~~", " ", text) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text) text = re.sub(r"\s+", " ", text) return text.strip() # === Step 2: Update Chunks === def update_chunks(pages_dict): all_chunks = [] if os.path.exists(CHUNKS_FILE): try: with open(CHUNKS_FILE, "r", encoding="utf-8") as f: all_chunks = json.load(f) except Exception as e: print(f"⚠️ Could not load old chunks: {e}") # Supprimer tous les chunks des URLs modifiées existing_urls = set(pages_dict.keys()) all_chunks = [c for c in all_chunks if c.get("url") not in existing_urls] new_chunks = [] chunk_id_start = len(all_chunks) for url, text in pages_dict.items(): cleaned = clean_text(text) if not cleaned: continue chunks = semantic_chunking(cleaned, with_metadata=True) for ch in chunks: if not isinstance(ch, dict) or not isinstance(ch.get("text"), str): print(f"⚠️ Ignored malformed chunk: {ch}") continue new_chunks.append({ "chunk_id": chunk_id_start + len(new_chunks), "text": ch["text"], "url": url, "word_count": ch["word_count"], "token_count": ch["token_count"] }) all_chunks.extend(new_chunks) with open(CHUNKS_FILE, "w", encoding="utf-8") as f: json.dump(all_chunks, f, indent=2, ensure_ascii=False) print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.") return all_chunks # === Step 3: Build Index === def build_and_save_index(chunks): if not chunks: print("⚠️ No chunks to index.") return None texts = [c["text"] for c in chunks] embeddings = model.encode(texts, convert_to_tensor=False, batch_size=4) if len(embeddings) == 0: print("⚠️ Embeddings generation failed or empty.") return None dim = len(embeddings[0]) index = faiss.IndexFlatL2(dim) index.add(np.array(embeddings).astype("float32")) faiss.write_index(index, INDEX_FILE) print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.") return index # === Full Pipeline === def full_update_pipeline(): print("🔄 Pipeline complet déclenché...") pages = scrape_pages() page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p} if not page_dict: print("⚠️ Aucune page récupérée.") return chunks = update_chunks(page_dict) build_and_save_index(chunks) # === Initialize if needed === def initialize_if_needed(): full_update_pipeline()