| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168 |
- import os
- import time
- import json
- import re
- import faiss
- import numpy as np
- from sentence_transformers import SentenceTransformer
- from agent_ai.chunking import split_into_chunks, count_tokens
- from agent_ai.scraping import scrape_pages
- # === Paths ===
- os.makedirs("data", exist_ok=True)
- CHUNKS_FILE = "data/All_chunks.json"
- INDEX_FILE = "data/faiss_index.index"
- RAW_PAGES_FILE = "data/raw_pages.json"
- URLS_FILE = "data/urls.json"
- # === Embedding Model ===
- model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True)
- # === Cleaning Utility ===
- def clean_text(text):
- text = re.sub(r"~~.*?~~", " ", text)
- text = re.sub(r"<[^>]+>", " ", text)
- text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
- text = re.sub(r"\s+", " ", text)
- return text.strip()
- # === Step 1: Update Pages ===
- def update_raw_pages(new_pages_dict):
- old_pages = {}
- try:
- with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
- old_pages = {p["url"]: p["text"] for p in json.load(f)}
- except Exception as e:
- print(f"⚠️ Could not load old raw pages: {e}")
- updated = {url: text for url, text in new_pages_dict.items()
- if url not in old_pages or old_pages[url] != text}
- # No need to pop, just update with new/modified pages
- old_pages.update(updated)
- with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
- json.dump([{"url": url, "text": text} for url, text in old_pages.items()], f, indent=2, ensure_ascii=False)
- # Update URL list
- all_urls = set(old_pages.keys())
- with open(URLS_FILE, "w", encoding="utf-8") as f:
- json.dump(list(all_urls), f, indent=2)
- print(f"📄 Raw pages updated: {len(updated)} pages modified.")
- return updated
- # === Step 2: Update Chunks ===
- def update_chunks_for_modified_pages(updated_pages):
- # Load old chunks
- all_chunks = []
- if os.path.exists(CHUNKS_FILE):
- try:
- with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
- all_chunks = json.load(f)
- except Exception as e:
- print(f"⚠️ Could not load old chunks: {e}")
- all_chunks = []
- # Remove outdated chunks for updated URLs
- updated_urls = set(updated_pages.keys())
- all_chunks = [c for c in all_chunks if c.get("url") not in updated_urls]
- new_chunks = []
- chunk_id_start = len(all_chunks)
- for url, text in updated_pages.items():
- cleaned = clean_text(text)
- if not cleaned:
- continue
- chunks = split_into_chunks(cleaned, with_metadata=True)
- for ch in chunks:
- # safety check: ch must be dict with string 'text'
- if not isinstance(ch, dict) or not isinstance(ch.get("text"), str):
- print(f"⚠️ Ignored malformed chunk: {ch}")
- continue
- new_chunks.append({
- "chunk_id": chunk_id_start + len(new_chunks),
- "text": ch["text"],
- "url": url,
- "word_count": ch["word_count"],
- "token_count": ch["token_count"]
- })
- all_chunks.extend(new_chunks)
- with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
- json.dump(all_chunks, f, indent=2, ensure_ascii=False)
- print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.")
- return all_chunks
- # === Step 3: Build Index ===
- def build_and_save_index(chunks):
- if not chunks:
- print("⚠️ No chunks to index.")
- return None
- texts = [c["text"] for c in chunks]
- embeddings = model.encode(texts, convert_to_tensor=False)
- if len(embeddings) == 0:
- print("⚠️ Embeddings generation failed or empty.")
- return None
- dim = len(embeddings[0])
- index = faiss.IndexFlatL2(dim)
- index.add(np.array(embeddings).astype("float32"))
- faiss.write_index(index, INDEX_FILE)
- print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.")
- return index
- # === Full Pipeline ===
- def full_update_pipeline():
- print("🔄 Pipeline complet déclenché...")
- pages = scrape_pages()
- page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p}
- if not page_dict:
- print("⚠️ Aucune page récupérée.")
- return
- index_exists = os.path.exists(INDEX_FILE)
-
- # First run: build everything from scratch
- if not index_exists:
- print("🚀 Première exécution : génération complète des chunks et de l'index.")
- with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
- json.dump([{"url": url, "text": text} for url, text in page_dict.items()], f, indent=2, ensure_ascii=False)
- with open(URLS_FILE, "w", encoding="utf-8") as f:
- json.dump(list(page_dict.keys()), f, indent=2)
- chunks = update_chunks_for_modified_pages(page_dict)
- build_and_save_index(chunks)
- else:
- updated = update_raw_pages(page_dict)
- if updated:
- chunks = update_chunks_for_modified_pages(updated)
- build_and_save_index(chunks)
- else:
- print("✅ Aucun changement détecté. Pas d'update nécessaire.")
- # === Init (if needed) ===
- def initialize_if_needed():
- if not os.path.exists(URLS_FILE) or not os.path.exists(INDEX_FILE):
- print("🚨 Initializing system...")
- full_update_pipeline()
- # === Auto-refresh every 30 mins ===
- def start_auto_refresh(interval_minutes=30):
- initialize_if_needed()
- while True:
- print(f"\n⏰ Checking for updates every {interval_minutes} minutes...")
- full_update_pipeline()
- time.sleep(interval_minutes * 60)
|