| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115 |
- import os
- import json
- import re
- import faiss
- import numpy as np
- from sentence_transformers import SentenceTransformer
- from src.agent_ai.chunking import semantic_chunking
- from src.agent_ai.scraping import scrape_pages
- # === Paths ===
- os.makedirs("data", exist_ok=True)
- CHUNKS_FILE = "data/All_chunks.json"
- INDEX_FILE = "data/faiss_index.index"
- RAW_PAGES_FILE = "data/raw_pages.json"
- # === Embedding Model ===
- '''model = SentenceTransformer(
- 'nomic-ai/nomic-embed-text-v1',
- device='cuda',
- trust_remote_code=True
- )'''
- model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', device='cpu')
- # === Cleaning Utility ===
- def clean_text(text):
- text = re.sub(r"~~.*?~~", " ", text)
- text = re.sub(r"<[^>]+>", " ", text)
- text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
- text = re.sub(r"\s+", " ", text)
- return text.strip()
- # === Step 2: Update Chunks ===
- def update_chunks(pages_dict):
- all_chunks = []
- if os.path.exists(CHUNKS_FILE):
- try:
- with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
- all_chunks = json.load(f)
- except Exception as e:
- print(f"⚠️ Could not load old chunks: {e}")
- # Supprimer tous les chunks des URLs modifiées
- existing_urls = set(pages_dict.keys())
- all_chunks = [c for c in all_chunks if c.get("url") not in existing_urls]
- new_chunks = []
- chunk_id_start = len(all_chunks)
- for url, text in pages_dict.items():
- cleaned = clean_text(text)
- if not cleaned:
- continue
- chunks = semantic_chunking(cleaned, with_metadata=True)
- for ch in chunks:
- if not isinstance(ch, dict) or not isinstance(ch.get("text"), str):
- print(f"⚠️ Ignored malformed chunk: {ch}")
- continue
- new_chunks.append({
- "chunk_id": chunk_id_start + len(new_chunks),
- "text": ch["text"],
- "url": url,
- "word_count": ch["word_count"],
- "token_count": ch["token_count"]
- })
- all_chunks.extend(new_chunks)
- with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
- json.dump(all_chunks, f, indent=2, ensure_ascii=False)
- print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.")
- return all_chunks
- # === Step 3: Build Index ===
- def build_and_save_index(chunks):
- if not chunks:
- print("⚠️ No chunks to index.")
- return None
- texts = [c["text"] for c in chunks]
- embeddings = model.encode(texts, convert_to_tensor=False, batch_size=4)
- if len(embeddings) == 0:
- print("⚠️ Embeddings generation failed or empty.")
- return None
- dim = len(embeddings[0])
- index = faiss.IndexFlatL2(dim)
- index.add(np.array(embeddings).astype("float32"))
- faiss.write_index(index, INDEX_FILE)
- print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.")
- return index
- # === Full Pipeline ===
- def full_update_pipeline():
- print("🔄 Pipeline complet déclenché...")
- pages = scrape_pages()
- page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p}
- if not page_dict:
- print("⚠️ Aucune page récupérée.")
- return
- chunks = update_chunks(page_dict)
- build_and_save_index(chunks)
- # === Initialize if needed ===
- def initialize_if_needed():
- full_update_pipeline()
|