| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150 |
- import os
- import json
- import faiss
- import numpy as np
- from sentence_transformers import SentenceTransformer
- from agent_ai.chunking import semantic_chunking
- from agent_ai.scraping import scrape_pages
- from nltk.tokenize import sent_tokenize
- import re
- import requests
- CHUNKS_FILE = "data/All_chunks.json"
- INDEX_FILE = "data/faiss_index.index"
- RAW_PAGES_FILE = "data/raw_pages.json"
- URLS_FILE = "data/urls.json"
- embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
- def clean_text(text):
- # Supprimer les balises DokuWiki comme ~~ODT~~ ou ~~TOC~~
- text = re.sub(r"~~.*?~~", " ", text)
- # Supprimer les balises HTML restantes éventuelles
- text = re.sub(r"<[^>]+>", " ", text)
- # supprimer les caractères non imprimables sauf accentués
- text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
- # supprimer les espaces multiples
- text = re.sub(r"\s+", " ", text)
- return text.strip()
- def update_raw_pages(new_pages_dict):
- # Charger les anciennes pages
- old_pages = {}
- if os.path.exists(RAW_PAGES_FILE):
- with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
- old_pages = json.load(f)
- # Comparer et détecter les modifications
- updated = {}
- for url, new_text in new_pages_dict.items():
- if url not in old_pages or old_pages[url] != new_text:
- updated[url] = new_text # Ajouté ou modifié
- # Supprimer les anciens contenus modifiés
- for url in updated.keys():
- old_pages.pop(url, None)
- # Ajouter les nouveaux contenus
- old_pages.update(updated)
- with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
- json.dump(old_pages, f, indent=2, ensure_ascii=False)
- # 🔁 Mettre à jour la liste des URLs
- if os.path.exists(URLS_FILE):
- with open(URLS_FILE, "r", encoding="utf-8") as f:
- all_urls = set(json.load(f))
- else:
- all_urls = set()
- all_urls.update(new_pages_dict.keys())
- with open(URLS_FILE, "w", encoding="utf-8") as f:
- json.dump(list(all_urls), f, indent=2)
- return updated
- def update_chunks_for_modified_pages(updated_pages):
- if os.path.exists(CHUNKS_FILE):
- with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
- all_chunks = json.load(f)
- else:
- all_chunks = []
- # Remove old chunks for updated URLs
- updated_urls = set(updated_pages.keys())
- all_chunks = [c for c in all_chunks if c["url"] not in updated_urls]
- # Add new chunks
- chunk_index_start = len(all_chunks)
- new_chunks = []
- for url, text in updated_pages.items():
- text = clean_text(text)
- sentences = sent_tokenize(text)
- chunks = semantic_chunking(sentences, similarity_threshold=0.6)
- for ch in chunks:
- new_chunks.append({
- "chunk_id": chunk_index_start + len(new_chunks),
- "text": ch,
- "url": url,
- "word_count": len(ch.split())
- })
- all_chunks.extend(new_chunks)
- with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
- json.dump(all_chunks, f, indent=2, ensure_ascii=False)
- return all_chunks
- def build_and_save_index(chunks):
- texts = [c["text"] for c in chunks]
- urls = [c["url"] for c in chunks]
- embeddings = embedding_model.encode(texts, convert_to_tensor=False)
- index = faiss.IndexFlatL2(len(embeddings[0]))
- index.add(np.array(embeddings).astype("float32"))
- # ✅ Save index to disk
- faiss.write_index(index, INDEX_FILE)
- return index, texts, urls
- def full_update_pipeline():
- print("🔄 Running full update pipeline...")
- pages = scrape_pages()
- page_dict = {p["url"]: p["text"] for p in pages}
- updated = update_raw_pages(page_dict)
- if updated:
- all_chunks = update_chunks_for_modified_pages(updated)
- else:
- if os.path.exists(CHUNKS_FILE):
- with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
- all_chunks = json.load(f)
- else:
- print("❌ Aucune donnée pour construire l'index.")
- return None, None, None
- index, texts, urls = build_and_save_index(all_chunks)
- print("✅ Index reconstruit (même sans mises à jour).")
- return index, texts, urls
- def load_index_from_disk():
- if not os.path.exists(INDEX_FILE) or not os.path.exists(CHUNKS_FILE):
- return None, [], []
- index = faiss.read_index(INDEX_FILE)
- with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
- chunks = json.load(f)
- texts = [chunk["text"] for chunk in chunks]
- urls = [chunk["url"] for chunk in chunks]
- return index, texts, urls
|