|
|
@@ -0,0 +1,150 @@
|
|
|
+import os
|
|
|
+import json
|
|
|
+import faiss
|
|
|
+import numpy as np
|
|
|
+from sentence_transformers import SentenceTransformer
|
|
|
+from agent_ai.chunking import semantic_chunking
|
|
|
+from agent_ai.scraping import scrape_pages
|
|
|
+from nltk.tokenize import sent_tokenize
|
|
|
+import re
|
|
|
+import requests
|
|
|
+
|
|
|
+
|
|
|
+CHUNKS_FILE = "data/All_chunks.json"
|
|
|
+INDEX_FILE = "data/faiss_index.index"
|
|
|
+RAW_PAGES_FILE = "data/raw_pages.json"
|
|
|
+URLS_FILE = "data/urls.json"
|
|
|
+
|
|
|
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
|
|
|
+
|
|
|
+def clean_text(text):
|
|
|
+ # Supprimer les balises DokuWiki comme ~~ODT~~ ou ~~TOC~~
|
|
|
+ text = re.sub(r"~~.*?~~", " ", text)
|
|
|
+
|
|
|
+ # Supprimer les balises HTML restantes éventuelles
|
|
|
+ text = re.sub(r"<[^>]+>", " ", text)
|
|
|
+
|
|
|
+ # supprimer les caractères non imprimables sauf accentués
|
|
|
+ text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
|
|
|
+
|
|
|
+ # supprimer les espaces multiples
|
|
|
+ text = re.sub(r"\s+", " ", text)
|
|
|
+
|
|
|
+ return text.strip()
|
|
|
+
|
|
|
+
|
|
|
+def update_raw_pages(new_pages_dict):
|
|
|
+ # Charger les anciennes pages
|
|
|
+ old_pages = {}
|
|
|
+ if os.path.exists(RAW_PAGES_FILE):
|
|
|
+ with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
|
|
|
+ old_pages = json.load(f)
|
|
|
+
|
|
|
+ # Comparer et détecter les modifications
|
|
|
+ updated = {}
|
|
|
+ for url, new_text in new_pages_dict.items():
|
|
|
+ if url not in old_pages or old_pages[url] != new_text:
|
|
|
+ updated[url] = new_text # Ajouté ou modifié
|
|
|
+
|
|
|
+ # Supprimer les anciens contenus modifiés
|
|
|
+ for url in updated.keys():
|
|
|
+ old_pages.pop(url, None)
|
|
|
+
|
|
|
+ # Ajouter les nouveaux contenus
|
|
|
+ old_pages.update(updated)
|
|
|
+ with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
|
|
|
+ json.dump(old_pages, f, indent=2, ensure_ascii=False)
|
|
|
+
|
|
|
+ # 🔁 Mettre à jour la liste des URLs
|
|
|
+ if os.path.exists(URLS_FILE):
|
|
|
+ with open(URLS_FILE, "r", encoding="utf-8") as f:
|
|
|
+ all_urls = set(json.load(f))
|
|
|
+ else:
|
|
|
+ all_urls = set()
|
|
|
+
|
|
|
+ all_urls.update(new_pages_dict.keys())
|
|
|
+ with open(URLS_FILE, "w", encoding="utf-8") as f:
|
|
|
+ json.dump(list(all_urls), f, indent=2)
|
|
|
+
|
|
|
+ return updated
|
|
|
+
|
|
|
+def update_chunks_for_modified_pages(updated_pages):
|
|
|
+ if os.path.exists(CHUNKS_FILE):
|
|
|
+ with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
|
|
|
+ all_chunks = json.load(f)
|
|
|
+ else:
|
|
|
+ all_chunks = []
|
|
|
+
|
|
|
+ # Remove old chunks for updated URLs
|
|
|
+ updated_urls = set(updated_pages.keys())
|
|
|
+ all_chunks = [c for c in all_chunks if c["url"] not in updated_urls]
|
|
|
+
|
|
|
+ # Add new chunks
|
|
|
+ chunk_index_start = len(all_chunks)
|
|
|
+ new_chunks = []
|
|
|
+ for url, text in updated_pages.items():
|
|
|
+ text = clean_text(text)
|
|
|
+ sentences = sent_tokenize(text)
|
|
|
+ chunks = semantic_chunking(sentences, similarity_threshold=0.6)
|
|
|
+ for ch in chunks:
|
|
|
+ new_chunks.append({
|
|
|
+ "chunk_id": chunk_index_start + len(new_chunks),
|
|
|
+ "text": ch,
|
|
|
+ "url": url,
|
|
|
+ "word_count": len(ch.split())
|
|
|
+ })
|
|
|
+
|
|
|
+ all_chunks.extend(new_chunks)
|
|
|
+
|
|
|
+ with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
|
|
|
+ json.dump(all_chunks, f, indent=2, ensure_ascii=False)
|
|
|
+
|
|
|
+ return all_chunks
|
|
|
+
|
|
|
+
|
|
|
+def build_and_save_index(chunks):
|
|
|
+ texts = [c["text"] for c in chunks]
|
|
|
+ urls = [c["url"] for c in chunks]
|
|
|
+ embeddings = embedding_model.encode(texts, convert_to_tensor=False)
|
|
|
+ index = faiss.IndexFlatL2(len(embeddings[0]))
|
|
|
+ index.add(np.array(embeddings).astype("float32"))
|
|
|
+
|
|
|
+ # ✅ Save index to disk
|
|
|
+ faiss.write_index(index, INDEX_FILE)
|
|
|
+
|
|
|
+ return index, texts, urls
|
|
|
+
|
|
|
+def full_update_pipeline():
|
|
|
+ print("🔄 Running full update pipeline...")
|
|
|
+
|
|
|
+ pages = scrape_pages()
|
|
|
+ page_dict = {p["url"]: p["text"] for p in pages}
|
|
|
+ updated = update_raw_pages(page_dict)
|
|
|
+ if updated:
|
|
|
+ all_chunks = update_chunks_for_modified_pages(updated)
|
|
|
+ else:
|
|
|
+ if os.path.exists(CHUNKS_FILE):
|
|
|
+ with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
|
|
|
+ all_chunks = json.load(f)
|
|
|
+ else:
|
|
|
+ print("❌ Aucune donnée pour construire l'index.")
|
|
|
+ return None, None, None
|
|
|
+
|
|
|
+ index, texts, urls = build_and_save_index(all_chunks)
|
|
|
+ print("✅ Index reconstruit (même sans mises à jour).")
|
|
|
+ return index, texts, urls
|
|
|
+
|
|
|
+
|
|
|
+def load_index_from_disk():
|
|
|
+ if not os.path.exists(INDEX_FILE) or not os.path.exists(CHUNKS_FILE):
|
|
|
+ return None, [], []
|
|
|
+
|
|
|
+ index = faiss.read_index(INDEX_FILE)
|
|
|
+ with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
|
|
|
+ chunks = json.load(f)
|
|
|
+ texts = [chunk["text"] for chunk in chunks]
|
|
|
+ urls = [chunk["url"] for chunk in chunks]
|
|
|
+
|
|
|
+ return index, texts, urls
|
|
|
+
|
|
|
+
|