Explorar el Código

version 2 automatic scraping

Nadine Elleuch hace 1 año
padre
commit
8cd2f98d8d
Se han modificado 1 ficheros con 150 adiciones y 0 borrados
  1. 150 0
      processing.py

+ 150 - 0
processing.py

@@ -0,0 +1,150 @@
+import os
+import json
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from agent_ai.chunking import semantic_chunking  
+from agent_ai.scraping import scrape_pages 
+from nltk.tokenize import sent_tokenize
+import re
+import requests
+
+
+CHUNKS_FILE = "data/All_chunks.json"
+INDEX_FILE = "data/faiss_index.index"
+RAW_PAGES_FILE = "data/raw_pages.json"
+URLS_FILE = "data/urls.json"
+
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+def clean_text(text):
+    # Supprimer les balises DokuWiki comme ~~ODT~~ ou ~~TOC~~
+    text = re.sub(r"~~.*?~~", " ", text)
+
+    # Supprimer les balises HTML restantes éventuelles
+    text = re.sub(r"<[^>]+>", " ", text)
+
+    # supprimer les caractères non imprimables sauf accentués
+    text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
+
+    # supprimer les espaces multiples
+    text = re.sub(r"\s+", " ", text)
+
+    return text.strip()
+
+
+def update_raw_pages(new_pages_dict):
+    # Charger les anciennes pages
+    old_pages = {}
+    if os.path.exists(RAW_PAGES_FILE):
+        with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
+            old_pages = json.load(f)
+
+    # Comparer et détecter les modifications
+    updated = {}
+    for url, new_text in new_pages_dict.items():
+        if url not in old_pages or old_pages[url] != new_text:
+            updated[url] = new_text  # Ajouté ou modifié
+
+    # Supprimer les anciens contenus modifiés
+    for url in updated.keys():
+        old_pages.pop(url, None)
+
+    # Ajouter les nouveaux contenus
+    old_pages.update(updated)
+    with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
+        json.dump(old_pages, f, indent=2, ensure_ascii=False)
+
+    # 🔁 Mettre à jour la liste des URLs
+    if os.path.exists(URLS_FILE):
+        with open(URLS_FILE, "r", encoding="utf-8") as f:
+            all_urls = set(json.load(f))
+    else:
+        all_urls = set()
+
+    all_urls.update(new_pages_dict.keys())
+    with open(URLS_FILE, "w", encoding="utf-8") as f:
+        json.dump(list(all_urls), f, indent=2)
+
+    return updated
+
+def update_chunks_for_modified_pages(updated_pages):
+    if os.path.exists(CHUNKS_FILE):
+        with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
+            all_chunks = json.load(f)
+    else:
+        all_chunks = []
+
+    # Remove old chunks for updated URLs
+    updated_urls = set(updated_pages.keys())
+    all_chunks = [c for c in all_chunks if c["url"] not in updated_urls]
+
+    # Add new chunks
+    chunk_index_start = len(all_chunks)
+    new_chunks = []
+    for url, text in updated_pages.items():
+        text = clean_text(text)
+        sentences = sent_tokenize(text)
+        chunks = semantic_chunking(sentences, similarity_threshold=0.6)  
+        for ch in chunks:
+            new_chunks.append({
+                "chunk_id": chunk_index_start + len(new_chunks),
+                "text": ch,
+                "url": url,
+                "word_count": len(ch.split())
+            })
+
+    all_chunks.extend(new_chunks)
+
+    with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
+        json.dump(all_chunks, f, indent=2, ensure_ascii=False)
+
+    return all_chunks
+
+
+def build_and_save_index(chunks):
+    texts = [c["text"] for c in chunks]
+    urls = [c["url"] for c in chunks]
+    embeddings = embedding_model.encode(texts, convert_to_tensor=False)
+    index = faiss.IndexFlatL2(len(embeddings[0]))
+    index.add(np.array(embeddings).astype("float32"))
+
+    # ✅ Save index to disk
+    faiss.write_index(index, INDEX_FILE)
+
+    return index, texts, urls
+
+def full_update_pipeline():
+    print("🔄 Running full update pipeline...")
+
+    pages = scrape_pages() 
+    page_dict = {p["url"]: p["text"] for p in pages}
+    updated = update_raw_pages(page_dict)
+    if updated:
+        all_chunks = update_chunks_for_modified_pages(updated)
+    else:
+        if os.path.exists(CHUNKS_FILE):
+            with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
+                all_chunks = json.load(f)
+        else:
+            print("❌ Aucune donnée pour construire l'index.")
+            return None, None, None
+
+    index, texts, urls = build_and_save_index(all_chunks)
+    print("✅ Index reconstruit (même sans mises à jour).")
+    return index, texts, urls
+
+
+def load_index_from_disk():
+    if not os.path.exists(INDEX_FILE) or not os.path.exists(CHUNKS_FILE):
+        return None, [], []
+
+    index = faiss.read_index(INDEX_FILE)
+    with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
+        chunks = json.load(f)
+    texts = [chunk["text"] for chunk in chunks]
+    urls = [chunk["url"] for chunk in chunks]
+
+    return index, texts, urls
+
+