processing.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. import os
  2. import json
  3. import faiss
  4. import numpy as np
  5. from sentence_transformers import SentenceTransformer
  6. from agent_ai.chunking import semantic_chunking
  7. from agent_ai.scraping import scrape_pages
  8. from nltk.tokenize import sent_tokenize
  9. import re
  10. import requests
  11. CHUNKS_FILE = "data/All_chunks.json"
  12. INDEX_FILE = "data/faiss_index.index"
  13. RAW_PAGES_FILE = "data/raw_pages.json"
  14. URLS_FILE = "data/urls.json"
  15. embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
  16. def clean_text(text):
  17. # Supprimer les balises DokuWiki comme ~~ODT~~ ou ~~TOC~~
  18. text = re.sub(r"~~.*?~~", " ", text)
  19. # Supprimer les balises HTML restantes éventuelles
  20. text = re.sub(r"<[^>]+>", " ", text)
  21. # supprimer les caractères non imprimables sauf accentués
  22. text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
  23. # supprimer les espaces multiples
  24. text = re.sub(r"\s+", " ", text)
  25. return text.strip()
  26. def update_raw_pages(new_pages_dict):
  27. # Charger les anciennes pages
  28. old_pages = {}
  29. if os.path.exists(RAW_PAGES_FILE):
  30. with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
  31. old_pages = json.load(f)
  32. # Comparer et détecter les modifications
  33. updated = {}
  34. for url, new_text in new_pages_dict.items():
  35. if url not in old_pages or old_pages[url] != new_text:
  36. updated[url] = new_text # Ajouté ou modifié
  37. # Supprimer les anciens contenus modifiés
  38. for url in updated.keys():
  39. old_pages.pop(url, None)
  40. # Ajouter les nouveaux contenus
  41. old_pages.update(updated)
  42. with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
  43. json.dump(old_pages, f, indent=2, ensure_ascii=False)
  44. # 🔁 Mettre à jour la liste des URLs
  45. if os.path.exists(URLS_FILE):
  46. with open(URLS_FILE, "r", encoding="utf-8") as f:
  47. all_urls = set(json.load(f))
  48. else:
  49. all_urls = set()
  50. all_urls.update(new_pages_dict.keys())
  51. with open(URLS_FILE, "w", encoding="utf-8") as f:
  52. json.dump(list(all_urls), f, indent=2)
  53. return updated
  54. def update_chunks_for_modified_pages(updated_pages):
  55. if os.path.exists(CHUNKS_FILE):
  56. with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
  57. all_chunks = json.load(f)
  58. else:
  59. all_chunks = []
  60. # Remove old chunks for updated URLs
  61. updated_urls = set(updated_pages.keys())
  62. all_chunks = [c for c in all_chunks if c["url"] not in updated_urls]
  63. # Add new chunks
  64. chunk_index_start = len(all_chunks)
  65. new_chunks = []
  66. for url, text in updated_pages.items():
  67. text = clean_text(text)
  68. sentences = sent_tokenize(text)
  69. chunks = semantic_chunking(sentences, similarity_threshold=0.8)
  70. for ch in chunks:
  71. new_chunks.append({
  72. "chunk_id": chunk_index_start + len(new_chunks),
  73. "text": ch,
  74. "url": url,
  75. "word_count": len(ch.split())
  76. })
  77. all_chunks.extend(new_chunks)
  78. with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
  79. json.dump(all_chunks, f, indent=2, ensure_ascii=False)
  80. return all_chunks
  81. def build_and_save_index(chunks):
  82. texts = [c["text"] for c in chunks]
  83. urls = [c["url"] for c in chunks]
  84. embeddings = embedding_model.encode(texts, convert_to_tensor=False)
  85. index = faiss.IndexFlatL2(len(embeddings[0]))
  86. index.add(np.array(embeddings).astype("float32"))
  87. # ✅ Save index to disk
  88. faiss.write_index(index, INDEX_FILE)
  89. return index, texts, urls
  90. def full_update_pipeline():
  91. print("🔄 Running full update pipeline...")
  92. pages = scrape_pages()
  93. page_dict = {p["url"]: p["text"] for p in pages}
  94. updated = update_raw_pages(page_dict)
  95. if updated:
  96. all_chunks = update_chunks_for_modified_pages(updated)
  97. else:
  98. if os.path.exists(CHUNKS_FILE):
  99. with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
  100. all_chunks = json.load(f)
  101. else:
  102. print("❌ Aucune donnée pour construire l'index.")
  103. return None, None, None
  104. index, texts, urls = build_and_save_index(all_chunks)
  105. print("✅ Index reconstruit (même sans mises à jour).")
  106. return index, texts, urls
  107. def load_index_from_disk():
  108. if not os.path.exists(INDEX_FILE) or not os.path.exists(CHUNKS_FILE):
  109. return None, [], []
  110. index = faiss.read_index(INDEX_FILE)
  111. with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
  112. chunks = json.load(f)
  113. texts = [chunk["text"] for chunk in chunks]
  114. urls = [chunk["url"] for chunk in chunks]
  115. return index, texts, urls