processing.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. import os
  2. import time
  3. import json
  4. import re
  5. import faiss
  6. import numpy as np
  7. from sentence_transformers import SentenceTransformer
  8. from agent_ai.chunking import split_into_chunks, count_tokens
  9. from agent_ai.scraping import scrape_pages
  10. # === Paths ===
  11. os.makedirs("data", exist_ok=True)
  12. CHUNKS_FILE = "data/All_chunks.json"
  13. INDEX_FILE = "data/faiss_index.index"
  14. RAW_PAGES_FILE = "data/raw_pages.json"
  15. URLS_FILE = "data/urls.json"
  16. # === Embedding Model ===
  17. model = SentenceTransformer('nomic-ai/nomic-embed-text-v1', trust_remote_code=True)
  18. # === Cleaning Utility ===
  19. def clean_text(text):
  20. text = re.sub(r"~~.*?~~", " ", text)
  21. text = re.sub(r"<[^>]+>", " ", text)
  22. text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
  23. text = re.sub(r"\s+", " ", text)
  24. return text.strip()
  25. # === Step 1: Update Pages ===
  26. def update_raw_pages(new_pages_dict):
  27. old_pages = {}
  28. try:
  29. with open(RAW_PAGES_FILE, "r", encoding="utf-8") as f:
  30. old_pages = {p["url"]: p["text"] for p in json.load(f)}
  31. except Exception as e:
  32. print(f"⚠️ Could not load old raw pages: {e}")
  33. updated = {url: text for url, text in new_pages_dict.items()
  34. if url not in old_pages or old_pages[url] != text}
  35. # No need to pop, just update with new/modified pages
  36. old_pages.update(updated)
  37. with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
  38. json.dump([{"url": url, "text": text} for url, text in old_pages.items()], f, indent=2, ensure_ascii=False)
  39. # Update URL list
  40. all_urls = set(old_pages.keys())
  41. with open(URLS_FILE, "w", encoding="utf-8") as f:
  42. json.dump(list(all_urls), f, indent=2)
  43. print(f"📄 Raw pages updated: {len(updated)} pages modified.")
  44. return updated
  45. # === Step 2: Update Chunks ===
  46. def update_chunks_for_modified_pages(updated_pages):
  47. # Load old chunks
  48. all_chunks = []
  49. if os.path.exists(CHUNKS_FILE):
  50. try:
  51. with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
  52. all_chunks = json.load(f)
  53. except Exception as e:
  54. print(f"⚠️ Could not load old chunks: {e}")
  55. all_chunks = []
  56. # Remove outdated chunks for updated URLs
  57. updated_urls = set(updated_pages.keys())
  58. all_chunks = [c for c in all_chunks if c.get("url") not in updated_urls]
  59. new_chunks = []
  60. chunk_id_start = len(all_chunks)
  61. for url, text in updated_pages.items():
  62. cleaned = clean_text(text)
  63. if not cleaned:
  64. continue
  65. chunks = split_into_chunks(cleaned, with_metadata=True)
  66. for ch in chunks:
  67. # safety check: ch must be dict with string 'text'
  68. if not isinstance(ch, dict) or not isinstance(ch.get("text"), str):
  69. print(f"⚠️ Ignored malformed chunk: {ch}")
  70. continue
  71. new_chunks.append({
  72. "chunk_id": chunk_id_start + len(new_chunks),
  73. "text": ch["text"],
  74. "url": url,
  75. "word_count": ch["word_count"],
  76. "token_count": ch["token_count"]
  77. })
  78. all_chunks.extend(new_chunks)
  79. with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
  80. json.dump(all_chunks, f, indent=2, ensure_ascii=False)
  81. print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.")
  82. return all_chunks
  83. # === Step 3: Build Index ===
  84. def build_and_save_index(chunks):
  85. if not chunks:
  86. print("⚠️ No chunks to index.")
  87. return None
  88. texts = [c["text"] for c in chunks]
  89. embeddings = model.encode(texts, convert_to_tensor=False)
  90. if len(embeddings) == 0:
  91. print("⚠️ Embeddings generation failed or empty.")
  92. return None
  93. dim = len(embeddings[0])
  94. index = faiss.IndexFlatL2(dim)
  95. index.add(np.array(embeddings).astype("float32"))
  96. faiss.write_index(index, INDEX_FILE)
  97. print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.")
  98. return index
  99. # === Full Pipeline ===
  100. def full_update_pipeline():
  101. print("🔄 Pipeline complet déclenché...")
  102. pages = scrape_pages()
  103. page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p}
  104. if not page_dict:
  105. print("⚠️ Aucune page récupérée.")
  106. return
  107. index_exists = os.path.exists(INDEX_FILE)
  108. # First run: build everything from scratch
  109. if not index_exists:
  110. print("🚀 Première exécution : génération complète des chunks et de l'index.")
  111. with open(RAW_PAGES_FILE, "w", encoding="utf-8") as f:
  112. json.dump([{"url": url, "text": text} for url, text in page_dict.items()], f, indent=2, ensure_ascii=False)
  113. with open(URLS_FILE, "w", encoding="utf-8") as f:
  114. json.dump(list(page_dict.keys()), f, indent=2)
  115. chunks = update_chunks_for_modified_pages(page_dict)
  116. build_and_save_index(chunks)
  117. else:
  118. updated = update_raw_pages(page_dict)
  119. if updated:
  120. chunks = update_chunks_for_modified_pages(updated)
  121. build_and_save_index(chunks)
  122. else:
  123. print("✅ Aucun changement détecté. Pas d'update nécessaire.")
  124. # === Init (if needed) ===
  125. def initialize_if_needed():
  126. if not os.path.exists(URLS_FILE) or not os.path.exists(INDEX_FILE):
  127. print("🚨 Initializing system...")
  128. full_update_pipeline()
  129. # === Auto-refresh every 30 mins ===
  130. def start_auto_refresh(interval_minutes=30):
  131. initialize_if_needed()
  132. while True:
  133. print(f"\n⏰ Checking for updates every {interval_minutes} minutes...")
  134. full_update_pipeline()
  135. time.sleep(interval_minutes * 60)