processing.py 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. import os
  2. import json
  3. import re
  4. import faiss
  5. import numpy as np
  6. from sentence_transformers import SentenceTransformer
  7. from src.agent_ai.chunking import semantic_chunking
  8. from src.agent_ai.scraping import scrape_pages
  9. # === Paths ===
  10. os.makedirs("data", exist_ok=True)
  11. CHUNKS_FILE = "data/All_chunks.json"
  12. INDEX_FILE = "data/faiss_index.index"
  13. RAW_PAGES_FILE = "data/raw_pages.json"
  14. # === Embedding Model ===
  15. '''model = SentenceTransformer(
  16. 'nomic-ai/nomic-embed-text-v1',
  17. device='cuda',
  18. trust_remote_code=True
  19. )'''
  20. model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', device='cpu')
  21. # === Cleaning Utility ===
  22. def clean_text(text):
  23. text = re.sub(r"~~.*?~~", " ", text)
  24. text = re.sub(r"<[^>]+>", " ", text)
  25. text = re.sub(r"[^\x00-\x7FÀ-ÿ]+", " ", text)
  26. text = re.sub(r"\s+", " ", text)
  27. return text.strip()
  28. # === Step 2: Update Chunks ===
  29. def update_chunks(pages_dict):
  30. all_chunks = []
  31. if os.path.exists(CHUNKS_FILE):
  32. try:
  33. with open(CHUNKS_FILE, "r", encoding="utf-8") as f:
  34. all_chunks = json.load(f)
  35. except Exception as e:
  36. print(f"⚠️ Could not load old chunks: {e}")
  37. # Supprimer tous les chunks des URLs modifiées
  38. existing_urls = set(pages_dict.keys())
  39. all_chunks = [c for c in all_chunks if c.get("url") not in existing_urls]
  40. new_chunks = []
  41. chunk_id_start = len(all_chunks)
  42. for url, text in pages_dict.items():
  43. cleaned = clean_text(text)
  44. if not cleaned:
  45. continue
  46. chunks = semantic_chunking(cleaned, with_metadata=True)
  47. for ch in chunks:
  48. if not isinstance(ch, dict) or not isinstance(ch.get("text"), str):
  49. print(f"⚠️ Ignored malformed chunk: {ch}")
  50. continue
  51. new_chunks.append({
  52. "chunk_id": chunk_id_start + len(new_chunks),
  53. "text": ch["text"],
  54. "url": url,
  55. "word_count": ch["word_count"],
  56. "token_count": ch["token_count"]
  57. })
  58. all_chunks.extend(new_chunks)
  59. with open(CHUNKS_FILE, "w", encoding="utf-8") as f:
  60. json.dump(all_chunks, f, indent=2, ensure_ascii=False)
  61. print(f"📄 Chunks updated: {len(new_chunks)} new chunks added, total chunks: {len(all_chunks)}.")
  62. return all_chunks
  63. # === Step 3: Build Index ===
  64. def build_and_save_index(chunks):
  65. if not chunks:
  66. print("⚠️ No chunks to index.")
  67. return None
  68. texts = [c["text"] for c in chunks]
  69. embeddings = model.encode(texts, convert_to_tensor=False, batch_size=4)
  70. if len(embeddings) == 0:
  71. print("⚠️ Embeddings generation failed or empty.")
  72. return None
  73. dim = len(embeddings[0])
  74. index = faiss.IndexFlatL2(dim)
  75. index.add(np.array(embeddings).astype("float32"))
  76. faiss.write_index(index, INDEX_FILE)
  77. print(f"✅ FAISS index saved at '{INDEX_FILE}' with {len(texts)} vectors.")
  78. return index
  79. # === Full Pipeline ===
  80. def full_update_pipeline():
  81. print("🔄 Pipeline complet déclenché...")
  82. pages = scrape_pages()
  83. page_dict = {p["url"]: p["text"] for p in pages if "url" in p and "text" in p}
  84. if not page_dict:
  85. print("⚠️ Aucune page récupérée.")
  86. return
  87. chunks = update_chunks(page_dict)
  88. build_and_save_index(chunks)
  89. # === Initialize if needed ===
  90. def initialize_if_needed():
  91. full_update_pipeline()