scraping.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183
  1. from bs4 import BeautifulSoup
  2. from agent_ai.auth import session
  3. from urllib.parse import urljoin, urlparse
  4. import time
  5. import hashlib
  6. import json
  7. import random
  8. from concurrent.futures import ThreadPoolExecutor, as_completed
  9. import os
  10. BASE_URL = "https://wiki.intersel.fr/"
  11. VISITED = set()
  12. USER_AGENTS = [
  13. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
  14. ]
  15. def random_headers():
  16. return {
  17. "User-Agent": random.choice(USER_AGENTS)
  18. }
  19. def hash_text(text):
  20. return hashlib.md5(text.encode("utf-8")).hexdigest()
  21. def fetch_single_page(url, old_hashes):
  22. try:
  23. headers = random_headers()
  24. edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
  25. response = session.get(edit_url, headers=headers, timeout=10)
  26. if response.status_code != 200:
  27. return url, None, None
  28. # 🔥 Forcer l'encodage UTF-8 pour éviter les caractères bizarres
  29. response.encoding = 'utf-8'
  30. if "text/html" not in response.headers.get("Content-Type", ""):
  31. return url, None, None
  32. soup = BeautifulSoup(response.text, "html.parser")
  33. textarea = soup.find("textarea", {"name": "wikitext"})
  34. if textarea:
  35. raw_text = textarea.get_text(separator="\n").strip()
  36. current_hash = hash_text(raw_text)
  37. if old_hashes.get(url) != current_hash:
  38. print(f"🆕 Modifiée : {url}")
  39. return url, raw_text, current_hash
  40. else:
  41. print(f"✅ Inchangée : {url}")
  42. else:
  43. print(f"⚠️ Pas de <textarea> pour {url}")
  44. except Exception as e:
  45. print(f"❌ Erreur pour {url}: {e}")
  46. # 💡 Petite pause aléatoire
  47. time.sleep(random.uniform(0.5, 2))
  48. return url, None, None
  49. def fetch_all_pages_content(urls, old_hashes):
  50. url_texts = {}
  51. new_hashes = {}
  52. with ThreadPoolExecutor(max_workers=5) as executor:
  53. futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls]
  54. for future in as_completed(futures):
  55. url, text, current_hash = future.result()
  56. if text:
  57. url_texts[url] = text
  58. new_hashes[url] = current_hash
  59. elif current_hash:
  60. new_hashes[url] = current_hash
  61. return url_texts, new_hashes
  62. def is_valid_wiki_url(url):
  63. parsed = urlparse(url)
  64. return (
  65. parsed.netloc == "wiki.intersel.fr"
  66. and "do=" not in parsed.query
  67. and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
  68. and "#" not in url
  69. )
  70. def crawl_wiki(start_url, existing_urls, max_pages=500):
  71. visited = set(existing_urls)
  72. to_visit = [start_url]
  73. while to_visit and len(visited) < max_pages:
  74. current_url = to_visit.pop(0)
  75. if current_url in visited:
  76. continue
  77. print(f"🔎 Visiting: {current_url}")
  78. visited.add(current_url)
  79. try:
  80. headers = random_headers()
  81. response = session.get(current_url, headers=headers, timeout=10)
  82. if response.status_code != 200:
  83. continue
  84. soup = BeautifulSoup(response.text, "html.parser")
  85. for link in soup.find_all("a", href=True):
  86. absolute_url = urljoin(current_url, link["href"])
  87. if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit:
  88. to_visit.append(absolute_url)
  89. time.sleep(random.uniform(1, 2))
  90. except Exception as e:
  91. print(f"❌ Error with {current_url}: {e}")
  92. return list(visited)
  93. def scrape_pages():
  94. start_url = "https://wiki.intersel.fr/"
  95. urls_path = os.path.join("data", "urls.json")
  96. raw_pages_path = "data/raw_pages.json"
  97. hashes_path = "data/page_hashes.json"
  98. # Charger anciens hashs
  99. try:
  100. with open(hashes_path, "r", encoding="utf-8") as f:
  101. old_hashes = json.load(f)
  102. except FileNotFoundError:
  103. old_hashes = {}
  104. # Charger anciennes URLs
  105. try:
  106. with open(urls_path, "r", encoding="utf-8") as f:
  107. urls = json.load(f)
  108. print("✅ URLs chargées depuis urls.json")
  109. except FileNotFoundError:
  110. print("🚀 Crawling initial du wiki")
  111. urls = []
  112. # Nouveau crawl
  113. crawled_urls = crawl_wiki(start_url, urls, max_pages=500)
  114. all_urls = sorted(set(urls + crawled_urls))
  115. print(f"🌐 Total URLs à vérifier : {len(all_urls)}")
  116. # Scraping des pages
  117. pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes)
  118. # Charger anciens raw_pages
  119. try:
  120. with open(raw_pages_path, "r", encoding="utf-8") as f:
  121. old_raw_pages = {p["url"]: p["text"] for p in json.load(f)}
  122. except FileNotFoundError:
  123. old_raw_pages = {}
  124. # Mettre à jour uniquement les pages modifiées
  125. for url, text in pages.items():
  126. old_raw_pages[url] = text
  127. # Enlever les URLs qui n’existent plus (optionnel)
  128. cleaned_raw_pages = {url: text for url, text in old_raw_pages.items() if url in all_urls}
  129. # Sauvegarder le nouveau contenu
  130. with open(raw_pages_path, "w", encoding="utf-8") as f:
  131. json.dump(
  132. [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()],
  133. f, ensure_ascii=False, indent=2
  134. )
  135. with open(hashes_path, "w", encoding="utf-8") as f:
  136. json.dump(new_hashes, f, ensure_ascii=False, indent=2)
  137. with open(urls_path, "w", encoding="utf-8") as f:
  138. json.dump(all_urls, f, ensure_ascii=False, indent=2)
  139. print("📄 Pages modifiées sauvegardées")
  140. print("📄 Contenu brut mis à jour enregistré dans 'raw_pages.json'")
  141. print("🔗 URLs enregistrées dans 'data/urls.json'")
  142. print("🔐 Hashes des pages enregistrés dans 'page_hashes.json'")
  143. return [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()]