scraping.py 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. from bs4 import BeautifulSoup
  2. from agent_ai.auth import session
  3. from urllib.parse import urljoin, urlparse
  4. import time
  5. import hashlib
  6. import json
  7. import random
  8. from concurrent.futures import ThreadPoolExecutor, as_completed
  9. import os
  10. BASE_URL = "https://wiki.intersel.fr/"
  11. VISITED = set()
  12. USER_AGENTS = [
  13. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
  14. ]
  15. def random_headers():
  16. return {
  17. "User-Agent": random.choice(USER_AGENTS)
  18. }
  19. def hash_text(text):
  20. return hashlib.md5(text.encode("utf-8")).hexdigest()
  21. def fetch_single_page(url, old_hashes):
  22. try:
  23. headers = random_headers()
  24. edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
  25. response = session.get(edit_url, headers=headers, timeout=10)
  26. if response.status_code != 200:
  27. return url, None, None
  28. # 🔥 Forcer l'encodage UTF-8 pour éviter les caractères bizarres
  29. response.encoding = 'utf-8'
  30. if "text/html" not in response.headers.get("Content-Type", ""):
  31. return url, None, None
  32. soup = BeautifulSoup(response.text, "html.parser")
  33. textarea = soup.find("textarea", {"name": "wikitext"})
  34. if textarea:
  35. raw_text = textarea.get_text(separator="\n").strip()
  36. current_hash = hash_text(raw_text)
  37. if old_hashes.get(url) != current_hash:
  38. print(f"🆕 Modifiée : {url}")
  39. return url, raw_text, current_hash
  40. else:
  41. print(f"✅ Inchangée : {url}")
  42. else:
  43. print(f"⚠️ Pas de <textarea> pour {url}")
  44. except Exception as e:
  45. print(f"❌ Erreur pour {url}: {e}")
  46. # 💡 Petite pause aléatoire
  47. time.sleep(random.uniform(0.5, 2))
  48. return url, None, None
  49. def fetch_single_page(url, old_hashes):
  50. try:
  51. headers = random_headers()
  52. edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
  53. response = session.get(edit_url, headers=headers, timeout=10)
  54. if response.status_code != 200:
  55. return url, None, None
  56. response.encoding = 'utf-8'
  57. if "text/html" not in response.headers.get("Content-Type", ""):
  58. return url, None, None
  59. soup = BeautifulSoup(response.text, "html.parser")
  60. textarea = soup.find("textarea", {"name": "wikitext"})
  61. if textarea:
  62. raw_text = textarea.get_text(separator="\n").strip()
  63. current_hash = hash_text(raw_text)
  64. if old_hashes.get(url) != current_hash:
  65. print(f"🆕 Modifiée : {url}")
  66. return url, raw_text, current_hash
  67. else:
  68. print(f"✅ Inchangée : {url}")
  69. else:
  70. print(f"⚠️ Pas de <textarea> pour {url}")
  71. except Exception as e:
  72. print(f"❌ Erreur pour {url}: {e}")
  73. time.sleep(random.uniform(0.5, 2))
  74. return url, None, None
  75. def fetch_all_pages_content(urls, old_hashes):
  76. url_texts = {}
  77. new_hashes = {}
  78. with ThreadPoolExecutor(max_workers=5) as executor:
  79. futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls]
  80. for future in as_completed(futures):
  81. url, text, current_hash = future.result()
  82. if text:
  83. url_texts[url] = text
  84. new_hashes[url] = current_hash
  85. elif current_hash:
  86. new_hashes[url] = current_hash
  87. return url_texts, new_hashes
  88. def is_valid_wiki_url(url):
  89. parsed = urlparse(url)
  90. return (
  91. parsed.netloc == "wiki.intersel.fr"
  92. and "do=" not in parsed.query
  93. and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
  94. and "#" not in url
  95. )
  96. def crawl_wiki(start_url, existing_urls, max_pages=500):
  97. visited = set(existing_urls)
  98. to_visit = [start_url]
  99. while to_visit and len(visited) < max_pages:
  100. current_url = to_visit.pop(0)
  101. if current_url in visited:
  102. continue
  103. print(f"🔎 Visiting: {current_url}")
  104. visited.add(current_url)
  105. try:
  106. headers = random_headers()
  107. response = session.get(current_url, headers=headers, timeout=10)
  108. if response.status_code != 200:
  109. continue
  110. soup = BeautifulSoup(response.text, "html.parser")
  111. for link in soup.find_all("a", href=True):
  112. absolute_url = urljoin(current_url, link["href"])
  113. if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit:
  114. to_visit.append(absolute_url)
  115. time.sleep(random.uniform(1, 2))
  116. except Exception as e:
  117. print(f"❌ Error with {current_url}: {e}")
  118. return list(visited)
  119. def scrape_pages():
  120. start_url = "https://wiki.intersel.fr/"
  121. urls_path = os.path.join("data", "urls.json")
  122. try:
  123. with open("page_hashes.json", "r", encoding="utf-8") as f:
  124. old_hashes = json.load(f)
  125. except FileNotFoundError:
  126. old_hashes = {}
  127. try:
  128. with open(urls_path, "r", encoding="utf-8") as f:
  129. urls = json.load(f)
  130. print("✅ URLs chargées depuis urls.json")
  131. except FileNotFoundError:
  132. print("🚀 Crawling initial du wiki")
  133. urls = []
  134. # Lancer un nouveau crawl en partant de l’URL de départ, tout en conservant les anciennes URLs
  135. crawled_urls = crawl_wiki(start_url, urls, max_pages=500)
  136. all_urls = sorted(set(urls + crawled_urls))
  137. print(f"🌐 Total URLs à vérifier : {len(all_urls)}")
  138. pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes)
  139. raw_pages = [{"url": url, "text": text} for url, text in pages.items()]
  140. with open("raw_pages.json", "w", encoding="utf-8") as f:
  141. json.dump(raw_pages, f, ensure_ascii=False, indent=2)
  142. with open("page_hashes.json", "w", encoding="utf-8") as f:
  143. json.dump(new_hashes, f, ensure_ascii=False, indent=2)
  144. # Enregistrer toutes les URLs dans data/urls.json
  145. with open(urls_path, "w", encoding="utf-8") as f:
  146. json.dump(all_urls, f, ensure_ascii=False, indent=2)
  147. print("📄 Pages modifiées sauvegardées")
  148. print("📄 Contenu brut mis à jour enregistré dans 'raw_pages.json'")
  149. print("🔗 URLs enregistrées dans 'data/urls.json'")
  150. print("🔐 Hashes des pages enregistrés dans 'page_hashes.json'")
  151. return raw_pages