scraping.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190
  1. from bs4 import BeautifulSoup
  2. from agent_ai.auth import session
  3. from urllib.parse import urljoin, urlparse
  4. import time
  5. import hashlib
  6. import json
  7. import random
  8. from concurrent.futures import ThreadPoolExecutor, as_completed
  9. import os
  10. BASE_URL = "https://wiki.intersel.fr/"
  11. VISITED = set()
  12. USER_AGENTS = [
  13. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
  14. ]
  15. def random_headers():
  16. return {
  17. "User-Agent": random.choice(USER_AGENTS)
  18. }
  19. def hash_text(text):
  20. return hashlib.md5(text.encode("utf-8")).hexdigest()
  21. def fetch_single_page(url, old_hashes, retries=2):
  22. """
  23. Fetch the rendered HTML content inside the div with id 'dokuwiki__content'
  24. instead of the wikitext textarea from the edit page.
  25. """
  26. for attempt in range(retries + 1):
  27. try:
  28. headers = random_headers()
  29. response = session.get(url, headers=headers, timeout=10) # No ?do=edit anymore
  30. response.encoding = 'utf-8'
  31. if response.status_code != 200:
  32. print(f"⚠️ {url} returned {response.status_code}")
  33. continue
  34. soup = BeautifulSoup(response.text, "html.parser")
  35. content_div = soup.find(id="dokuwiki__content")
  36. if content_div:
  37. # Extract the inner HTML as text content (or you could get .get_text() if plain text preferred)
  38. raw_text = content_div.get_text(separator="\n").strip()
  39. current_hash = hash_text(raw_text)
  40. if old_hashes.get(url) != current_hash:
  41. print(f"🆕 Modifiée : {url}")
  42. return url, raw_text, current_hash
  43. else:
  44. print(f"✅ Inchangée : {url}")
  45. return url, None, current_hash
  46. else:
  47. print(f"⚠️ No 'dokuwiki__content' found on attempt {attempt + 1}: {url}")
  48. except Exception as e:
  49. print(f"❌ Error on attempt {attempt + 1} for {url}: {e}")
  50. time.sleep(random.uniform(1, 2))
  51. return url, None, None
  52. def fetch_all_pages_content(urls, old_hashes):
  53. url_texts = {}
  54. new_hashes = {}
  55. with ThreadPoolExecutor(max_workers=5) as executor:
  56. futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls]
  57. for future in as_completed(futures):
  58. url, text, current_hash = future.result()
  59. print(f"🧪 Result from {url} -> {'✓ text found' if text else '⛔ no content'}")
  60. if text:
  61. url_texts[url] = text
  62. new_hashes[url] = current_hash
  63. elif current_hash:
  64. new_hashes[url] = current_hash
  65. skipped = [url for url in urls if url not in url_texts]
  66. print(f"🚫 Skipped or errored URLs ({len(skipped)}):")
  67. for u in skipped:
  68. print(" -", u)
  69. return url_texts, new_hashes
  70. def is_valid_wiki_url(url):
  71. parsed = urlparse(url)
  72. return (
  73. parsed.netloc == "wiki.intersel.fr"
  74. and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
  75. and "#" not in url
  76. )
  77. def crawl_wiki(start_url, existing_urls, max_pages=20):
  78. visited = set(existing_urls)
  79. to_visit = [start_url]
  80. while to_visit and len(visited) < max_pages:
  81. current_url = to_visit.pop(0)
  82. if current_url in visited:
  83. continue
  84. print(f"🔎 Visiting: {current_url}")
  85. visited.add(current_url)
  86. try:
  87. headers = random_headers()
  88. response = session.get(current_url, headers=headers, timeout=10)
  89. if response.status_code != 200:
  90. continue
  91. soup = BeautifulSoup(response.text, "html.parser")
  92. for link in soup.find_all("a", href=True):
  93. absolute_url = urljoin(current_url, link["href"])
  94. if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit:
  95. to_visit.append(absolute_url)
  96. time.sleep(random.uniform(1, 2))
  97. except Exception as e:
  98. print(f"❌ Error with {current_url}: {e}")
  99. return list(visited)
  100. def scrape_pages():
  101. start_url = "https://wiki.intersel.fr/"
  102. urls_path = os.path.join("data", "urls.json")
  103. raw_pages_path = "data/raw_pages.json"
  104. hashes_path = "data/page_hashes.json"
  105. # Charger anciens hashs
  106. try:
  107. with open(hashes_path, "r", encoding="utf-8") as f:
  108. old_hashes = json.load(f)
  109. except FileNotFoundError:
  110. old_hashes = {}
  111. # Charger anciennes URLs
  112. try:
  113. with open(urls_path, "r", encoding="utf-8") as f:
  114. urls = json.load(f)
  115. print("✅ URLs chargées depuis urls.json")
  116. except FileNotFoundError:
  117. print("🚀 Crawling initial du wiki")
  118. urls = []
  119. # Nouveau crawl
  120. crawled_urls = crawl_wiki(start_url, urls, max_pages=20)
  121. all_urls = sorted(set(urls + crawled_urls))
  122. print(f"🌐 Total URLs à vérifier : {len(all_urls)}")
  123. # Scraping des pages
  124. pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes)
  125. # Charger anciens raw_pages
  126. try:
  127. with open(raw_pages_path, "r", encoding="utf-8") as f:
  128. old_raw_pages = {p["url"]: p["text"] for p in json.load(f)}
  129. except FileNotFoundError:
  130. old_raw_pages = {}
  131. # Mettre à jour uniquement les pages modifiées
  132. for url, text in pages.items():
  133. old_raw_pages[url] = text
  134. # Enlever les URLs qui n’existent plus (optionnel)
  135. cleaned_raw_pages = {url: text for url, text in old_raw_pages.items() if url in all_urls}
  136. # Sauvegarder le nouveau contenu
  137. with open(raw_pages_path, "w", encoding="utf-8") as f:
  138. json.dump(
  139. [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()],
  140. f, ensure_ascii=False, indent=2
  141. )
  142. with open(hashes_path, "w", encoding="utf-8") as f:
  143. json.dump(new_hashes, f, ensure_ascii=False, indent=2)
  144. with open(urls_path, "w", encoding="utf-8") as f:
  145. json.dump(all_urls, f, ensure_ascii=False, indent=2)
  146. print("📄 Pages modifiées sauvegardées")
  147. print("📄 Contenu brut mis à jour enregistré dans 'raw_pages.json'")
  148. print("🔗 URLs enregistrées dans 'data/urls.json'")
  149. print("🔐 Hashes des pages enregistrés dans 'page_hashes.json'")
  150. return [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()]