from bs4 import BeautifulSoup from agent_ai.auth import session from urllib.parse import urljoin, urlparse import time import hashlib import json import random from concurrent.futures import ThreadPoolExecutor, as_completed import os BASE_URL = "https://wiki.intersel.fr/" VISITED = set() USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" ] def random_headers(): return { "User-Agent": random.choice(USER_AGENTS) } def hash_text(text): return hashlib.md5(text.encode("utf-8")).hexdigest() def fetch_single_page(url, old_hashes, retries=2): """ Fetch the rendered HTML content inside the div with id 'dokuwiki__content' instead of the wikitext textarea from the edit page. """ for attempt in range(retries + 1): try: headers = random_headers() response = session.get(url, headers=headers, timeout=10) # No ?do=edit anymore response.encoding = 'utf-8' if response.status_code != 200: print(f"⚠️ {url} returned {response.status_code}") continue soup = BeautifulSoup(response.text, "html.parser") content_div = soup.find(id="dokuwiki__content") if content_div: # Extract the inner HTML as text content (or you could get .get_text() if plain text preferred) raw_text = content_div.get_text(separator="\n").strip() current_hash = hash_text(raw_text) if old_hashes.get(url) != current_hash: print(f"🆕 Modifiée : {url}") return url, raw_text, current_hash else: print(f"✅ Inchangée : {url}") return url, None, current_hash else: print(f"⚠️ No 'dokuwiki__content' found on attempt {attempt + 1}: {url}") except Exception as e: print(f"❌ Error on attempt {attempt + 1} for {url}: {e}") time.sleep(random.uniform(1, 2)) return url, None, None def fetch_all_pages_content(urls, old_hashes): url_texts = {} new_hashes = {} with ThreadPoolExecutor(max_workers=5) as executor: futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls] for future in as_completed(futures): url, text, current_hash = future.result() print(f"🧪 Result from {url} -> {'✓ text found' if text else '⛔ no content'}") if text: url_texts[url] = text new_hashes[url] = current_hash elif current_hash: new_hashes[url] = current_hash skipped = [url for url in urls if url not in url_texts] print(f"🚫 Skipped or errored URLs ({len(skipped)}):") for u in skipped: print(" -", u) return url_texts, new_hashes def is_valid_wiki_url(url): parsed = urlparse(url) return ( parsed.netloc == "wiki.intersel.fr" and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx")) and "#" not in url ) def crawl_wiki(start_url, existing_urls, max_pages=20): visited = set(existing_urls) to_visit = [start_url] while to_visit and len(visited) < max_pages: current_url = to_visit.pop(0) if current_url in visited: continue print(f"🔎 Visiting: {current_url}") visited.add(current_url) try: headers = random_headers() response = session.get(current_url, headers=headers, timeout=10) if response.status_code != 200: continue soup = BeautifulSoup(response.text, "html.parser") for link in soup.find_all("a", href=True): absolute_url = urljoin(current_url, link["href"]) if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit: to_visit.append(absolute_url) time.sleep(random.uniform(1, 2)) except Exception as e: print(f"❌ Error with {current_url}: {e}") return list(visited) def scrape_pages(): start_url = "https://wiki.intersel.fr/" urls_path = os.path.join("data", "urls.json") raw_pages_path = "data/raw_pages.json" hashes_path = "data/page_hashes.json" # Charger anciens hashs try: with open(hashes_path, "r", encoding="utf-8") as f: old_hashes = json.load(f) except FileNotFoundError: old_hashes = {} # Charger anciennes URLs try: with open(urls_path, "r", encoding="utf-8") as f: urls = json.load(f) print("✅ URLs chargées depuis urls.json") except FileNotFoundError: print("🚀 Crawling initial du wiki") urls = [] # Nouveau crawl crawled_urls = crawl_wiki(start_url, urls, max_pages=20) all_urls = sorted(set(urls + crawled_urls)) print(f"🌐 Total URLs à vérifier : {len(all_urls)}") # Scraping des pages pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes) # Charger anciens raw_pages try: with open(raw_pages_path, "r", encoding="utf-8") as f: old_raw_pages = {p["url"]: p["text"] for p in json.load(f)} except FileNotFoundError: old_raw_pages = {} # Mettre à jour uniquement les pages modifiées for url, text in pages.items(): old_raw_pages[url] = text # Enlever les URLs qui n’existent plus (optionnel) cleaned_raw_pages = {url: text for url, text in old_raw_pages.items() if url in all_urls} # Sauvegarder le nouveau contenu with open(raw_pages_path, "w", encoding="utf-8") as f: json.dump( [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()], f, ensure_ascii=False, indent=2 ) with open(hashes_path, "w", encoding="utf-8") as f: json.dump(new_hashes, f, ensure_ascii=False, indent=2) with open(urls_path, "w", encoding="utf-8") as f: json.dump(all_urls, f, ensure_ascii=False, indent=2) print("📄 Pages modifiées sauvegardées") print("📄 Contenu brut mis à jour enregistré dans 'raw_pages.json'") print("🔗 URLs enregistrées dans 'data/urls.json'") print("🔐 Hashes des pages enregistrés dans 'page_hashes.json'") return [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()]