from bs4 import BeautifulSoup from agent_ai.auth import session from urllib.parse import urljoin, urlparse import time import hashlib import json import random from concurrent.futures import ThreadPoolExecutor, as_completed import os BASE_URL = "https://wiki.intersel.fr/" VISITED = set() USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36" ] def random_headers(): return { "User-Agent": random.choice(USER_AGENTS) } def hash_text(text): return hashlib.md5(text.encode("utf-8")).hexdigest() def fetch_single_page(url, old_hashes): try: headers = random_headers() edit_url = url + "&do=edit" if "?" in url else url + "?do=edit" response = session.get(edit_url, headers=headers, timeout=10) if response.status_code != 200: return url, None, None # 🔥 Forcer l'encodage UTF-8 pour éviter les caractères bizarres response.encoding = 'utf-8' if "text/html" not in response.headers.get("Content-Type", ""): return url, None, None soup = BeautifulSoup(response.text, "html.parser") textarea = soup.find("textarea", {"name": "wikitext"}) if textarea: raw_text = textarea.get_text(separator="\n").strip() current_hash = hash_text(raw_text) if old_hashes.get(url) != current_hash: print(f"🆕 Modifiée : {url}") return url, raw_text, current_hash else: print(f"✅ Inchangée : {url}") else: print(f"⚠️ Pas de