|
@@ -1,197 +1,485 @@
|
|
|
-from bs4 import BeautifulSoup
|
|
|
|
|
-from src.agent_ai.auth import session
|
|
|
|
|
-from urllib.parse import urljoin, urlparse
|
|
|
|
|
-import time
|
|
|
|
|
-import hashlib
|
|
|
|
|
|
|
+"""
|
|
|
|
|
+Scraping du wiki Intersel via l'API JSON-RPC officielle DokuWiki.
|
|
|
|
|
+
|
|
|
|
|
+Utilise core.listPages (toutes les pages) et core.getPage (contenu wikitext).
|
|
|
|
|
+Authentification : core.login (WIKI_USERNAME, WIKI_PASSWORD).
|
|
|
|
|
+
|
|
|
|
|
+Prérequis DokuWiki : l'option "remote" doit être activée dans la configuration.
|
|
|
|
|
+Voir https://www.dokuwiki.org/config:remote
|
|
|
|
|
+
|
|
|
|
|
+Fallback : si l'API n'est pas disponible, utilise le scraping HTML classique.
|
|
|
|
|
+"""
|
|
|
|
|
+
|
|
|
|
|
+import os
|
|
|
import json
|
|
import json
|
|
|
|
|
+import hashlib
|
|
|
|
|
+import time
|
|
|
import random
|
|
import random
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
-import os
|
|
|
|
|
|
|
+from urllib.parse import urljoin
|
|
|
|
|
+
|
|
|
|
|
+import requests
|
|
|
|
|
|
|
|
-base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
|
|
|
|
-data_dir = os.path.join(base_dir, 'data')
|
|
|
|
|
|
|
+from config import DEBUG
|
|
|
|
|
+
|
|
|
|
|
+base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
|
+data_dir = os.path.join(base_dir, "data")
|
|
|
|
|
|
|
|
BASE_URL = "https://wiki.intersel.fr/"
|
|
BASE_URL = "https://wiki.intersel.fr/"
|
|
|
-VISITED = set()
|
|
|
|
|
|
|
+JSONRPC_URL = "https://wiki.intersel.fr/lib/exe/jsonrpc.php"
|
|
|
|
|
|
|
|
-# Nombre maximum d'URLs à scraper (crawl + fetch). Voir Documents/Choix_techniques.md.
|
|
|
|
|
-MAX_URLS_TO_SCRAPE = 15
|
|
|
|
|
|
|
+# Max URLs : env MAX_URLS (ex: 500), ou 15 si DEBUG, sinon pas de limite
|
|
|
|
|
+_max = os.getenv("MAX_URLS")
|
|
|
|
|
+MAX_URLS = int(_max) if _max and str(_max).isdigit() else (15 if DEBUG else None)
|
|
|
|
|
|
|
|
USER_AGENTS = [
|
|
USER_AGENTS = [
|
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
-def random_headers():
|
|
|
|
|
- return {
|
|
|
|
|
- "User-Agent": random.choice(USER_AGENTS)
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
-def hash_text(text):
|
|
|
|
|
- return hashlib.md5(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
|
|
|
|
-def fetch_single_page(url, old_hashes):
|
|
|
|
|
|
|
+def _jsonrpc_call(method: str, params: dict | None = None, session: requests.Session | None = None) -> dict:
|
|
|
|
|
+ """
|
|
|
|
|
+ Appel JSON-RPC DokuWiki (format simplifié : POST /method avec params en body).
|
|
|
|
|
+ Retourne {"result": ...} ou {"error": {"code": ..., "message": ...}}.
|
|
|
|
|
+ """
|
|
|
|
|
+ sess = session or requests.Session()
|
|
|
|
|
+ url = f"{JSONRPC_URL}/{method}"
|
|
|
|
|
+ headers = {
|
|
|
|
|
+ "Content-Type": "application/json",
|
|
|
|
|
+ "User-Agent": random.choice(USER_AGENTS),
|
|
|
|
|
+ }
|
|
|
|
|
+ body = params if params is not None else {}
|
|
|
|
|
+ try:
|
|
|
|
|
+ resp = sess.post(url, json=body, headers=headers, timeout=15)
|
|
|
|
|
+ resp.raise_for_status()
|
|
|
|
|
+ data = resp.json()
|
|
|
|
|
+ return data
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ return {"error": {"code": -1, "message": str(e)}}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _page_id_to_url(page_id: str) -> str:
|
|
|
|
|
+ """Convertit un page_id DokuWiki en URL complète."""
|
|
|
|
|
+ if not page_id or page_id == "start":
|
|
|
|
|
+ return BASE_URL.rstrip("/")
|
|
|
|
|
+ return f"{BASE_URL}doku.php?id={page_id}"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _login_jsonrpc(session: requests.Session) -> bool:
|
|
|
|
|
+ """Authentification via core.login (déconseillé mais simple). Utilise les cookies pour les appels suivants."""
|
|
|
|
|
+ user = os.getenv("WIKI_USERNAME")
|
|
|
|
|
+ password = os.getenv("WIKI_PASSWORD")
|
|
|
|
|
+ if not user or not password:
|
|
|
|
|
+ return False
|
|
|
|
|
+ r = _jsonrpc_call("core.login", {"user": user, "pass": password}, session)
|
|
|
|
|
+ if "error" in r:
|
|
|
|
|
+ error_code = r["error"].get("code", -1)
|
|
|
|
|
+ if error_code != 0: # 0 = succès, autre = erreur
|
|
|
|
|
+ print(f"❌ JSON-RPC login failed: {r['error'].get('message', r)}")
|
|
|
|
|
+ return False
|
|
|
|
|
+ if r.get("result"):
|
|
|
|
|
+ print("✅ JSON-RPC login OK")
|
|
|
|
|
+ return True
|
|
|
|
|
+ return False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _list_all_pages_via_api(session: requests.Session) -> list[dict]:
|
|
|
|
|
+ """
|
|
|
|
|
+ core.listPages avec namespace="" et depth=0 = toutes les pages du wiki.
|
|
|
|
|
+ Retourne une liste de {"id", "revision", "size", "title", "hash", ...}.
|
|
|
|
|
+ """
|
|
|
|
|
+ r = _jsonrpc_call("core.listPages", {"namespace": "", "depth": 0, "hash": True}, session)
|
|
|
|
|
+ if "error" in r:
|
|
|
|
|
+ error_code = r["error"].get("code", -1)
|
|
|
|
|
+ if error_code != 0: # 0 = succès
|
|
|
|
|
+ print(f"⚠️ core.listPages failed: {r['error'].get('message', r)}")
|
|
|
|
|
+ return []
|
|
|
|
|
+ pages = r.get("result") or []
|
|
|
|
|
+ return pages
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _get_page_content_via_api(page_id: str, session: requests.Session) -> str | None:
|
|
|
|
|
+ """core.getPage : retourne le wikitext de la page."""
|
|
|
|
|
+ r = _jsonrpc_call("core.getPage", {"page": page_id}, session)
|
|
|
|
|
+ if "error" in r:
|
|
|
|
|
+ error_code = r["error"].get("code", -1)
|
|
|
|
|
+ if error_code != 0: # 0 = succès
|
|
|
|
|
+ return None
|
|
|
|
|
+ return r.get("result") or ""
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _fetch_page(page_id: str, old_hashes: dict, cookies: dict) -> tuple[str, str | None, str | None]:
|
|
|
|
|
+ """Récupère une page via l'API avec session thread-safe. Retourne (page_id, text, hash)."""
|
|
|
|
|
+ # Créer une nouvelle session pour chaque thread (thread-safe)
|
|
|
|
|
+ session = requests.Session()
|
|
|
|
|
+ session.cookies.update(cookies)
|
|
|
|
|
+ session.headers.update({"User-Agent": random.choice(USER_AGENTS)})
|
|
|
|
|
+
|
|
|
|
|
+ text = _get_page_content_via_api(page_id, session)
|
|
|
|
|
+ if text is None:
|
|
|
|
|
+ return page_id, None, None
|
|
|
|
|
+
|
|
|
|
|
+ current_hash = hashlib.md5(text.encode("utf-8")).hexdigest()
|
|
|
|
|
+ url = _page_id_to_url(page_id)
|
|
|
|
|
+
|
|
|
|
|
+ if old_hashes.get(url) != current_hash:
|
|
|
|
|
+ print(f"🆕 Modifiée : {page_id}")
|
|
|
|
|
+ return page_id, text, current_hash # Retourner le texte seulement si modifié
|
|
|
|
|
+ else:
|
|
|
|
|
+ print(f"✅ Inchangée : {page_id}")
|
|
|
|
|
+ return page_id, None, current_hash # Pas de texte si inchangée
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _scrape_via_jsonrpc(session: requests.Session | None = None) -> tuple[list[dict], dict]:
|
|
|
|
|
+ """
|
|
|
|
|
+ Scraping complet via API JSON-RPC.
|
|
|
|
|
+ Si session fournie (avec cookies du login web), on l'utilise sans core.login.
|
|
|
|
|
+ """
|
|
|
|
|
+ sess = session or requests.Session()
|
|
|
|
|
+ sess.headers.setdefault("User-Agent", random.choice(USER_AGENTS))
|
|
|
|
|
+
|
|
|
|
|
+ if session is None:
|
|
|
|
|
+ if not _login_jsonrpc(sess):
|
|
|
|
|
+ return [], {"total_urls": 0, "pages_with_content": 0, "failed_urls": [], "mode": "jsonrpc", "error": "Login failed"}
|
|
|
|
|
+ else:
|
|
|
|
|
+ print("✅ Utilisation des cookies du login web pour l'API")
|
|
|
|
|
+
|
|
|
|
|
+ page_infos = _list_all_pages_via_api(sess)
|
|
|
|
|
+ if not page_infos:
|
|
|
|
|
+ return [], {"total_urls": 0, "pages_with_content": 0, "failed_urls": [], "mode": "jsonrpc", "error": "No pages from API"}
|
|
|
|
|
+
|
|
|
|
|
+ # Limiter si DEBUG
|
|
|
|
|
+ if MAX_URLS:
|
|
|
|
|
+ page_infos = page_infos[:MAX_URLS]
|
|
|
|
|
+ print(f"📋 Limite à {MAX_URLS} URLs")
|
|
|
|
|
+ print(f"📋 API : {len(page_infos)} pages à récupérer")
|
|
|
|
|
+
|
|
|
|
|
+ # Charger anciens hashes
|
|
|
|
|
+ hashes_path = os.path.join(data_dir, "page_hashes.json")
|
|
|
try:
|
|
try:
|
|
|
- headers = random_headers()
|
|
|
|
|
- edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
|
|
|
|
|
- response = session.get(edit_url, headers=headers, timeout=10)
|
|
|
|
|
|
|
+ with open(hashes_path, "r", encoding="utf-8") as f:
|
|
|
|
|
+ old_hashes = json.load(f)
|
|
|
|
|
+ except FileNotFoundError:
|
|
|
|
|
+ old_hashes = {}
|
|
|
|
|
|
|
|
- if response.status_code != 200:
|
|
|
|
|
- return url, None, None
|
|
|
|
|
|
|
+ # Récupérer le contenu de chaque page (parallèle)
|
|
|
|
|
+ all_page_ids = [p["id"] for p in page_infos]
|
|
|
|
|
+ url_to_text = {}
|
|
|
|
|
+ new_hashes = {}
|
|
|
|
|
|
|
|
- # Forcer l'encodage UTF-8 pour éviter les caractères bizarres
|
|
|
|
|
- response.encoding = 'utf-8'
|
|
|
|
|
|
|
+ # Extraire les cookies de la session pour les passer aux threads
|
|
|
|
|
+ session_cookies = dict(sess.cookies)
|
|
|
|
|
|
|
|
- if "text/html" not in response.headers.get("Content-Type", ""):
|
|
|
|
|
- return url, None, None
|
|
|
|
|
|
|
+ total = len(all_page_ids)
|
|
|
|
|
+ done = 0
|
|
|
|
|
+ with ThreadPoolExecutor(max_workers=3) as executor:
|
|
|
|
|
+ futures = {
|
|
|
|
|
+ executor.submit(_fetch_page, pid, old_hashes, session_cookies): pid
|
|
|
|
|
+ for pid in all_page_ids
|
|
|
|
|
+ }
|
|
|
|
|
+ for future in as_completed(futures):
|
|
|
|
|
+ page_id, text, h = future.result()
|
|
|
|
|
+ url = _page_id_to_url(page_id)
|
|
|
|
|
+ done += 1
|
|
|
|
|
+ print(f" [{done}/{total}] {url}")
|
|
|
|
|
+ if text is not None:
|
|
|
|
|
+ url_to_text[url] = text
|
|
|
|
|
+ if h is not None:
|
|
|
|
|
+ new_hashes[url] = h
|
|
|
|
|
+
|
|
|
|
|
+ # Fusionner avec ancien contenu (pages inchangées)
|
|
|
|
|
+ raw_pages_path = os.path.join(data_dir, "raw_pages.json")
|
|
|
|
|
+ try:
|
|
|
|
|
+ with open(raw_pages_path, "r", encoding="utf-8") as f:
|
|
|
|
|
+ old_raw = {p["url"]: p["text"] for p in json.load(f)}
|
|
|
|
|
+ except FileNotFoundError:
|
|
|
|
|
+ old_raw = {}
|
|
|
|
|
+ for u, t in url_to_text.items():
|
|
|
|
|
+ old_raw[u] = t
|
|
|
|
|
+ all_urls = sorted(set(_page_id_to_url(pid) for pid in all_page_ids))
|
|
|
|
|
+ cleaned = {u: old_raw[u] for u in all_urls if u in old_raw}
|
|
|
|
|
|
|
|
- soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
|
- textarea = soup.find("textarea", {"name": "wikitext"})
|
|
|
|
|
|
|
+ failed = [u for u in all_urls if u not in cleaned]
|
|
|
|
|
|
|
|
- if textarea:
|
|
|
|
|
- raw_text = textarea.get_text(separator="\n").strip()
|
|
|
|
|
- current_hash = hash_text(raw_text)
|
|
|
|
|
|
|
+ urls_path = os.path.join(data_dir, "urls.json")
|
|
|
|
|
+ with open(urls_path, "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump(all_urls, f, ensure_ascii=False, indent=2)
|
|
|
|
|
+ with open(raw_pages_path, "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump([{"url": u, "text": t} for u, t in cleaned.items()], f, ensure_ascii=False, indent=2)
|
|
|
|
|
+ with open(hashes_path, "w", encoding="utf-8") as f:
|
|
|
|
|
+ json.dump(new_hashes, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
|
|
- if old_hashes.get(url) != current_hash:
|
|
|
|
|
- print(f"🆕 Modifiée : {url}")
|
|
|
|
|
- return url, raw_text, current_hash
|
|
|
|
|
- else:
|
|
|
|
|
- print(f"✅ Inchangée : {url}")
|
|
|
|
|
- else:
|
|
|
|
|
- print(f"⚠️ Pas de <textarea> pour {url}")
|
|
|
|
|
- except Exception as e:
|
|
|
|
|
- print(f"❌ Erreur pour {url}: {e}")
|
|
|
|
|
|
|
+ print(f"📄 {len(cleaned)} pages récupérées via JSON-RPC")
|
|
|
|
|
+ print(f"🔗 URLs enregistrées dans '{urls_path}'")
|
|
|
|
|
|
|
|
- # 💡 Petite pause aléatoire
|
|
|
|
|
- time.sleep(random.uniform(0.5, 2))
|
|
|
|
|
- return url, None, None
|
|
|
|
|
|
|
+ pages_list = [{"url": u, "text": t} for u, t in cleaned.items()]
|
|
|
|
|
+ stats = {
|
|
|
|
|
+ "total_urls": len(all_page_ids),
|
|
|
|
|
+ "pages_with_content": len(cleaned),
|
|
|
|
|
+ "failed_urls": failed,
|
|
|
|
|
+ "mode": "jsonrpc",
|
|
|
|
|
+ }
|
|
|
|
|
+ return pages_list, stats
|
|
|
|
|
|
|
|
|
|
|
|
|
-def fetch_all_pages_content(urls, old_hashes):
|
|
|
|
|
- url_texts = {}
|
|
|
|
|
- new_hashes = {}
|
|
|
|
|
|
|
+# --- Fallback : scraping HTML (si API désactivée) ---
|
|
|
|
|
|
|
|
- with ThreadPoolExecutor(max_workers=2) as executor:
|
|
|
|
|
- futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls]
|
|
|
|
|
|
|
+def _scrape_via_html() -> tuple[list[dict], dict]:
|
|
|
|
|
+ """Fallback : scraping HTML classique (do=edit, textarea)."""
|
|
|
|
|
+ from src.agent_ai.auth import session
|
|
|
|
|
+ from bs4 import BeautifulSoup
|
|
|
|
|
+ from urllib.parse import urljoin, urlparse
|
|
|
|
|
|
|
|
- for future in as_completed(futures):
|
|
|
|
|
- url, text, current_hash = future.result()
|
|
|
|
|
- if text:
|
|
|
|
|
- url_texts[url] = text
|
|
|
|
|
- new_hashes[url] = current_hash
|
|
|
|
|
- elif current_hash:
|
|
|
|
|
- new_hashes[url] = current_hash
|
|
|
|
|
-
|
|
|
|
|
- return url_texts, new_hashes
|
|
|
|
|
-
|
|
|
|
|
-def is_valid_wiki_url(url):
|
|
|
|
|
- parsed = urlparse(url)
|
|
|
|
|
- return (
|
|
|
|
|
- parsed.netloc == "wiki.intersel.fr"
|
|
|
|
|
- and "do=" not in parsed.query
|
|
|
|
|
- and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
|
|
|
|
|
- and "#" not in url
|
|
|
|
|
- )
|
|
|
|
|
-
|
|
|
|
|
-def crawl_wiki(start_url, existing_urls):
|
|
|
|
|
- visited = set(existing_urls)
|
|
|
|
|
- to_visit = [start_url]
|
|
|
|
|
-
|
|
|
|
|
- while to_visit and len(visited) < MAX_URLS_TO_SCRAPE:
|
|
|
|
|
- current_url = to_visit.pop(0)
|
|
|
|
|
- if current_url in visited:
|
|
|
|
|
- continue
|
|
|
|
|
-
|
|
|
|
|
- print(f"🔎 Visiting: {current_url}")
|
|
|
|
|
- visited.add(current_url)
|
|
|
|
|
|
|
+ def is_valid_wiki_url(url):
|
|
|
|
|
+ parsed = urlparse(url)
|
|
|
|
|
+ return (
|
|
|
|
|
+ parsed.netloc == "wiki.intersel.fr"
|
|
|
|
|
+ and "do=" not in parsed.query
|
|
|
|
|
+ and "idx=" not in parsed.query
|
|
|
|
|
+ and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
|
|
|
|
|
+ and "#" not in url
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
|
|
+ def _fetch_index_page_links(index_url):
|
|
|
|
|
+ """Récupère les liens d'une page index : pages (id=) et sous-index (do=index&idx=)."""
|
|
|
|
|
+ content_urls, sub_index_urls = set(), set()
|
|
|
try:
|
|
try:
|
|
|
- headers = random_headers()
|
|
|
|
|
- response = session.get(current_url, headers=headers, timeout=10)
|
|
|
|
|
- if response.status_code != 200:
|
|
|
|
|
- continue
|
|
|
|
|
-
|
|
|
|
|
- soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
|
-
|
|
|
|
|
|
|
+ resp = session.get(index_url, headers={"User-Agent": random.choice(USER_AGENTS)}, timeout=15)
|
|
|
|
|
+ if resp.status_code != 200:
|
|
|
|
|
+ return content_urls, sub_index_urls
|
|
|
|
|
+ soup = BeautifulSoup(resp.text, "html.parser")
|
|
|
for link in soup.find_all("a", href=True):
|
|
for link in soup.find_all("a", href=True):
|
|
|
- absolute_url = urljoin(current_url, link["href"])
|
|
|
|
|
- if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit:
|
|
|
|
|
- to_visit.append(absolute_url)
|
|
|
|
|
-
|
|
|
|
|
- time.sleep(random.uniform(1, 2))
|
|
|
|
|
|
|
+ abs_url = urljoin(index_url, link["href"])
|
|
|
|
|
+ if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
|
|
|
|
|
+ continue
|
|
|
|
|
+ parsed = urlparse(abs_url)
|
|
|
|
|
+ if "do=index" in parsed.query:
|
|
|
|
|
+ sub_index_urls.add(abs_url)
|
|
|
|
|
+ elif "do=" not in parsed.query and "idx=" not in parsed.query and "id=" in parsed.query:
|
|
|
|
|
+ content_urls.add(abs_url)
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
- print(f"❌ Error with {current_url}: {e}")
|
|
|
|
|
-
|
|
|
|
|
- return list(visited)
|
|
|
|
|
-
|
|
|
|
|
|
|
+ print(f"⚠️ Index {index_url}: {e}")
|
|
|
|
|
+ return content_urls, sub_index_urls
|
|
|
|
|
+
|
|
|
|
|
+ def fetch_crawl_bfs_urls(max_pages=200):
|
|
|
|
|
+ """Crawl BFS depuis la homepage : suit tous les liens internes. Meilleure source (1239+ URLs)."""
|
|
|
|
|
+ base = BASE_URL.rstrip("/")
|
|
|
|
|
+ to_visit = [base, f"{base}/doku.php?id=start"]
|
|
|
|
|
+ visited = set()
|
|
|
|
|
+ content_urls = set()
|
|
|
|
|
+
|
|
|
|
|
+ while to_visit and len(visited) < max_pages:
|
|
|
|
|
+ url = to_visit.pop(0)
|
|
|
|
|
+ if url in visited:
|
|
|
|
|
+ continue
|
|
|
|
|
+ visited.add(url)
|
|
|
|
|
+ try:
|
|
|
|
|
+ r = session.get(url, headers={"User-Agent": random.choice(USER_AGENTS)}, timeout=10)
|
|
|
|
|
+ if r.status_code != 200:
|
|
|
|
|
+ continue
|
|
|
|
|
+ soup = BeautifulSoup(r.text, "html.parser")
|
|
|
|
|
+ for link in soup.find_all("a", href=True):
|
|
|
|
|
+ abs_url = urljoin(url, link["href"])
|
|
|
|
|
+ if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
|
|
|
|
|
+ continue
|
|
|
|
|
+ parsed = urlparse(abs_url)
|
|
|
|
|
+ if "do=" in parsed.query or "idx=" in parsed.query:
|
|
|
|
|
+ continue
|
|
|
|
|
+ if "id=" in parsed.query or abs_url.rstrip("/") == base:
|
|
|
|
|
+ clean = abs_url.split("&")[0].split("#")[0]
|
|
|
|
|
+ if clean not in visited and clean not in to_visit:
|
|
|
|
|
+ to_visit.append(clean)
|
|
|
|
|
+ content_urls.add(clean)
|
|
|
|
|
+ content_urls.add(url.split("&")[0].split("#")[0])
|
|
|
|
|
+ time.sleep(random.uniform(0.15, 0.35))
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ pass
|
|
|
|
|
+
|
|
|
|
|
+ content_urls.add(base)
|
|
|
|
|
+
|
|
|
|
|
+ def norm(u):
|
|
|
|
|
+ u = u.split("#")[0].split("&")[0].rstrip("/")
|
|
|
|
|
+ if u.endswith("/doku.php") or u == base + "/":
|
|
|
|
|
+ return base
|
|
|
|
|
+ return u
|
|
|
|
|
+
|
|
|
|
|
+ urls = [norm(u) for u in content_urls if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u]
|
|
|
|
|
+ # Filtrer et dédupliquer
|
|
|
|
|
+ urls = sorted(set(u for u in urls if is_valid_wiki_url(u)))
|
|
|
|
|
+ if urls:
|
|
|
|
|
+ print(f"📋 Crawl BFS : {len(urls)} URLs")
|
|
|
|
|
+ return urls
|
|
|
|
|
+
|
|
|
|
|
+ def fetch_sitemap_urls():
|
|
|
|
|
+ """Tente de récupérer les URLs depuis le sitemap (si activé)."""
|
|
|
|
|
+ import re
|
|
|
|
|
+ for sitemap_url in [
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?sitemap=xml",
|
|
|
|
|
+ "https://wiki.intersel.fr/doku.php?do=sitemap",
|
|
|
|
|
+ ]:
|
|
|
|
|
+ try:
|
|
|
|
|
+ r = session.get(sitemap_url, headers={"User-Agent": random.choice(USER_AGENTS)}, timeout=10)
|
|
|
|
|
+ if r.status_code != 200:
|
|
|
|
|
+ continue
|
|
|
|
|
+ text = r.text or ""
|
|
|
|
|
+ urls = []
|
|
|
|
|
+ if "<loc>" in text:
|
|
|
|
|
+ found = re.findall(r"<loc>([^<]+)</loc>", text)
|
|
|
|
|
+ urls = [u.strip() for u in found if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u]
|
|
|
|
|
+ elif "doku.php?id=" in text:
|
|
|
|
|
+ found = re.findall(r'https://wiki\.intersel\.fr/doku\.php\?id=[^"\'<>\s&]+', text)
|
|
|
|
|
+ urls = [u.split("&")[0].rstrip(".,;:)") for u in found]
|
|
|
|
|
+ urls = list(dict.fromkeys(urls))
|
|
|
|
|
+ if urls:
|
|
|
|
|
+ urls.insert(0, BASE_URL.rstrip("/"))
|
|
|
|
|
+ urls = list(dict.fromkeys(urls))
|
|
|
|
|
+ print(f"📋 Sitemap : {len(urls)} URLs")
|
|
|
|
|
+ return urls
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ pass
|
|
|
|
|
+ return []
|
|
|
|
|
+
|
|
|
|
|
+ def fetch_index_urls():
|
|
|
|
|
+ """Parcourt l'index et les sous-index pour collecter toutes les URLs."""
|
|
|
|
|
+ base_index = "https://wiki.intersel.fr/doku.php?do=index"
|
|
|
|
|
+ all_content = set()
|
|
|
|
|
+ to_visit, visited = [base_index], set()
|
|
|
|
|
+ while to_visit:
|
|
|
|
|
+ url = to_visit.pop(0)
|
|
|
|
|
+ if url in visited:
|
|
|
|
|
+ continue
|
|
|
|
|
+ visited.add(url)
|
|
|
|
|
+ content, sub = _fetch_index_page_links(url)
|
|
|
|
|
+ all_content |= content
|
|
|
|
|
+ for s in sub:
|
|
|
|
|
+ if s not in visited and s not in to_visit:
|
|
|
|
|
+ to_visit.append(s)
|
|
|
|
|
+ time.sleep(random.uniform(0.3, 0.6))
|
|
|
|
|
+ all_content.add(BASE_URL.rstrip("/"))
|
|
|
|
|
+ return list(all_content)
|
|
|
|
|
+
|
|
|
|
|
+ def _to_edit_url(url):
|
|
|
|
|
+ base = BASE_URL.rstrip("/")
|
|
|
|
|
+ if url == base or url == base + "/":
|
|
|
|
|
+ return f"{base}/doku.php?id=start&do=edit"
|
|
|
|
|
+ return url + ("&do=edit" if "?" in url else "?do=edit")
|
|
|
|
|
+
|
|
|
|
|
+ def fetch_page(url, old_hashes):
|
|
|
|
|
+ try:
|
|
|
|
|
+ edit_url = _to_edit_url(url)
|
|
|
|
|
+ r = session.get(edit_url, timeout=15)
|
|
|
|
|
+ if r.status_code != 200:
|
|
|
|
|
+ return url, None, None
|
|
|
|
|
+ soup = BeautifulSoup(r.text, "html.parser")
|
|
|
|
|
+ ta = soup.find("textarea", {"name": "wikitext"})
|
|
|
|
|
+ if ta:
|
|
|
|
|
+ text = ta.get_text(separator="\n").strip()
|
|
|
|
|
+ h = hashlib.md5(text.encode("utf-8")).hexdigest()
|
|
|
|
|
+ return url, text if old_hashes.get(url) != h else None, h
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ print(f"❌ {url}: {e}")
|
|
|
|
|
+ return url, None, None
|
|
|
|
|
|
|
|
-def scrape_pages():
|
|
|
|
|
- start_url = "https://wiki.intersel.fr/"
|
|
|
|
|
urls_path = os.path.join(data_dir, "urls.json")
|
|
urls_path = os.path.join(data_dir, "urls.json")
|
|
|
raw_pages_path = os.path.join(data_dir, "raw_pages.json")
|
|
raw_pages_path = os.path.join(data_dir, "raw_pages.json")
|
|
|
hashes_path = os.path.join(data_dir, "page_hashes.json")
|
|
hashes_path = os.path.join(data_dir, "page_hashes.json")
|
|
|
|
|
|
|
|
- # Charger anciens hashs
|
|
|
|
|
try:
|
|
try:
|
|
|
with open(hashes_path, "r", encoding="utf-8") as f:
|
|
with open(hashes_path, "r", encoding="utf-8") as f:
|
|
|
old_hashes = json.load(f)
|
|
old_hashes = json.load(f)
|
|
|
except FileNotFoundError:
|
|
except FileNotFoundError:
|
|
|
old_hashes = {}
|
|
old_hashes = {}
|
|
|
|
|
|
|
|
- # Charger anciennes URLs
|
|
|
|
|
try:
|
|
try:
|
|
|
with open(urls_path, "r", encoding="utf-8") as f:
|
|
with open(urls_path, "r", encoding="utf-8") as f:
|
|
|
- urls = json.load(f)
|
|
|
|
|
- print("✅ URLs chargées depuis urls.json")
|
|
|
|
|
|
|
+ existing = set(json.load(f))
|
|
|
except FileNotFoundError:
|
|
except FileNotFoundError:
|
|
|
- print("🚀 Crawling initial du wiki")
|
|
|
|
|
- urls = []
|
|
|
|
|
-
|
|
|
|
|
- # Nouveau crawl (limité à MAX_URLS_TO_SCRAPE)
|
|
|
|
|
- crawled_urls = crawl_wiki(start_url, urls)
|
|
|
|
|
- all_urls = sorted(set(urls + crawled_urls))[:MAX_URLS_TO_SCRAPE]
|
|
|
|
|
-
|
|
|
|
|
- print(f"🌐 Total URLs à vérifier (max {MAX_URLS_TO_SCRAPE}) : {len(all_urls)}")
|
|
|
|
|
-
|
|
|
|
|
- # Scraping des pages
|
|
|
|
|
- pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes)
|
|
|
|
|
|
|
+ existing = set()
|
|
|
|
|
+
|
|
|
|
|
+ crawl_urls = fetch_crawl_bfs_urls(max_pages=200)
|
|
|
|
|
+ sitemap_urls = fetch_sitemap_urls() if not crawl_urls else []
|
|
|
|
|
+ index_urls = fetch_index_urls() if not (crawl_urls or sitemap_urls) else []
|
|
|
|
|
+ all_urls = sorted(existing | set(crawl_urls or sitemap_urls or index_urls) | {BASE_URL.rstrip("/")})
|
|
|
|
|
+ all_urls = [u for u in all_urls if is_valid_wiki_url(u)]
|
|
|
|
|
+ if MAX_URLS:
|
|
|
|
|
+ all_urls = all_urls[:MAX_URLS]
|
|
|
|
|
+ src = "crawl BFS" if crawl_urls else ("sitemap" if sitemap_urls else "index")
|
|
|
|
|
+ print(f"🌐 HTML fallback : {len(all_urls)} URLs ({src})")
|
|
|
|
|
+
|
|
|
|
|
+ url_to_text = {}
|
|
|
|
|
+ new_hashes = {}
|
|
|
|
|
+ total = len(all_urls)
|
|
|
|
|
+ for i, url in enumerate(all_urls, 1):
|
|
|
|
|
+ print(f" [{i}/{total}] {url}")
|
|
|
|
|
+ _, text, h = fetch_page(url, old_hashes)
|
|
|
|
|
+ if text:
|
|
|
|
|
+ url_to_text[url] = text
|
|
|
|
|
+ if h:
|
|
|
|
|
+ new_hashes[url] = h
|
|
|
|
|
+ time.sleep(random.uniform(0.3, 0.8))
|
|
|
|
|
|
|
|
- # Charger anciens raw_pages
|
|
|
|
|
try:
|
|
try:
|
|
|
with open(raw_pages_path, "r", encoding="utf-8") as f:
|
|
with open(raw_pages_path, "r", encoding="utf-8") as f:
|
|
|
- old_raw_pages = {p["url"]: p["text"] for p in json.load(f)}
|
|
|
|
|
|
|
+ old_raw = {p["url"]: p["text"] for p in json.load(f)}
|
|
|
except FileNotFoundError:
|
|
except FileNotFoundError:
|
|
|
- old_raw_pages = {}
|
|
|
|
|
|
|
+ old_raw = {}
|
|
|
|
|
+ for u, t in url_to_text.items():
|
|
|
|
|
+ old_raw[u] = t
|
|
|
|
|
+ cleaned = {u: t for u, t in old_raw.items() if u in all_urls}
|
|
|
|
|
|
|
|
- # Mettre à jour uniquement les pages modifiées
|
|
|
|
|
- for url, text in pages.items():
|
|
|
|
|
- old_raw_pages[url] = text
|
|
|
|
|
-
|
|
|
|
|
- # Enlever les URLs qui n’existent plus (optionnel)
|
|
|
|
|
- cleaned_raw_pages = {url: text for url, text in old_raw_pages.items() if url in all_urls}
|
|
|
|
|
-
|
|
|
|
|
- # Sauvegarder le nouveau contenu
|
|
|
|
|
with open(raw_pages_path, "w", encoding="utf-8") as f:
|
|
with open(raw_pages_path, "w", encoding="utf-8") as f:
|
|
|
- json.dump(
|
|
|
|
|
- [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()],
|
|
|
|
|
- f, ensure_ascii=False, indent=2
|
|
|
|
|
- )
|
|
|
|
|
-
|
|
|
|
|
|
|
+ json.dump([{"url": u, "text": t} for u, t in cleaned.items()], f, ensure_ascii=False, indent=2)
|
|
|
with open(hashes_path, "w", encoding="utf-8") as f:
|
|
with open(hashes_path, "w", encoding="utf-8") as f:
|
|
|
json.dump(new_hashes, f, ensure_ascii=False, indent=2)
|
|
json.dump(new_hashes, f, ensure_ascii=False, indent=2)
|
|
|
-
|
|
|
|
|
with open(urls_path, "w", encoding="utf-8") as f:
|
|
with open(urls_path, "w", encoding="utf-8") as f:
|
|
|
json.dump(all_urls, f, ensure_ascii=False, indent=2)
|
|
json.dump(all_urls, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
|
|
- print("📄 Pages modifiées sauvegardées")
|
|
|
|
|
- print(f"📄 Contenu brut mis à jour enregistré dans '{raw_pages_path}'")
|
|
|
|
|
- print(f"🔗 URLs enregistrées dans '{urls_path}'")
|
|
|
|
|
- print(f"🔐 Hashes des pages enregistrés dans '{hashes_path}'")
|
|
|
|
|
|
|
+ failed = [u for u in all_urls if u not in cleaned]
|
|
|
|
|
+ pages_list = [{"url": u, "text": t} for u, t in cleaned.items()]
|
|
|
|
|
+ stats = {"total_urls": len(all_urls), "pages_with_content": len(cleaned), "failed_urls": failed, "mode": "html"}
|
|
|
|
|
+ return pages_list, stats
|
|
|
|
|
|
|
|
- pages_list = [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()]
|
|
|
|
|
- failed_urls = [u for u in all_urls if u not in cleaned_raw_pages]
|
|
|
|
|
- stats = {
|
|
|
|
|
- "total_urls": len(all_urls),
|
|
|
|
|
- "pages_with_content": len(cleaned_raw_pages),
|
|
|
|
|
- "failed_urls": failed_urls,
|
|
|
|
|
- }
|
|
|
|
|
- return pages_list, stats
|
|
|
|
|
|
|
+
|
|
|
|
|
+def _is_api_available(session: requests.Session | None = None) -> bool:
|
|
|
|
|
+ """Test disponibilité de l'API JSON-RPC. Option: session avec cookies (après login web)."""
|
|
|
|
|
+ sess = session or requests.Session()
|
|
|
|
|
+ url = f"{JSONRPC_URL}/core.getAPIVersion"
|
|
|
|
|
+ try:
|
|
|
|
|
+ r = sess.post(
|
|
|
|
|
+ url,
|
|
|
|
|
+ json={},
|
|
|
|
|
+ headers={"Content-Type": "application/json", "User-Agent": random.choice(USER_AGENTS)},
|
|
|
|
|
+ timeout=5,
|
|
|
|
|
+ )
|
|
|
|
|
+ data = r.json() if r.text else {}
|
|
|
|
|
+ ok = r.status_code == 200 and "result" in data
|
|
|
|
|
+ if not ok:
|
|
|
|
|
+ print(f" → API: status={r.status_code}, response={str(data)[:80]}...")
|
|
|
|
|
+ return ok
|
|
|
|
|
+ except Exception as e:
|
|
|
|
|
+ print(f" → API indisponible: {e}")
|
|
|
|
|
+ return False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def scrape_pages() -> tuple[list[dict], dict]:
|
|
|
|
|
+ """
|
|
|
|
|
+ Point d'entrée principal. Essaie d'abord l'API JSON-RPC, puis fallback HTML.
|
|
|
|
|
+ Stratégie : login web d'abord, puis test API avec les cookies (fonctionne même si remote
|
|
|
|
|
+ exige une session valide). Sinon fallback HTML avec parcours des sous-index.
|
|
|
|
|
+ """
|
|
|
|
|
+ from src.agent_ai.auth import session as auth_session
|
|
|
|
|
+ from src.service.auth_service import auto_login
|
|
|
|
|
+
|
|
|
|
|
+ auto_login() # Toujours login web (pour fallback ou pour cookies API)
|
|
|
|
|
+
|
|
|
|
|
+ # 1. Tester l'API avec la session authentifiée (cookies du login web)
|
|
|
|
|
+ if _is_api_available(auth_session):
|
|
|
|
|
+ pages_list, stats = _scrape_via_jsonrpc(session=auth_session)
|
|
|
|
|
+ if not stats.get("error"):
|
|
|
|
|
+ return pages_list, stats
|
|
|
|
|
+ print(f"⚠️ API error, fallback HTML: {stats.get('error')}")
|
|
|
|
|
+ else:
|
|
|
|
|
+ print("⚠️ API non disponible, fallback HTML")
|
|
|
|
|
+
|
|
|
|
|
+ # 2. Fallback : scraping HTML avec parcours des sous-index
|
|
|
|
|
+ return _scrape_via_html()
|