""" Test de récupération d'URLs du wiki par différentes méthodes. Depuis agent_docker : py .\test-to-delete\test_fetch_urls.py """ import sys import os agent_docker_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if agent_docker_root not in sys.path: sys.path.insert(0, agent_docker_root) os.chdir(agent_docker_root) from dotenv import load_dotenv load_dotenv() import time import requests from bs4 import BeautifulSoup from urllib.parse import urljoin, urlparse BASE_URL = "https://wiki.intersel.fr/" JSONRPC_URL = "https://wiki.intersel.fr/lib/exe/jsonrpc.php" XMLRPC_URL = "https://wiki.intersel.fr/lib/exe/xmlrpc.php" INDEX_URL = "https://wiki.intersel.fr/doku.php?do=index" USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0" def is_valid_page_url(url): """URL de page de contenu (id=, pas do=, pas idx=).""" parsed = urlparse(url) return ( "wiki.intersel.fr" in url and "do=" not in parsed.query and "idx=" not in parsed.query and "id=" in parsed.query and "#" not in url ) def test_jsonrpc(session): """Test JSON-RPC core.getAPIVersion.""" print("\n--- 1. JSON-RPC ---") try: r = session.post( f"{JSONRPC_URL}/core.getAPIVersion", json={}, headers={"Content-Type": "application/json", "User-Agent": USER_AGENT}, timeout=5, ) data = r.json() if r.text else {} print(f" Status: {r.status_code}") print(f" Response: {data}") if r.status_code == 200 and "result" in data: print(" ✅ JSON-RPC OK") return True print(" ❌ JSON-RPC non disponible") return False except Exception as e: print(f" ❌ Erreur: {e}") return False def test_xmlrpc(session): """Test XML-RPC (alternative à JSON-RPC).""" print("\n--- 2. XML-RPC ---") try: xml_body = """ wiki.getVersion """ r = session.post( XMLRPC_URL, data=xml_body, headers={"Content-Type": "text/xml", "User-Agent": USER_AGENT}, timeout=5, ) print(f" Status: {r.status_code}") print(f" Response (100 chars): {r.text[:100] if r.text else 'vide'}...") if r.status_code == 200 and "methodResponse" in (r.text or ""): print(" ✅ XML-RPC répond") return True print(" ❌ XML-RPC non disponible") return False except Exception as e: print(f" ❌ Erreur: {e}") return False def test_index_page(session): """Récupère les URLs depuis la page index (sans sous-index).""" print("\n--- 3. Index HTML (do=index) ---") urls = set() try: r = session.get(INDEX_URL, headers={"User-Agent": USER_AGENT}, timeout=10) print(f" Status: {r.status_code}") if r.status_code != 200: return [] soup = BeautifulSoup(r.text, "html.parser") for link in soup.find_all("a", href=True): abs_url = urljoin(INDEX_URL, link["href"]) if is_valid_page_url(abs_url): urls.add(abs_url) urls.add(BASE_URL.rstrip("/")) print(f" ✅ {len(urls)} URLs (pages id= + homepage)") return sorted(urls) except Exception as e: print(f" ❌ Erreur: {e}") return [] def test_crawl_bfs(session, max_pages=100): """Crawl BFS depuis la homepage : suit tous les liens internes du wiki.""" print("\n--- 4. Crawl BFS (depuis homepage) ---") base = BASE_URL.rstrip("/") to_visit = [base, f"{base}/doku.php?id=start"] visited = set() content_urls = set() while to_visit and len(visited) < max_pages: url = to_visit.pop(0) if url in visited: continue visited.add(url) try: r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10) if r.status_code != 200: continue soup = BeautifulSoup(r.text, "html.parser") for link in soup.find_all("a", href=True): abs_url = urljoin(url, link["href"]) if "wiki.intersel.fr" not in abs_url or "#" in abs_url: continue parsed = urlparse(abs_url) if "do=" in parsed.query or "idx=" in parsed.query: continue if "id=" in parsed.query or abs_url.rstrip("/") == base: clean = abs_url.split("&")[0].split("#")[0] if clean not in visited and clean not in to_visit: to_visit.append(clean) content_urls.add(clean) content_urls.add(url.split("&")[0].split("#")[0]) time.sleep(0.2) except Exception as e: print(f" ⚠️ {url[:50]}: {e}") content_urls.add(base) def norm(u): u = u.split("#")[0].split("&")[0].rstrip("/") if u.endswith("/doku.php") or u == base + "/": return base return u urls = sorted(set(norm(u) for u in content_urls if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u)) print(f" ✅ {len(urls)} URLs (après {len(visited)} pages visitées)") return urls def test_index_with_subindex(session): """Parcourt index + sous-index (do=index&idx=...).""" print("\n--- 5. Index + sous-index (parcours récursif) ---") content_urls = set() sub_indexes = set() to_visit = [INDEX_URL] visited = set() while to_visit: url = to_visit.pop(0) if url in visited: continue visited.add(url) try: r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10) if r.status_code != 200: continue soup = BeautifulSoup(r.text, "html.parser") for link in soup.find_all("a", href=True): abs_url = urljoin(url, link["href"]) if "wiki.intersel.fr" not in abs_url or "#" in abs_url: continue parsed = urlparse(abs_url) if "do=index" in parsed.query: sub_indexes.add(abs_url) if abs_url not in visited and abs_url not in to_visit: to_visit.append(abs_url) elif is_valid_page_url(abs_url): content_urls.add(abs_url) except Exception as e: print(f" ⚠️ {url}: {e}") content_urls.add(BASE_URL.rstrip("/")) print(f" ✅ {len(content_urls)} URLs (après {len(visited)} pages d'index)") return sorted(content_urls) def test_sitemap(session): """Test et parse le sitemap (plusieurs URLs possibles).""" print("\n--- 6. Sitemap ---") sitemap_urls = [ "https://wiki.intersel.fr/doku.php?sitemap=xml", "https://wiki.intersel.fr/doku.php?do=sitemap", ] urls = [] for sitemap_url in sitemap_urls: try: r = session.get(sitemap_url, headers={"User-Agent": USER_AGENT}, timeout=10) print(f" {sitemap_url}: status={r.status_code}") text = r.text or "" # Debug : afficher le début pour comprendre le format if not urls and text: print(f" [DEBUG] Aperçu (300 chars): {repr(text[:300])}") import re # Format XML sitemap : https://... if "" in text: found = re.findall(r"([^<]+)", text) urls = [u.strip() for u in found if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u] # Format HTML : extraire les liens doku.php?id= elif "doku.php" in text or "id=" in text: found = re.findall(r'https://wiki\.intersel\.fr/doku\.php\?id=[^"\'<>\s&]+', text) urls = [u.split("&")[0].rstrip(".,;:)") for u in found] urls = list(dict.fromkeys(urls)) if urls: urls.insert(0, BASE_URL.rstrip("/")) urls = list(dict.fromkeys(urls)) print(f" ✅ {len(urls)} URLs extraites") return urls except Exception as e: print(f" Erreur: {e}") print(" ⚠️ 0 URLs extraites") return urls def main(): print("=" * 50) print("Test récupération URLs wiki Intersel") print("=" * 50) from src.service.auth_service import auto_login from src.agent_ai.auth import session auto_login() # Tests test_jsonrpc(session) test_xmlrpc(session) urls_index = test_index_page(session) urls_crawl = test_crawl_bfs(session, max_pages=150) urls_subindex = test_index_with_subindex(session) urls_sitemap = test_sitemap(session) # Résumé print("\n" + "=" * 50) print("RÉSUMÉ") print("=" * 50) print(f"Index simple : {len(urls_index)} URLs") print(f"Crawl BFS : {len(urls_crawl)} URLs") print(f"Index + sous-index: {len(urls_subindex)} URLs") print(f"Sitemap : {len(urls_sitemap)} URLs") best = max([urls_crawl, urls_subindex, urls_sitemap], key=len) if best: print(f"\n>>> Meilleure source: {len(best)} URLs") for u in best[:8]: print(f" - {u[:75]}...") if len(best) > 8: print(f" ... et {len(best) - 8} autres") if __name__ == "__main__": main()