| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270 |
- """
- Test de récupération d'URLs du wiki par différentes méthodes.
- Depuis agent_docker : py .\test-to-delete\test_fetch_urls.py
- """
- import sys
- import os
- agent_docker_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
- if agent_docker_root not in sys.path:
- sys.path.insert(0, agent_docker_root)
- os.chdir(agent_docker_root)
- from dotenv import load_dotenv
- load_dotenv()
- import time
- import requests
- from bs4 import BeautifulSoup
- from urllib.parse import urljoin, urlparse
- BASE_URL = "https://wiki.intersel.fr/"
- JSONRPC_URL = "https://wiki.intersel.fr/lib/exe/jsonrpc.php"
- XMLRPC_URL = "https://wiki.intersel.fr/lib/exe/xmlrpc.php"
- INDEX_URL = "https://wiki.intersel.fr/doku.php?do=index"
- USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0"
- def is_valid_page_url(url):
- """URL de page de contenu (id=, pas do=, pas idx=)."""
- parsed = urlparse(url)
- return (
- "wiki.intersel.fr" in url
- and "do=" not in parsed.query
- and "idx=" not in parsed.query
- and "id=" in parsed.query
- and "#" not in url
- )
- def test_jsonrpc(session):
- """Test JSON-RPC core.getAPIVersion."""
- print("\n--- 1. JSON-RPC ---")
- try:
- r = session.post(
- f"{JSONRPC_URL}/core.getAPIVersion",
- json={},
- headers={"Content-Type": "application/json", "User-Agent": USER_AGENT},
- timeout=5,
- )
- data = r.json() if r.text else {}
- print(f" Status: {r.status_code}")
- print(f" Response: {data}")
- if r.status_code == 200 and "result" in data:
- print(" ✅ JSON-RPC OK")
- return True
- print(" ❌ JSON-RPC non disponible")
- return False
- except Exception as e:
- print(f" ❌ Erreur: {e}")
- return False
- def test_xmlrpc(session):
- """Test XML-RPC (alternative à JSON-RPC)."""
- print("\n--- 2. XML-RPC ---")
- try:
- xml_body = """<?xml version="1.0"?>
- <methodCall>
- <methodName>wiki.getVersion</methodName>
- <params></params>
- </methodCall>"""
- r = session.post(
- XMLRPC_URL,
- data=xml_body,
- headers={"Content-Type": "text/xml", "User-Agent": USER_AGENT},
- timeout=5,
- )
- print(f" Status: {r.status_code}")
- print(f" Response (100 chars): {r.text[:100] if r.text else 'vide'}...")
- if r.status_code == 200 and "methodResponse" in (r.text or ""):
- print(" ✅ XML-RPC répond")
- return True
- print(" ❌ XML-RPC non disponible")
- return False
- except Exception as e:
- print(f" ❌ Erreur: {e}")
- return False
- def test_index_page(session):
- """Récupère les URLs depuis la page index (sans sous-index)."""
- print("\n--- 3. Index HTML (do=index) ---")
- urls = set()
- try:
- r = session.get(INDEX_URL, headers={"User-Agent": USER_AGENT}, timeout=10)
- print(f" Status: {r.status_code}")
- if r.status_code != 200:
- return []
- soup = BeautifulSoup(r.text, "html.parser")
- for link in soup.find_all("a", href=True):
- abs_url = urljoin(INDEX_URL, link["href"])
- if is_valid_page_url(abs_url):
- urls.add(abs_url)
- urls.add(BASE_URL.rstrip("/"))
- print(f" ✅ {len(urls)} URLs (pages id= + homepage)")
- return sorted(urls)
- except Exception as e:
- print(f" ❌ Erreur: {e}")
- return []
- def test_crawl_bfs(session, max_pages=100):
- """Crawl BFS depuis la homepage : suit tous les liens internes du wiki."""
- print("\n--- 4. Crawl BFS (depuis homepage) ---")
- base = BASE_URL.rstrip("/")
- to_visit = [base, f"{base}/doku.php?id=start"]
- visited = set()
- content_urls = set()
- while to_visit and len(visited) < max_pages:
- url = to_visit.pop(0)
- if url in visited:
- continue
- visited.add(url)
- try:
- r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
- if r.status_code != 200:
- continue
- soup = BeautifulSoup(r.text, "html.parser")
- for link in soup.find_all("a", href=True):
- abs_url = urljoin(url, link["href"])
- if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
- continue
- parsed = urlparse(abs_url)
- if "do=" in parsed.query or "idx=" in parsed.query:
- continue
- if "id=" in parsed.query or abs_url.rstrip("/") == base:
- clean = abs_url.split("&")[0].split("#")[0]
- if clean not in visited and clean not in to_visit:
- to_visit.append(clean)
- content_urls.add(clean)
- content_urls.add(url.split("&")[0].split("#")[0])
- time.sleep(0.2)
- except Exception as e:
- print(f" ⚠️ {url[:50]}: {e}")
- content_urls.add(base)
- def norm(u):
- u = u.split("#")[0].split("&")[0].rstrip("/")
- if u.endswith("/doku.php") or u == base + "/":
- return base
- return u
- urls = sorted(set(norm(u) for u in content_urls if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u))
- print(f" ✅ {len(urls)} URLs (après {len(visited)} pages visitées)")
- return urls
- def test_index_with_subindex(session):
- """Parcourt index + sous-index (do=index&idx=...)."""
- print("\n--- 5. Index + sous-index (parcours récursif) ---")
- content_urls = set()
- sub_indexes = set()
- to_visit = [INDEX_URL]
- visited = set()
- while to_visit:
- url = to_visit.pop(0)
- if url in visited:
- continue
- visited.add(url)
- try:
- r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
- if r.status_code != 200:
- continue
- soup = BeautifulSoup(r.text, "html.parser")
- for link in soup.find_all("a", href=True):
- abs_url = urljoin(url, link["href"])
- if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
- continue
- parsed = urlparse(abs_url)
- if "do=index" in parsed.query:
- sub_indexes.add(abs_url)
- if abs_url not in visited and abs_url not in to_visit:
- to_visit.append(abs_url)
- elif is_valid_page_url(abs_url):
- content_urls.add(abs_url)
- except Exception as e:
- print(f" ⚠️ {url}: {e}")
- content_urls.add(BASE_URL.rstrip("/"))
- print(f" ✅ {len(content_urls)} URLs (après {len(visited)} pages d'index)")
- return sorted(content_urls)
- def test_sitemap(session):
- """Test et parse le sitemap (plusieurs URLs possibles)."""
- print("\n--- 6. Sitemap ---")
- sitemap_urls = [
- "https://wiki.intersel.fr/doku.php?sitemap=xml",
- "https://wiki.intersel.fr/doku.php?do=sitemap",
- ]
- urls = []
- for sitemap_url in sitemap_urls:
- try:
- r = session.get(sitemap_url, headers={"User-Agent": USER_AGENT}, timeout=10)
- print(f" {sitemap_url}: status={r.status_code}")
- text = r.text or ""
- # Debug : afficher le début pour comprendre le format
- if not urls and text:
- print(f" [DEBUG] Aperçu (300 chars): {repr(text[:300])}")
- import re
- # Format XML sitemap : <loc>https://...</loc>
- if "<loc>" in text:
- found = re.findall(r"<loc>([^<]+)</loc>", text)
- urls = [u.strip() for u in found if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u]
- # Format HTML : extraire les liens doku.php?id=
- elif "doku.php" in text or "id=" in text:
- found = re.findall(r'https://wiki\.intersel\.fr/doku\.php\?id=[^"\'<>\s&]+', text)
- urls = [u.split("&")[0].rstrip(".,;:)") for u in found]
- urls = list(dict.fromkeys(urls))
- if urls:
- urls.insert(0, BASE_URL.rstrip("/"))
- urls = list(dict.fromkeys(urls))
- print(f" ✅ {len(urls)} URLs extraites")
- return urls
- except Exception as e:
- print(f" Erreur: {e}")
- print(" ⚠️ 0 URLs extraites")
- return urls
- def main():
- print("=" * 50)
- print("Test récupération URLs wiki Intersel")
- print("=" * 50)
- from src.service.auth_service import auto_login
- from src.agent_ai.auth import session
- auto_login()
- # Tests
- test_jsonrpc(session)
- test_xmlrpc(session)
- urls_index = test_index_page(session)
- urls_crawl = test_crawl_bfs(session, max_pages=150)
- urls_subindex = test_index_with_subindex(session)
- urls_sitemap = test_sitemap(session)
- # Résumé
- print("\n" + "=" * 50)
- print("RÉSUMÉ")
- print("=" * 50)
- print(f"Index simple : {len(urls_index)} URLs")
- print(f"Crawl BFS : {len(urls_crawl)} URLs")
- print(f"Index + sous-index: {len(urls_subindex)} URLs")
- print(f"Sitemap : {len(urls_sitemap)} URLs")
- best = max([urls_crawl, urls_subindex, urls_sitemap], key=len)
- if best:
- print(f"\n>>> Meilleure source: {len(best)} URLs")
- for u in best[:8]:
- print(f" - {u[:75]}...")
- if len(best) > 8:
- print(f" ... et {len(best) - 8} autres")
- if __name__ == "__main__":
- main()
|