test_fetch_urls.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. """
  2. Test de récupération d'URLs du wiki par différentes méthodes.
  3. Depuis agent_docker : py .\test-to-delete\test_fetch_urls.py
  4. """
  5. import sys
  6. import os
  7. agent_docker_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
  8. if agent_docker_root not in sys.path:
  9. sys.path.insert(0, agent_docker_root)
  10. os.chdir(agent_docker_root)
  11. from dotenv import load_dotenv
  12. load_dotenv()
  13. import time
  14. import requests
  15. from bs4 import BeautifulSoup
  16. from urllib.parse import urljoin, urlparse
  17. BASE_URL = "https://wiki.intersel.fr/"
  18. JSONRPC_URL = "https://wiki.intersel.fr/lib/exe/jsonrpc.php"
  19. XMLRPC_URL = "https://wiki.intersel.fr/lib/exe/xmlrpc.php"
  20. INDEX_URL = "https://wiki.intersel.fr/doku.php?do=index"
  21. USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/115.0.0"
  22. def is_valid_page_url(url):
  23. """URL de page de contenu (id=, pas do=, pas idx=)."""
  24. parsed = urlparse(url)
  25. return (
  26. "wiki.intersel.fr" in url
  27. and "do=" not in parsed.query
  28. and "idx=" not in parsed.query
  29. and "id=" in parsed.query
  30. and "#" not in url
  31. )
  32. def test_jsonrpc(session):
  33. """Test JSON-RPC core.getAPIVersion."""
  34. print("\n--- 1. JSON-RPC ---")
  35. try:
  36. r = session.post(
  37. f"{JSONRPC_URL}/core.getAPIVersion",
  38. json={},
  39. headers={"Content-Type": "application/json", "User-Agent": USER_AGENT},
  40. timeout=5,
  41. )
  42. data = r.json() if r.text else {}
  43. print(f" Status: {r.status_code}")
  44. print(f" Response: {data}")
  45. if r.status_code == 200 and "result" in data:
  46. print(" ✅ JSON-RPC OK")
  47. return True
  48. print(" ❌ JSON-RPC non disponible")
  49. return False
  50. except Exception as e:
  51. print(f" ❌ Erreur: {e}")
  52. return False
  53. def test_xmlrpc(session):
  54. """Test XML-RPC (alternative à JSON-RPC)."""
  55. print("\n--- 2. XML-RPC ---")
  56. try:
  57. xml_body = """<?xml version="1.0"?>
  58. <methodCall>
  59. <methodName>wiki.getVersion</methodName>
  60. <params></params>
  61. </methodCall>"""
  62. r = session.post(
  63. XMLRPC_URL,
  64. data=xml_body,
  65. headers={"Content-Type": "text/xml", "User-Agent": USER_AGENT},
  66. timeout=5,
  67. )
  68. print(f" Status: {r.status_code}")
  69. print(f" Response (100 chars): {r.text[:100] if r.text else 'vide'}...")
  70. if r.status_code == 200 and "methodResponse" in (r.text or ""):
  71. print(" ✅ XML-RPC répond")
  72. return True
  73. print(" ❌ XML-RPC non disponible")
  74. return False
  75. except Exception as e:
  76. print(f" ❌ Erreur: {e}")
  77. return False
  78. def test_index_page(session):
  79. """Récupère les URLs depuis la page index (sans sous-index)."""
  80. print("\n--- 3. Index HTML (do=index) ---")
  81. urls = set()
  82. try:
  83. r = session.get(INDEX_URL, headers={"User-Agent": USER_AGENT}, timeout=10)
  84. print(f" Status: {r.status_code}")
  85. if r.status_code != 200:
  86. return []
  87. soup = BeautifulSoup(r.text, "html.parser")
  88. for link in soup.find_all("a", href=True):
  89. abs_url = urljoin(INDEX_URL, link["href"])
  90. if is_valid_page_url(abs_url):
  91. urls.add(abs_url)
  92. urls.add(BASE_URL.rstrip("/"))
  93. print(f" ✅ {len(urls)} URLs (pages id= + homepage)")
  94. return sorted(urls)
  95. except Exception as e:
  96. print(f" ❌ Erreur: {e}")
  97. return []
  98. def test_crawl_bfs(session, max_pages=100):
  99. """Crawl BFS depuis la homepage : suit tous les liens internes du wiki."""
  100. print("\n--- 4. Crawl BFS (depuis homepage) ---")
  101. base = BASE_URL.rstrip("/")
  102. to_visit = [base, f"{base}/doku.php?id=start"]
  103. visited = set()
  104. content_urls = set()
  105. while to_visit and len(visited) < max_pages:
  106. url = to_visit.pop(0)
  107. if url in visited:
  108. continue
  109. visited.add(url)
  110. try:
  111. r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
  112. if r.status_code != 200:
  113. continue
  114. soup = BeautifulSoup(r.text, "html.parser")
  115. for link in soup.find_all("a", href=True):
  116. abs_url = urljoin(url, link["href"])
  117. if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
  118. continue
  119. parsed = urlparse(abs_url)
  120. if "do=" in parsed.query or "idx=" in parsed.query:
  121. continue
  122. if "id=" in parsed.query or abs_url.rstrip("/") == base:
  123. clean = abs_url.split("&")[0].split("#")[0]
  124. if clean not in visited and clean not in to_visit:
  125. to_visit.append(clean)
  126. content_urls.add(clean)
  127. content_urls.add(url.split("&")[0].split("#")[0])
  128. time.sleep(0.2)
  129. except Exception as e:
  130. print(f" ⚠️ {url[:50]}: {e}")
  131. content_urls.add(base)
  132. def norm(u):
  133. u = u.split("#")[0].split("&")[0].rstrip("/")
  134. if u.endswith("/doku.php") or u == base + "/":
  135. return base
  136. return u
  137. urls = sorted(set(norm(u) for u in content_urls if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u))
  138. print(f" ✅ {len(urls)} URLs (après {len(visited)} pages visitées)")
  139. return urls
  140. def test_index_with_subindex(session):
  141. """Parcourt index + sous-index (do=index&idx=...)."""
  142. print("\n--- 5. Index + sous-index (parcours récursif) ---")
  143. content_urls = set()
  144. sub_indexes = set()
  145. to_visit = [INDEX_URL]
  146. visited = set()
  147. while to_visit:
  148. url = to_visit.pop(0)
  149. if url in visited:
  150. continue
  151. visited.add(url)
  152. try:
  153. r = session.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
  154. if r.status_code != 200:
  155. continue
  156. soup = BeautifulSoup(r.text, "html.parser")
  157. for link in soup.find_all("a", href=True):
  158. abs_url = urljoin(url, link["href"])
  159. if "wiki.intersel.fr" not in abs_url or "#" in abs_url:
  160. continue
  161. parsed = urlparse(abs_url)
  162. if "do=index" in parsed.query:
  163. sub_indexes.add(abs_url)
  164. if abs_url not in visited and abs_url not in to_visit:
  165. to_visit.append(abs_url)
  166. elif is_valid_page_url(abs_url):
  167. content_urls.add(abs_url)
  168. except Exception as e:
  169. print(f" ⚠️ {url}: {e}")
  170. content_urls.add(BASE_URL.rstrip("/"))
  171. print(f" ✅ {len(content_urls)} URLs (après {len(visited)} pages d'index)")
  172. return sorted(content_urls)
  173. def test_sitemap(session):
  174. """Test et parse le sitemap (plusieurs URLs possibles)."""
  175. print("\n--- 6. Sitemap ---")
  176. sitemap_urls = [
  177. "https://wiki.intersel.fr/doku.php?sitemap=xml",
  178. "https://wiki.intersel.fr/doku.php?do=sitemap",
  179. ]
  180. urls = []
  181. for sitemap_url in sitemap_urls:
  182. try:
  183. r = session.get(sitemap_url, headers={"User-Agent": USER_AGENT}, timeout=10)
  184. print(f" {sitemap_url}: status={r.status_code}")
  185. text = r.text or ""
  186. # Debug : afficher le début pour comprendre le format
  187. if not urls and text:
  188. print(f" [DEBUG] Aperçu (300 chars): {repr(text[:300])}")
  189. import re
  190. # Format XML sitemap : <loc>https://...</loc>
  191. if "<loc>" in text:
  192. found = re.findall(r"<loc>([^<]+)</loc>", text)
  193. urls = [u.strip() for u in found if "wiki.intersel.fr" in u and "do=" not in u and "idx=" not in u]
  194. # Format HTML : extraire les liens doku.php?id=
  195. elif "doku.php" in text or "id=" in text:
  196. found = re.findall(r'https://wiki\.intersel\.fr/doku\.php\?id=[^"\'<>\s&]+', text)
  197. urls = [u.split("&")[0].rstrip(".,;:)") for u in found]
  198. urls = list(dict.fromkeys(urls))
  199. if urls:
  200. urls.insert(0, BASE_URL.rstrip("/"))
  201. urls = list(dict.fromkeys(urls))
  202. print(f" ✅ {len(urls)} URLs extraites")
  203. return urls
  204. except Exception as e:
  205. print(f" Erreur: {e}")
  206. print(" ⚠️ 0 URLs extraites")
  207. return urls
  208. def main():
  209. print("=" * 50)
  210. print("Test récupération URLs wiki Intersel")
  211. print("=" * 50)
  212. from src.service.auth_service import auto_login
  213. from src.agent_ai.auth import session
  214. auto_login()
  215. # Tests
  216. test_jsonrpc(session)
  217. test_xmlrpc(session)
  218. urls_index = test_index_page(session)
  219. urls_crawl = test_crawl_bfs(session, max_pages=150)
  220. urls_subindex = test_index_with_subindex(session)
  221. urls_sitemap = test_sitemap(session)
  222. # Résumé
  223. print("\n" + "=" * 50)
  224. print("RÉSUMÉ")
  225. print("=" * 50)
  226. print(f"Index simple : {len(urls_index)} URLs")
  227. print(f"Crawl BFS : {len(urls_crawl)} URLs")
  228. print(f"Index + sous-index: {len(urls_subindex)} URLs")
  229. print(f"Sitemap : {len(urls_sitemap)} URLs")
  230. best = max([urls_crawl, urls_subindex, urls_sitemap], key=len)
  231. if best:
  232. print(f"\n>>> Meilleure source: {len(best)} URLs")
  233. for u in best[:8]:
  234. print(f" - {u[:75]}...")
  235. if len(best) > 8:
  236. print(f" ... et {len(best) - 8} autres")
  237. if __name__ == "__main__":
  238. main()