Parcourir la source

Votre message

augustin.podvin il y a 8 mois
Parent
commit
b6ac977b39
1 fichiers modifiés avec 7 ajouts et 1 suppressions
  1. 7 1
      src/agent_ai/scraping.py

+ 7 - 1
src/agent_ai/scraping.py

@@ -170,6 +170,8 @@ def _scrape_via_jsonrpc(session: requests.Session | None = None) -> tuple[list[d
     # Extraire les cookies de la session pour les passer aux threads
     session_cookies = dict(sess.cookies)
 
+    total = len(all_page_ids)
+    done = 0
     with ThreadPoolExecutor(max_workers=3) as executor:
         futures = {
             executor.submit(_fetch_page, pid, old_hashes, session_cookies): pid
@@ -178,6 +180,8 @@ def _scrape_via_jsonrpc(session: requests.Session | None = None) -> tuple[list[d
         for future in as_completed(futures):
             page_id, text, h = future.result()
             url = _page_id_to_url(page_id)
+            done += 1
+            print(f"  [{done}/{total}] {url}")
             if text is not None:
                 url_to_text[url] = text
             if h is not None:
@@ -404,7 +408,9 @@ def _scrape_via_html() -> tuple[list[dict], dict]:
 
     url_to_text = {}
     new_hashes = {}
-    for url in all_urls:
+    total = len(all_urls)
+    for i, url in enumerate(all_urls, 1):
+        print(f"  [{i}/{total}] {url}")
         _, text, h = fetch_page(url, old_hashes)
         if text:
             url_to_text[url] = text