|
|
@@ -170,6 +170,8 @@ def _scrape_via_jsonrpc(session: requests.Session | None = None) -> tuple[list[d
|
|
|
# Extraire les cookies de la session pour les passer aux threads
|
|
|
session_cookies = dict(sess.cookies)
|
|
|
|
|
|
+ total = len(all_page_ids)
|
|
|
+ done = 0
|
|
|
with ThreadPoolExecutor(max_workers=3) as executor:
|
|
|
futures = {
|
|
|
executor.submit(_fetch_page, pid, old_hashes, session_cookies): pid
|
|
|
@@ -178,6 +180,8 @@ def _scrape_via_jsonrpc(session: requests.Session | None = None) -> tuple[list[d
|
|
|
for future in as_completed(futures):
|
|
|
page_id, text, h = future.result()
|
|
|
url = _page_id_to_url(page_id)
|
|
|
+ done += 1
|
|
|
+ print(f" [{done}/{total}] {url}")
|
|
|
if text is not None:
|
|
|
url_to_text[url] = text
|
|
|
if h is not None:
|
|
|
@@ -404,7 +408,9 @@ def _scrape_via_html() -> tuple[list[dict], dict]:
|
|
|
|
|
|
url_to_text = {}
|
|
|
new_hashes = {}
|
|
|
- for url in all_urls:
|
|
|
+ total = len(all_urls)
|
|
|
+ for i, url in enumerate(all_urls, 1):
|
|
|
+ print(f" [{i}/{total}] {url}")
|
|
|
_, text, h = fetch_page(url, old_hashes)
|
|
|
if text:
|
|
|
url_to_text[url] = text
|