| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190 |
- from bs4 import BeautifulSoup
- from agent_ai.auth import session
- from urllib.parse import urljoin, urlparse
- import time
- import hashlib
- import json
- import random
- from concurrent.futures import ThreadPoolExecutor, as_completed
- import os
- BASE_URL = "https://wiki.intersel.fr/"
- VISITED = set()
- USER_AGENTS = [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
- ]
- def random_headers():
- return {
- "User-Agent": random.choice(USER_AGENTS)
- }
- def hash_text(text):
- return hashlib.md5(text.encode("utf-8")).hexdigest()
- def fetch_single_page(url, old_hashes, retries=2):
- """
- Fetch the rendered HTML content inside the div with id 'dokuwiki__content'
- instead of the wikitext textarea from the edit page.
- """
- for attempt in range(retries + 1):
- try:
- headers = random_headers()
- response = session.get(url, headers=headers, timeout=10) # No ?do=edit anymore
- response.encoding = 'utf-8'
- if response.status_code != 200:
- print(f"⚠️ {url} returned {response.status_code}")
- continue
- soup = BeautifulSoup(response.text, "html.parser")
- content_div = soup.find(id="dokuwiki__content")
- if content_div:
- # Extract the inner HTML as text content (or you could get .get_text() if plain text preferred)
- raw_text = content_div.get_text(separator="\n").strip()
- current_hash = hash_text(raw_text)
- if old_hashes.get(url) != current_hash:
- print(f"🆕 Modifiée : {url}")
- return url, raw_text, current_hash
- else:
- print(f"✅ Inchangée : {url}")
- return url, None, current_hash
- else:
- print(f"⚠️ No 'dokuwiki__content' found on attempt {attempt + 1}: {url}")
- except Exception as e:
- print(f"❌ Error on attempt {attempt + 1} for {url}: {e}")
- time.sleep(random.uniform(1, 2))
- return url, None, None
- def fetch_all_pages_content(urls, old_hashes):
- url_texts = {}
- new_hashes = {}
- with ThreadPoolExecutor(max_workers=5) as executor:
- futures = [executor.submit(fetch_single_page, url, old_hashes) for url in urls]
- for future in as_completed(futures):
- url, text, current_hash = future.result()
- print(f"🧪 Result from {url} -> {'✓ text found' if text else '⛔ no content'}")
- if text:
- url_texts[url] = text
- new_hashes[url] = current_hash
- elif current_hash:
- new_hashes[url] = current_hash
-
- skipped = [url for url in urls if url not in url_texts]
- print(f"🚫 Skipped or errored URLs ({len(skipped)}):")
- for u in skipped:
- print(" -", u)
- return url_texts, new_hashes
- def is_valid_wiki_url(url):
- parsed = urlparse(url)
- return (
- parsed.netloc == "wiki.intersel.fr"
- and not url.endswith((".jpg", ".png", ".pdf", ".docx", ".xlsx"))
- and "#" not in url
- )
- def crawl_wiki(start_url, existing_urls, max_pages=20):
- visited = set(existing_urls)
- to_visit = [start_url]
- while to_visit and len(visited) < max_pages:
- current_url = to_visit.pop(0)
- if current_url in visited:
- continue
- print(f"🔎 Visiting: {current_url}")
- visited.add(current_url)
- try:
- headers = random_headers()
- response = session.get(current_url, headers=headers, timeout=10)
- if response.status_code != 200:
- continue
- soup = BeautifulSoup(response.text, "html.parser")
- for link in soup.find_all("a", href=True):
- absolute_url = urljoin(current_url, link["href"])
- if is_valid_wiki_url(absolute_url) and absolute_url not in visited and absolute_url not in to_visit:
- to_visit.append(absolute_url)
- time.sleep(random.uniform(1, 2))
- except Exception as e:
- print(f"❌ Error with {current_url}: {e}")
- return list(visited)
- def scrape_pages():
- start_url = "https://wiki.intersel.fr/"
- urls_path = os.path.join("data", "urls.json")
- raw_pages_path = "data/raw_pages.json"
- hashes_path = "data/page_hashes.json"
- # Charger anciens hashs
- try:
- with open(hashes_path, "r", encoding="utf-8") as f:
- old_hashes = json.load(f)
- except FileNotFoundError:
- old_hashes = {}
- # Charger anciennes URLs
- try:
- with open(urls_path, "r", encoding="utf-8") as f:
- urls = json.load(f)
- print("✅ URLs chargées depuis urls.json")
- except FileNotFoundError:
- print("🚀 Crawling initial du wiki")
- urls = []
- # Nouveau crawl
- crawled_urls = crawl_wiki(start_url, urls, max_pages=20)
- all_urls = sorted(set(urls + crawled_urls))
- print(f"🌐 Total URLs à vérifier : {len(all_urls)}")
- # Scraping des pages
- pages, new_hashes = fetch_all_pages_content(all_urls, old_hashes)
- # Charger anciens raw_pages
- try:
- with open(raw_pages_path, "r", encoding="utf-8") as f:
- old_raw_pages = {p["url"]: p["text"] for p in json.load(f)}
- except FileNotFoundError:
- old_raw_pages = {}
- # Mettre à jour uniquement les pages modifiées
- for url, text in pages.items():
- old_raw_pages[url] = text
- # Enlever les URLs qui n’existent plus (optionnel)
- cleaned_raw_pages = {url: text for url, text in old_raw_pages.items() if url in all_urls}
- # Sauvegarder le nouveau contenu
- with open(raw_pages_path, "w", encoding="utf-8") as f:
- json.dump(
- [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()],
- f, ensure_ascii=False, indent=2
- )
- with open(hashes_path, "w", encoding="utf-8") as f:
- json.dump(new_hashes, f, ensure_ascii=False, indent=2)
- with open(urls_path, "w", encoding="utf-8") as f:
- json.dump(all_urls, f, ensure_ascii=False, indent=2)
- print("📄 Pages modifiées sauvegardées")
- print("📄 Contenu brut mis à jour enregistré dans 'raw_pages.json'")
- print("🔗 URLs enregistrées dans 'data/urls.json'")
- print("🔐 Hashes des pages enregistrés dans 'page_hashes.json'")
- return [{"url": url, "text": text} for url, text in cleaned_raw_pages.items()]
|