import requests from bs4 import BeautifulSoup from langchain_community.llms import Ollama from langchain.prompts import PromptTemplate from langchain.chains import LLMChain import warnings import nltk import spacy from nltk.corpus import stopwords from sentence_transformers import SentenceTransformer, CrossEncoder import torch import pandas as pd import faiss import numpy as np from langchain.text_splitter import RecursiveCharacterTextSplitter import json from urllib.parse import urljoin, urlparse from bs4 import SoupStrainer from sklearn.metrics.pairwise import cosine_similarity import getpass #username = input("Nom d'utilisateur : ") #password = getpass.getpass("Mot de passe : ") #login_to_wiki(username, password) warnings.filterwarnings("ignore") nltk.download('punkt') nltk.download('stopwords') stop_words = set(stopwords.words('french')) nlp = spacy.load('fr_core_news_sm') session = requests.Session() def login_to_wiki(username, password): login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login" resp = session.get(login_url) soup = BeautifulSoup(resp.text, "html.parser") sectok_input = soup.find("input", {"name": "sectok"}) if not sectok_input: print("❌ Impossible de trouver le token CSRF.") return False login_data = { "u": username, "p": password, "do": "login", "sectok": sectok_input.get("value") } headers = { "User-Agent": "Mozilla/5.0", "Referer": login_url } login_resp = session.post(login_url, data=login_data, headers=headers) if "logout" in login_resp.text.lower(): print("✅ Connexion réussie.") return True else: print("❌ Échec de la connexion.") return False def extract_visible_text(html): soup = BeautifulSoup(html, "html.parser") for tag in soup(["script", "style", "header", "footer", "nav"]): tag.decompose() text = soup.get_text(separator="\n") lines = [line.strip() for line in text.splitlines() if line.strip()] return "\n".join(lines) #def clean_text(text): # doc = nlp(text) # tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""] # return ' '.join(tokens) def chunk_by_titles(text): """ Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc). Chaque chunk correspond à une section commençant par un titre. """ chunks = [] current_chunk = [] lines = text.split('\n') for line in lines: # Si la ligne commence par un signe '=' => début d'un nouveau chunk if line.strip().startswith('='): # Si on a déjà un chunk en cours, on l'ajoute à la liste if current_chunk: chunks.append('\n'.join(current_chunk).strip()) current_chunk = [] current_chunk.append(line) else: current_chunk.append(line) # Ajouter le dernier chunk if current_chunk: chunks.append('\n'.join(current_chunk).strip()) return chunks def fetch_all_pages_content(urls): url_texts = {} for url in urls: try: edit_url = url + "&do=edit" if "?" in url else url + "?do=edit" response = session.get(edit_url) if response.status_code == 200: soup = BeautifulSoup(response.text, "html.parser") textarea = soup.find("textarea", {"name": "wikitext"}) if textarea: raw_text = textarea.text url_texts[url] = raw_text else: print(f"⚠️ Pas de balise