import requests from bs4 import BeautifulSoup from langchain_community.llms import Ollama from langchain.prompts import PromptTemplate from langchain.chains import LLMChain import warnings import nltk import spacy from nltk.corpus import stopwords from sentence_transformers import SentenceTransformer, CrossEncoder import torch import pandas as pd import faiss import numpy as np from langchain.text_splitter import RecursiveCharacterTextSplitter import json from urllib.parse import urljoin, urlparse from bs4 import SoupStrainer warnings.filterwarnings("ignore") nltk.download('punkt') nltk.download('stopwords') stop_words = set(stopwords.words('french')) nlp = spacy.load('fr_core_news_sm') session = requests.Session() def login_to_wiki(username, password): login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login" resp = session.get(login_url) soup = BeautifulSoup(resp.text, "html.parser") sectok_input = soup.find("input", {"name": "sectok"}) if not sectok_input: print("❌ Impossible de trouver le token CSRF.") return False login_data = { "u": username, "p": password, "do": "login", "sectok": sectok_input.get("value") } headers = { "User-Agent": "Mozilla/5.0", "Referer": login_url } login_resp = session.post(login_url, data=login_data, headers=headers) if "logout" in login_resp.text.lower(): print("✅ Connexion réussie.") return True else: print("❌ Échec de la connexion.") return False def extract_visible_text(html): soup = BeautifulSoup(html, "html.parser") for tag in soup(["script", "style", "header", "footer", "nav"]): tag.decompose() text = soup.get_text(separator="\n") lines = [line.strip() for line in text.splitlines() if line.strip()] return "\n".join(lines) def clean_text(text): doc = nlp(text) tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""] return ' '.join(tokens) def chunk_by_titles(text): """ Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc). Chaque chunk correspond à une section commençant par un titre. """ chunks = [] current_chunk = [] lines = text.split('\n') for line in lines: # Si la ligne commence par un signe '=' => début d'un nouveau chunk if line.strip().startswith('='): # Si on a déjà un chunk en cours, on l'ajoute à la liste if current_chunk: chunks.append('\n'.join(current_chunk).strip()) current_chunk = [] current_chunk.append(line) else: current_chunk.append(line) # Ajouter le dernier chunk if current_chunk: chunks.append('\n'.join(current_chunk).strip()) return chunks def semantic_chunking(raw_text, max_tokens=200): sentences = nltk.sent_tokenize(raw_text, language='french') chunks = [] current_chunk = "" current_len = 0 for sentence in sentences: sentence_len = len(sentence.split()) if current_len + sentence_len > max_tokens: chunks.append(current_chunk.strip()) current_chunk = sentence + " " current_len = sentence_len else: current_chunk += sentence + " " current_len += sentence_len if current_chunk: chunks.append(current_chunk.strip()) return chunks def fetch_all_pages_content(urls): url_texts = {} for url in urls: try: response = session.get(url) if response.status_code == 200: page_text = extract_visible_text(response.text) url_texts[url] = page_text else: print(f"⚠️ Erreur d'accès à {url} : code {response.status_code}") except Exception as e: print(f"❌ Exception pour {url}: {e}") return url_texts def build_faiss_index(embeddings): dimension = embeddings.shape[1] index = faiss.IndexFlatIP(dimension) faiss.normalize_L2(embeddings) index.add(embeddings) return index def search_faiss(index, query_embedding, top_k=10): faiss.normalize_L2(query_embedding) distances, indices = index.search(query_embedding, top_k) return distances[0], indices[0] def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=20, rerank_top_k=5): question_embedding = model.encode([question], convert_to_tensor=False) question_embedding = np.array(question_embedding).astype('float32') distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k) candidates = [] for idx, dist in zip(indices, distances): candidates.append({ "chunk": chunks[idx], "url": chunk_urls[idx], "index": idx, "faiss_score": float(dist) }) # Re-ranking avec CrossEncoder pairs = [[question, c["chunk"]] for c in candidates] rerank_scores = cross_encoder.predict(pairs) for i in range(len(candidates)): candidates[i]["rerank_score"] = float(rerank_scores[i]) # Tri final selon score CrossEncoder reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k] print("\n🔍 Résultats re-rankés :") df = pd.DataFrame([{ "Index": r["index"], "URL": r["url"], "Score CrossEncoder": round(r["rerank_score"], 4), "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"] } for r in reranked]) print(df) context = "\n\n".join([r["chunk"] for r in reranked]) result = qa_chain.run(context=context, question=question) print(f"\n🤖 Réponse :\n{result}\n") if __name__ == "__main__": if login_to_wiki("nadine.elleuch", "witpowgeu!75"): wiki_urls = [ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe", "https://wiki.intersel.fr/doku.php?id=intersel:presentation", "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires", "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start" ] url_texts = fetch_all_pages_content(wiki_urls) all_chunks = [] chunk_urls = [] for url, text in url_texts.items(): print(f"\n--- Traitement de la page {url} ---") chunks = chunk_by_titles(text) print(f"Nombre de chunks par titres : {len(chunks)}") # Nettoyage + découpage en sous-chunks si trop long cleaned_chunks = [] for chunk in chunks: cleaned = clean_text(chunk) if len(cleaned.split()) > 200: subchunks = semantic_chunking(cleaned, max_tokens=200) cleaned_chunks.extend(subchunks) else: cleaned_chunks.append(cleaned) for i, c in enumerate(cleaned_chunks): print(f"Chunk {i+1} ({len(c.split())} mots): {c[:100]}{'...' if len(c) > 100 else ''}") all_chunks.append(c) chunk_urls.append(url) import json chunk_data = [ {"url": url, "chunk_index": i+1, "chunk": chunk} for i, (url, chunk) in enumerate(zip(chunk_urls, all_chunks)) ] with open("chunks_pretty.json", "w", encoding="utf-8") as f: json.dump(chunk_data, f, ensure_ascii=False, indent=2) print("📁 Chunks enregistrés dans 'chunks_pretty.json'") model = SentenceTransformer('all-mpnet-base-v2') cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2') print("\n✅ Génération des embeddings des chunks...") chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True) chunk_embeddings = np.array(chunk_embeddings).astype('float32') print("\n✅ Construction de l'index FAISS...") index = build_faiss_index(chunk_embeddings) print("✅ Index FAISS construit avec succès.") prompt_template = PromptTemplate( input_variables=["context", "question"], template=""" Tu es un assistant expert du contenu du wiki interne d'Intersel. Ta tâche est de répondre à la question en utilisant uniquement les informations pertinentes du contexte fourni. Si l'information existe, donne une réponse claire, concise et précise. CONTEXTE : {context} QUESTION : {question} RÉPONSE : """ ) llm = Ollama(model="llama3") qa_chain = LLMChain(llm=llm, prompt=prompt_template) while True: question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ") if question.lower() in ["exit", "quit"]: break ask_question_with_reranking(question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain)