|
|
@@ -0,0 +1,267 @@
|
|
|
+import requests
|
|
|
+from bs4 import BeautifulSoup
|
|
|
+from langchain_community.llms import Ollama
|
|
|
+from langchain.prompts import PromptTemplate
|
|
|
+from langchain.chains import LLMChain
|
|
|
+import warnings
|
|
|
+import nltk
|
|
|
+import spacy
|
|
|
+from nltk.corpus import stopwords
|
|
|
+from sentence_transformers import SentenceTransformer, CrossEncoder
|
|
|
+import torch
|
|
|
+import pandas as pd
|
|
|
+import faiss
|
|
|
+import numpy as np
|
|
|
+from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
+import json
|
|
|
+from urllib.parse import urljoin, urlparse
|
|
|
+from bs4 import SoupStrainer
|
|
|
+
|
|
|
+warnings.filterwarnings("ignore")
|
|
|
+
|
|
|
+nltk.download('punkt')
|
|
|
+nltk.download('stopwords')
|
|
|
+
|
|
|
+stop_words = set(stopwords.words('french'))
|
|
|
+nlp = spacy.load('fr_core_news_sm')
|
|
|
+
|
|
|
+session = requests.Session()
|
|
|
+
|
|
|
+def login_to_wiki(username, password):
|
|
|
+ login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login"
|
|
|
+ resp = session.get(login_url)
|
|
|
+ soup = BeautifulSoup(resp.text, "html.parser")
|
|
|
+ sectok_input = soup.find("input", {"name": "sectok"})
|
|
|
+
|
|
|
+ if not sectok_input:
|
|
|
+ print("❌ Impossible de trouver le token CSRF.")
|
|
|
+ return False
|
|
|
+
|
|
|
+ login_data = {
|
|
|
+ "u": username,
|
|
|
+ "p": password,
|
|
|
+ "do": "login",
|
|
|
+ "sectok": sectok_input.get("value")
|
|
|
+ }
|
|
|
+
|
|
|
+ headers = {
|
|
|
+ "User-Agent": "Mozilla/5.0",
|
|
|
+ "Referer": login_url
|
|
|
+ }
|
|
|
+
|
|
|
+ login_resp = session.post(login_url, data=login_data, headers=headers)
|
|
|
+
|
|
|
+ if "logout" in login_resp.text.lower():
|
|
|
+ print("✅ Connexion réussie.")
|
|
|
+ return True
|
|
|
+ else:
|
|
|
+ print("❌ Échec de la connexion.")
|
|
|
+ return False
|
|
|
+
|
|
|
+def extract_visible_text(html):
|
|
|
+ soup = BeautifulSoup(html, "html.parser")
|
|
|
+ for tag in soup(["script", "style", "header", "footer", "nav"]):
|
|
|
+ tag.decompose()
|
|
|
+ text = soup.get_text(separator="\n")
|
|
|
+ lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
|
+ return "\n".join(lines)
|
|
|
+
|
|
|
+def clean_text(text):
|
|
|
+ doc = nlp(text)
|
|
|
+ tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
|
|
|
+ return ' '.join(tokens)
|
|
|
+
|
|
|
+def chunk_by_titles(text):
|
|
|
+ """
|
|
|
+ Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc).
|
|
|
+ Chaque chunk correspond à une section commençant par un titre.
|
|
|
+ """
|
|
|
+ chunks = []
|
|
|
+ current_chunk = []
|
|
|
+
|
|
|
+ lines = text.split('\n')
|
|
|
+ for line in lines:
|
|
|
+ # Si la ligne commence par un signe '=' => début d'un nouveau chunk
|
|
|
+ if line.strip().startswith('='):
|
|
|
+ # Si on a déjà un chunk en cours, on l'ajoute à la liste
|
|
|
+ if current_chunk:
|
|
|
+ chunks.append('\n'.join(current_chunk).strip())
|
|
|
+ current_chunk = []
|
|
|
+ current_chunk.append(line)
|
|
|
+ else:
|
|
|
+ current_chunk.append(line)
|
|
|
+
|
|
|
+ # Ajouter le dernier chunk
|
|
|
+ if current_chunk:
|
|
|
+ chunks.append('\n'.join(current_chunk).strip())
|
|
|
+
|
|
|
+ return chunks
|
|
|
+
|
|
|
+def semantic_chunking(raw_text, max_tokens=200):
|
|
|
+ sentences = nltk.sent_tokenize(raw_text, language='french')
|
|
|
+ chunks = []
|
|
|
+ current_chunk = ""
|
|
|
+ current_len = 0
|
|
|
+
|
|
|
+ for sentence in sentences:
|
|
|
+ sentence_len = len(sentence.split())
|
|
|
+ if current_len + sentence_len > max_tokens:
|
|
|
+ chunks.append(current_chunk.strip())
|
|
|
+ current_chunk = sentence + " "
|
|
|
+ current_len = sentence_len
|
|
|
+ else:
|
|
|
+ current_chunk += sentence + " "
|
|
|
+ current_len += sentence_len
|
|
|
+
|
|
|
+ if current_chunk:
|
|
|
+ chunks.append(current_chunk.strip())
|
|
|
+
|
|
|
+ return chunks
|
|
|
+
|
|
|
+def fetch_all_pages_content(urls):
|
|
|
+ url_texts = {}
|
|
|
+ for url in urls:
|
|
|
+ try:
|
|
|
+ response = session.get(url)
|
|
|
+ if response.status_code == 200:
|
|
|
+ page_text = extract_visible_text(response.text)
|
|
|
+ url_texts[url] = page_text
|
|
|
+ else:
|
|
|
+ print(f"⚠️ Erreur d'accès à {url} : code {response.status_code}")
|
|
|
+ except Exception as e:
|
|
|
+ print(f"❌ Exception pour {url}: {e}")
|
|
|
+ return url_texts
|
|
|
+
|
|
|
+def build_faiss_index(embeddings):
|
|
|
+ dimension = embeddings.shape[1]
|
|
|
+ index = faiss.IndexFlatIP(dimension)
|
|
|
+ faiss.normalize_L2(embeddings)
|
|
|
+ index.add(embeddings)
|
|
|
+ return index
|
|
|
+
|
|
|
+def search_faiss(index, query_embedding, top_k=10):
|
|
|
+ faiss.normalize_L2(query_embedding)
|
|
|
+ distances, indices = index.search(query_embedding, top_k)
|
|
|
+ return distances[0], indices[0]
|
|
|
+
|
|
|
+def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=20, rerank_top_k=5):
|
|
|
+ question_embedding = model.encode([question], convert_to_tensor=False)
|
|
|
+ question_embedding = np.array(question_embedding).astype('float32')
|
|
|
+
|
|
|
+ distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
|
|
|
+
|
|
|
+ candidates = []
|
|
|
+ for idx, dist in zip(indices, distances):
|
|
|
+ candidates.append({
|
|
|
+ "chunk": chunks[idx],
|
|
|
+ "url": chunk_urls[idx],
|
|
|
+ "index": idx,
|
|
|
+ "faiss_score": float(dist)
|
|
|
+ })
|
|
|
+
|
|
|
+ # Re-ranking avec CrossEncoder
|
|
|
+ pairs = [[question, c["chunk"]] for c in candidates]
|
|
|
+ rerank_scores = cross_encoder.predict(pairs)
|
|
|
+
|
|
|
+ for i in range(len(candidates)):
|
|
|
+ candidates[i]["rerank_score"] = float(rerank_scores[i])
|
|
|
+
|
|
|
+ # Tri final selon score CrossEncoder
|
|
|
+ reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
|
|
|
+
|
|
|
+ print("\n🔍 Résultats re-rankés :")
|
|
|
+ df = pd.DataFrame([{
|
|
|
+ "Index": r["index"],
|
|
|
+ "URL": r["url"],
|
|
|
+ "Score CrossEncoder": round(r["rerank_score"], 4),
|
|
|
+ "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
|
|
|
+ } for r in reranked])
|
|
|
+ print(df)
|
|
|
+
|
|
|
+ context = "\n\n".join([r["chunk"] for r in reranked])
|
|
|
+ result = qa_chain.run(context=context, question=question)
|
|
|
+ print(f"\n🤖 Réponse :\n{result}\n")
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ if login_to_wiki("nadine.elleuch", "witpowgeu!75"):
|
|
|
+ wiki_urls = [
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
|
|
|
+ "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
|
|
|
+ ]
|
|
|
+
|
|
|
+ url_texts = fetch_all_pages_content(wiki_urls)
|
|
|
+
|
|
|
+ all_chunks = []
|
|
|
+ chunk_urls = []
|
|
|
+
|
|
|
+ for url, text in url_texts.items():
|
|
|
+ print(f"\n--- Traitement de la page {url} ---")
|
|
|
+ chunks = chunk_by_titles(text)
|
|
|
+ print(f"Nombre de chunks par titres : {len(chunks)}")
|
|
|
+
|
|
|
+ # Nettoyage + découpage en sous-chunks si trop long
|
|
|
+ cleaned_chunks = []
|
|
|
+ for chunk in chunks:
|
|
|
+ cleaned = clean_text(chunk)
|
|
|
+ if len(cleaned.split()) > 200:
|
|
|
+ subchunks = semantic_chunking(cleaned, max_tokens=200)
|
|
|
+ cleaned_chunks.extend(subchunks)
|
|
|
+ else:
|
|
|
+ cleaned_chunks.append(cleaned)
|
|
|
+
|
|
|
+ for i, c in enumerate(cleaned_chunks):
|
|
|
+ print(f"Chunk {i+1} ({len(c.split())} mots): {c[:100]}{'...' if len(c) > 100 else ''}")
|
|
|
+ all_chunks.append(c)
|
|
|
+ chunk_urls.append(url)
|
|
|
+ import json
|
|
|
+
|
|
|
+ chunk_data = [
|
|
|
+ {"url": url, "chunk_index": i+1, "chunk": chunk}
|
|
|
+ for i, (url, chunk) in enumerate(zip(chunk_urls, all_chunks))
|
|
|
+ ]
|
|
|
+
|
|
|
+ with open("chunks_pretty.json", "w", encoding="utf-8") as f:
|
|
|
+ json.dump(chunk_data, f, ensure_ascii=False, indent=2)
|
|
|
+
|
|
|
+ print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
|
|
|
+
|
|
|
+ model = SentenceTransformer('all-mpnet-base-v2')
|
|
|
+ cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
|
|
|
+
|
|
|
+ print("\n✅ Génération des embeddings des chunks...")
|
|
|
+ chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
|
|
|
+ chunk_embeddings = np.array(chunk_embeddings).astype('float32')
|
|
|
+
|
|
|
+ print("\n✅ Construction de l'index FAISS...")
|
|
|
+ index = build_faiss_index(chunk_embeddings)
|
|
|
+ print("✅ Index FAISS construit avec succès.")
|
|
|
+
|
|
|
+ prompt_template = PromptTemplate(
|
|
|
+ input_variables=["context", "question"],
|
|
|
+ template="""
|
|
|
+Tu es un assistant expert du contenu du wiki interne d'Intersel. Ta tâche est de répondre à la question en utilisant uniquement les informations pertinentes du contexte fourni.
|
|
|
+
|
|
|
+Si l'information existe, donne une réponse claire, concise et précise.
|
|
|
+
|
|
|
+CONTEXTE :
|
|
|
+{context}
|
|
|
+
|
|
|
+QUESTION :
|
|
|
+{question}
|
|
|
+
|
|
|
+RÉPONSE :
|
|
|
+"""
|
|
|
+ )
|
|
|
+
|
|
|
+ llm = Ollama(model="llama3")
|
|
|
+ qa_chain = LLMChain(llm=llm, prompt=prompt_template)
|
|
|
+
|
|
|
+ while True:
|
|
|
+ question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
|
|
|
+ if question.lower() in ["exit", "quit"]:
|
|
|
+ break
|
|
|
+ ask_question_with_reranking(question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain)
|
|
|
+
|