wiki.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. import requests
  2. from bs4 import BeautifulSoup
  3. from langchain_community.llms import Ollama
  4. from langchain.prompts import PromptTemplate
  5. from langchain.chains import LLMChain
  6. import warnings
  7. import nltk
  8. import spacy
  9. from nltk.corpus import stopwords
  10. from sentence_transformers import SentenceTransformer, CrossEncoder
  11. import torch
  12. import pandas as pd
  13. import faiss
  14. import numpy as np
  15. from langchain.text_splitter import RecursiveCharacterTextSplitter
  16. import json
  17. from urllib.parse import urljoin, urlparse
  18. from bs4 import SoupStrainer
  19. from sklearn.metrics.pairwise import cosine_similarity
  20. import getpass
  21. #username = input("Nom d'utilisateur : ")
  22. #password = getpass.getpass("Mot de passe : ")
  23. #login_to_wiki(username, password)
  24. warnings.filterwarnings("ignore")
  25. nltk.download('punkt')
  26. nltk.download('stopwords')
  27. stop_words = set(stopwords.words('french'))
  28. nlp = spacy.load('fr_core_news_sm')
  29. session = requests.Session()
  30. def login_to_wiki(username, password):
  31. login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login"
  32. resp = session.get(login_url)
  33. soup = BeautifulSoup(resp.text, "html.parser")
  34. sectok_input = soup.find("input", {"name": "sectok"})
  35. if not sectok_input:
  36. print("❌ Impossible de trouver le token CSRF.")
  37. return False
  38. login_data = {
  39. "u": username,
  40. "p": password,
  41. "do": "login",
  42. "sectok": sectok_input.get("value")
  43. }
  44. headers = {
  45. "User-Agent": "Mozilla/5.0",
  46. "Referer": login_url
  47. }
  48. login_resp = session.post(login_url, data=login_data, headers=headers)
  49. if "logout" in login_resp.text.lower():
  50. print("✅ Connexion réussie.")
  51. return True
  52. else:
  53. print("❌ Échec de la connexion.")
  54. return False
  55. def extract_visible_text(html):
  56. soup = BeautifulSoup(html, "html.parser")
  57. for tag in soup(["script", "style", "header", "footer", "nav"]):
  58. tag.decompose()
  59. text = soup.get_text(separator="\n")
  60. lines = [line.strip() for line in text.splitlines() if line.strip()]
  61. return "\n".join(lines)
  62. #def clean_text(text):
  63. # doc = nlp(text)
  64. # tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
  65. # return ' '.join(tokens)
  66. def chunk_by_titles(text):
  67. """
  68. Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc).
  69. Chaque chunk correspond à une section commençant par un titre.
  70. """
  71. chunks = []
  72. current_chunk = []
  73. lines = text.split('\n')
  74. for line in lines:
  75. # Si la ligne commence par un signe '=' => début d'un nouveau chunk
  76. if line.strip().startswith('='):
  77. # Si on a déjà un chunk en cours, on l'ajoute à la liste
  78. if current_chunk:
  79. chunks.append('\n'.join(current_chunk).strip())
  80. current_chunk = []
  81. current_chunk.append(line)
  82. else:
  83. current_chunk.append(line)
  84. # Ajouter le dernier chunk
  85. if current_chunk:
  86. chunks.append('\n'.join(current_chunk).strip())
  87. return chunks
  88. def fetch_all_pages_content(urls):
  89. url_texts = {}
  90. for url in urls:
  91. try:
  92. edit_url = url + "&do=edit" if "?" in url else url + "?do=edit"
  93. response = session.get(edit_url)
  94. if response.status_code == 200:
  95. soup = BeautifulSoup(response.text, "html.parser")
  96. textarea = soup.find("textarea", {"name": "wikitext"})
  97. if textarea:
  98. raw_text = textarea.text
  99. url_texts[url] = raw_text
  100. else:
  101. print(f"⚠️ Pas de balise <textarea> trouvée pour {edit_url}.")
  102. else:
  103. print(f"⚠️ Erreur d'accès à {edit_url} : code {response.status_code}")
  104. except Exception as e:
  105. print(f"❌ Exception pour {url}: {e}")
  106. return url_texts
  107. def build_faiss_index(embeddings):
  108. dimension = embeddings.shape[1]
  109. index = faiss.IndexFlatIP(dimension)
  110. faiss.normalize_L2(embeddings)
  111. index.add(embeddings)
  112. return index
  113. def search_faiss(index, query_embedding, top_k=10):
  114. faiss.normalize_L2(query_embedding)
  115. distances, indices = index.search(query_embedding, top_k)
  116. return distances[0], indices[0]
  117. def get_relevance_label(score):
  118. if score >= 2.5:
  119. return "✅ bon"
  120. elif score >= 0:
  121. return "⚠️ moyen"
  122. else:
  123. return "❌ faible"
  124. def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=30, rerank_top_k=10):
  125. question_embedding = model.encode([question], convert_to_tensor=False)
  126. question_embedding = np.array(question_embedding).astype('float32')
  127. if question_embedding.ndim == 1:
  128. question_embedding = np.expand_dims(question_embedding, axis=0)
  129. distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
  130. candidates = []
  131. for idx, dist in zip(indices, distances):
  132. candidates.append({
  133. "chunk": chunks[idx],
  134. "url": chunk_urls[idx],
  135. "index": idx,
  136. "faiss_score": float(dist)
  137. })
  138. # Reranking
  139. pairs = [[question, c["chunk"]] for c in candidates]
  140. rerank_scores = cross_encoder.predict(pairs)
  141. for i in range(len(candidates)):
  142. candidates[i]["rerank_score"] = float(rerank_scores[i])
  143. reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
  144. print("\n🔍 Résultats re-rankés :")
  145. df = pd.DataFrame([{
  146. "Index": r["index"],
  147. "URL": r["url"],
  148. "Score CrossEncoder": round(r["rerank_score"], 4),
  149. "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
  150. } for r in reranked])
  151. df["Pertinence"] = df["Score CrossEncoder"].apply(get_relevance_label)
  152. pd.set_option('display.max_colwidth', None)
  153. df_reranked = df.copy()
  154. # Affichage formaté ligne par ligne
  155. for i, row in df_reranked.iterrows():
  156. print(f"{i:>3} {row['URL']:<70} {row['Score CrossEncoder']:.4f} {row['Pertinence']}")
  157. print(f"{row['Chunk'][:150]}...\n")
  158. # Tableau propre
  159. colonnes_a_afficher = ['Index', 'URL', 'Score CrossEncoder', 'Pertinence', 'Chunk']
  160. df_affichage = df_reranked[colonnes_a_afficher].copy()
  161. df_affichage.reset_index(drop=True, inplace=True)
  162. print("\n🎯 Résultats re-rankés (top 10) :")
  163. print(df_affichage.head(10).to_string(index=True))
  164. # Si tous les top chunks viennent d'une seule page
  165. top_urls = [r["url"] for r in reranked]
  166. unique_top_urls = list(set(top_urls))
  167. if len(unique_top_urls) == 1:
  168. print("ℹ️ Tous les meilleurs chunks viennent de la même page. Utilisation de l'ensemble de la page.")
  169. full_context = "\n\n".join([
  170. chunks[i] for i in range(len(chunks)) if chunk_urls[i] == unique_top_urls[0]
  171. ])
  172. else:
  173. full_context = "\n\n".join([r["chunk"] for r in reranked])
  174. result = qa_chain.run(context=full_context, question=question)
  175. print(f"\n🤖 Réponse :\n{result}\n")
  176. def semantic_chunking(sentences, similarity_threshold=0.5):
  177. chunks = []
  178. current_chunk = []
  179. current_embeddings = []
  180. for sentence in sentences:
  181. embedding = model.encode(sentence)
  182. if not current_embeddings:
  183. current_chunk.append(sentence)
  184. current_embeddings.append(embedding)
  185. else:
  186. similarities = cosine_similarity([embedding], current_embeddings)[0]
  187. if max(similarities) > similarity_threshold:
  188. current_chunk.append(sentence)
  189. current_embeddings.append(embedding)
  190. else:
  191. chunks.append(" ".join(current_chunk))
  192. current_chunk = [sentence]
  193. current_embeddings = [embedding]
  194. if current_chunk:
  195. chunks.append(" ".join(current_chunk))
  196. return chunks
  197. if __name__ == "__main__":
  198. model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')
  199. if login_to_wiki("nadine.elleuch", "witpowgeu!75"):
  200. wiki_urls = [
  201. "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
  202. "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
  203. "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
  204. "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
  205. ]
  206. url_texts = fetch_all_pages_content(wiki_urls)
  207. all_chunks = []
  208. chunk_urls = []
  209. for url, text in url_texts.items():
  210. print(f"\n--- Traitement de la page {url} ---")
  211. # Split the text into sentences
  212. sentences = nltk.sent_tokenize(text, language='french')
  213. # Semantic chunking using similarity
  214. chunks = semantic_chunking(sentences)
  215. print(f"Nombre de chunks : {len(chunks)}")
  216. for i, chunk in enumerate(chunks):
  217. print(f"Chunk {i+1} ({len(chunk.split())} mots): {chunk[:100]}{'...' if len(chunk) > 100 else ''}")
  218. all_chunks.append(chunk)
  219. chunk_urls.append(url)
  220. # Prepare chunk metadata
  221. chunk_data = [
  222. {"url": url_, "chunk_index": i + 1, "chunk": chunk}
  223. for i, (url_, chunk) in enumerate(zip(chunk_urls, all_chunks))
  224. ]
  225. with open("chunks_pretty.json", "w", encoding="utf-8") as f:
  226. json.dump(chunk_data, f, ensure_ascii=False, indent=2)
  227. print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
  228. # Load reranker
  229. cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
  230. # Encode the chunks
  231. print("\n✅ Génération des embeddings des chunks...")
  232. chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
  233. chunk_embeddings = np.array(chunk_embeddings).astype('float32')
  234. # Build FAISS index
  235. print("\n✅ Construction de l'index FAISS...")
  236. index = build_faiss_index(chunk_embeddings)
  237. print("✅ Index FAISS construit avec succès.")
  238. # Define prompt
  239. prompt_template = PromptTemplate(
  240. input_variables=["context", "question"],
  241. template="""
  242. Tu es un assistant expert du wiki interne d'Intersel. Réponds uniquement à partir du CONTEXTE donné.
  243. Si aucune information pertinente n'est disponible, dis "Je ne sais pas." sans inventer.
  244. CONTEXTE :
  245. {context}
  246. QUESTION :
  247. {question}
  248. RÉPONSE :
  249. """
  250. )
  251. llm = Ollama(model="llama3")
  252. qa_chain = LLMChain(llm=llm, prompt=prompt_template)
  253. while True:
  254. question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
  255. if question.lower() in ["exit", "quit"]:
  256. break
  257. ask_question_with_reranking(
  258. question, model, cross_encoder, index,
  259. all_chunks, chunk_urls, qa_chain
  260. )