wiki.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267
  1. import requests
  2. from bs4 import BeautifulSoup
  3. from langchain_community.llms import Ollama
  4. from langchain.prompts import PromptTemplate
  5. from langchain.chains import LLMChain
  6. import warnings
  7. import nltk
  8. import spacy
  9. from nltk.corpus import stopwords
  10. from sentence_transformers import SentenceTransformer, CrossEncoder
  11. import torch
  12. import pandas as pd
  13. import faiss
  14. import numpy as np
  15. from langchain.text_splitter import RecursiveCharacterTextSplitter
  16. import json
  17. from urllib.parse import urljoin, urlparse
  18. from bs4 import SoupStrainer
  19. warnings.filterwarnings("ignore")
  20. nltk.download('punkt')
  21. nltk.download('stopwords')
  22. stop_words = set(stopwords.words('french'))
  23. nlp = spacy.load('fr_core_news_sm')
  24. session = requests.Session()
  25. def login_to_wiki(username, password):
  26. login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login"
  27. resp = session.get(login_url)
  28. soup = BeautifulSoup(resp.text, "html.parser")
  29. sectok_input = soup.find("input", {"name": "sectok"})
  30. if not sectok_input:
  31. print("❌ Impossible de trouver le token CSRF.")
  32. return False
  33. login_data = {
  34. "u": username,
  35. "p": password,
  36. "do": "login",
  37. "sectok": sectok_input.get("value")
  38. }
  39. headers = {
  40. "User-Agent": "Mozilla/5.0",
  41. "Referer": login_url
  42. }
  43. login_resp = session.post(login_url, data=login_data, headers=headers)
  44. if "logout" in login_resp.text.lower():
  45. print("✅ Connexion réussie.")
  46. return True
  47. else:
  48. print("❌ Échec de la connexion.")
  49. return False
  50. def extract_visible_text(html):
  51. soup = BeautifulSoup(html, "html.parser")
  52. for tag in soup(["script", "style", "header", "footer", "nav"]):
  53. tag.decompose()
  54. text = soup.get_text(separator="\n")
  55. lines = [line.strip() for line in text.splitlines() if line.strip()]
  56. return "\n".join(lines)
  57. def clean_text(text):
  58. doc = nlp(text)
  59. tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
  60. return ' '.join(tokens)
  61. def chunk_by_titles(text):
  62. """
  63. Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc).
  64. Chaque chunk correspond à une section commençant par un titre.
  65. """
  66. chunks = []
  67. current_chunk = []
  68. lines = text.split('\n')
  69. for line in lines:
  70. # Si la ligne commence par un signe '=' => début d'un nouveau chunk
  71. if line.strip().startswith('='):
  72. # Si on a déjà un chunk en cours, on l'ajoute à la liste
  73. if current_chunk:
  74. chunks.append('\n'.join(current_chunk).strip())
  75. current_chunk = []
  76. current_chunk.append(line)
  77. else:
  78. current_chunk.append(line)
  79. # Ajouter le dernier chunk
  80. if current_chunk:
  81. chunks.append('\n'.join(current_chunk).strip())
  82. return chunks
  83. def semantic_chunking(raw_text, max_tokens=200):
  84. sentences = nltk.sent_tokenize(raw_text, language='french')
  85. chunks = []
  86. current_chunk = ""
  87. current_len = 0
  88. for sentence in sentences:
  89. sentence_len = len(sentence.split())
  90. if current_len + sentence_len > max_tokens:
  91. chunks.append(current_chunk.strip())
  92. current_chunk = sentence + " "
  93. current_len = sentence_len
  94. else:
  95. current_chunk += sentence + " "
  96. current_len += sentence_len
  97. if current_chunk:
  98. chunks.append(current_chunk.strip())
  99. return chunks
  100. def fetch_all_pages_content(urls):
  101. url_texts = {}
  102. for url in urls:
  103. try:
  104. response = session.get(url)
  105. if response.status_code == 200:
  106. page_text = extract_visible_text(response.text)
  107. url_texts[url] = page_text
  108. else:
  109. print(f"⚠️ Erreur d'accès à {url} : code {response.status_code}")
  110. except Exception as e:
  111. print(f"❌ Exception pour {url}: {e}")
  112. return url_texts
  113. def build_faiss_index(embeddings):
  114. dimension = embeddings.shape[1]
  115. index = faiss.IndexFlatIP(dimension)
  116. faiss.normalize_L2(embeddings)
  117. index.add(embeddings)
  118. return index
  119. def search_faiss(index, query_embedding, top_k=10):
  120. faiss.normalize_L2(query_embedding)
  121. distances, indices = index.search(query_embedding, top_k)
  122. return distances[0], indices[0]
  123. def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=20, rerank_top_k=5):
  124. question_embedding = model.encode([question], convert_to_tensor=False)
  125. question_embedding = np.array(question_embedding).astype('float32')
  126. distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
  127. candidates = []
  128. for idx, dist in zip(indices, distances):
  129. candidates.append({
  130. "chunk": chunks[idx],
  131. "url": chunk_urls[idx],
  132. "index": idx,
  133. "faiss_score": float(dist)
  134. })
  135. # Re-ranking avec CrossEncoder
  136. pairs = [[question, c["chunk"]] for c in candidates]
  137. rerank_scores = cross_encoder.predict(pairs)
  138. for i in range(len(candidates)):
  139. candidates[i]["rerank_score"] = float(rerank_scores[i])
  140. # Tri final selon score CrossEncoder
  141. reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
  142. print("\n🔍 Résultats re-rankés :")
  143. df = pd.DataFrame([{
  144. "Index": r["index"],
  145. "URL": r["url"],
  146. "Score CrossEncoder": round(r["rerank_score"], 4),
  147. "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
  148. } for r in reranked])
  149. print(df)
  150. context = "\n\n".join([r["chunk"] for r in reranked])
  151. result = qa_chain.run(context=context, question=question)
  152. print(f"\n🤖 Réponse :\n{result}\n")
  153. if __name__ == "__main__":
  154. if login_to_wiki("", ""):
  155. wiki_urls = [
  156. "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
  157. "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
  158. "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
  159. "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
  160. ]
  161. url_texts = fetch_all_pages_content(wiki_urls)
  162. all_chunks = []
  163. chunk_urls = []
  164. for url, text in url_texts.items():
  165. print(f"\n--- Traitement de la page {url} ---")
  166. chunks = chunk_by_titles(text)
  167. print(f"Nombre de chunks par titres : {len(chunks)}")
  168. # Nettoyage + découpage en sous-chunks si trop long
  169. cleaned_chunks = []
  170. for chunk in chunks:
  171. cleaned = clean_text(chunk)
  172. if len(cleaned.split()) > 200:
  173. subchunks = semantic_chunking(cleaned, max_tokens=200)
  174. cleaned_chunks.extend(subchunks)
  175. else:
  176. cleaned_chunks.append(cleaned)
  177. for i, c in enumerate(cleaned_chunks):
  178. print(f"Chunk {i+1} ({len(c.split())} mots): {c[:100]}{'...' if len(c) > 100 else ''}")
  179. all_chunks.append(c)
  180. chunk_urls.append(url)
  181. import json
  182. chunk_data = [
  183. {"url": url, "chunk_index": i+1, "chunk": chunk}
  184. for i, (url, chunk) in enumerate(zip(chunk_urls, all_chunks))
  185. ]
  186. with open("chunks_pretty.json", "w", encoding="utf-8") as f:
  187. json.dump(chunk_data, f, ensure_ascii=False, indent=2)
  188. print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
  189. model = SentenceTransformer('all-mpnet-base-v2')
  190. cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
  191. print("\n✅ Génération des embeddings des chunks...")
  192. chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
  193. chunk_embeddings = np.array(chunk_embeddings).astype('float32')
  194. print("\n✅ Construction de l'index FAISS...")
  195. index = build_faiss_index(chunk_embeddings)
  196. print("✅ Index FAISS construit avec succès.")
  197. prompt_template = PromptTemplate(
  198. input_variables=["context", "question"],
  199. template="""
  200. Tu es un assistant expert du contenu du wiki interne d'Intersel. Ta tâche est de répondre à la question en utilisant uniquement les informations pertinentes du contexte fourni.
  201. Si l'information existe, donne une réponse claire, concise et précise.
  202. CONTEXTE :
  203. {context}
  204. QUESTION :
  205. {question}
  206. RÉPONSE :
  207. """
  208. )
  209. llm = Ollama(model="llama3")
  210. qa_chain = LLMChain(llm=llm, prompt=prompt_template)
  211. while True:
  212. question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
  213. if question.lower() in ["exit", "quit"]:
  214. break
  215. ask_question_with_reranking(question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain)