| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267 |
- import requests
- from bs4 import BeautifulSoup
- from langchain_community.llms import Ollama
- from langchain.prompts import PromptTemplate
- from langchain.chains import LLMChain
- import warnings
- import nltk
- import spacy
- from nltk.corpus import stopwords
- from sentence_transformers import SentenceTransformer, CrossEncoder
- import torch
- import pandas as pd
- import faiss
- import numpy as np
- from langchain.text_splitter import RecursiveCharacterTextSplitter
- import json
- from urllib.parse import urljoin, urlparse
- from bs4 import SoupStrainer
- warnings.filterwarnings("ignore")
- nltk.download('punkt')
- nltk.download('stopwords')
- stop_words = set(stopwords.words('french'))
- nlp = spacy.load('fr_core_news_sm')
- session = requests.Session()
- def login_to_wiki(username, password):
- login_url = "https://wiki.intersel.fr/doku.php?id=start&do=login"
- resp = session.get(login_url)
- soup = BeautifulSoup(resp.text, "html.parser")
- sectok_input = soup.find("input", {"name": "sectok"})
- if not sectok_input:
- print("❌ Impossible de trouver le token CSRF.")
- return False
- login_data = {
- "u": username,
- "p": password,
- "do": "login",
- "sectok": sectok_input.get("value")
- }
- headers = {
- "User-Agent": "Mozilla/5.0",
- "Referer": login_url
- }
- login_resp = session.post(login_url, data=login_data, headers=headers)
- if "logout" in login_resp.text.lower():
- print("✅ Connexion réussie.")
- return True
- else:
- print("❌ Échec de la connexion.")
- return False
- def extract_visible_text(html):
- soup = BeautifulSoup(html, "html.parser")
- for tag in soup(["script", "style", "header", "footer", "nav"]):
- tag.decompose()
- text = soup.get_text(separator="\n")
- lines = [line.strip() for line in text.splitlines() if line.strip()]
- return "\n".join(lines)
- def clean_text(text):
- doc = nlp(text)
- tokens = [token.text for token in doc if token.text.lower() not in stop_words and token.text.strip() != ""]
- return ' '.join(tokens)
- def chunk_by_titles(text):
- """
- Découpe le texte en chunks basés sur les titres DokuWiki (=, ==, === etc).
- Chaque chunk correspond à une section commençant par un titre.
- """
- chunks = []
- current_chunk = []
- lines = text.split('\n')
- for line in lines:
- # Si la ligne commence par un signe '=' => début d'un nouveau chunk
- if line.strip().startswith('='):
- # Si on a déjà un chunk en cours, on l'ajoute à la liste
- if current_chunk:
- chunks.append('\n'.join(current_chunk).strip())
- current_chunk = []
- current_chunk.append(line)
- else:
- current_chunk.append(line)
- # Ajouter le dernier chunk
- if current_chunk:
- chunks.append('\n'.join(current_chunk).strip())
- return chunks
- def semantic_chunking(raw_text, max_tokens=200):
- sentences = nltk.sent_tokenize(raw_text, language='french')
- chunks = []
- current_chunk = ""
- current_len = 0
- for sentence in sentences:
- sentence_len = len(sentence.split())
- if current_len + sentence_len > max_tokens:
- chunks.append(current_chunk.strip())
- current_chunk = sentence + " "
- current_len = sentence_len
- else:
- current_chunk += sentence + " "
- current_len += sentence_len
- if current_chunk:
- chunks.append(current_chunk.strip())
- return chunks
- def fetch_all_pages_content(urls):
- url_texts = {}
- for url in urls:
- try:
- response = session.get(url)
- if response.status_code == 200:
- page_text = extract_visible_text(response.text)
- url_texts[url] = page_text
- else:
- print(f"⚠️ Erreur d'accès à {url} : code {response.status_code}")
- except Exception as e:
- print(f"❌ Exception pour {url}: {e}")
- return url_texts
- def build_faiss_index(embeddings):
- dimension = embeddings.shape[1]
- index = faiss.IndexFlatIP(dimension)
- faiss.normalize_L2(embeddings)
- index.add(embeddings)
- return index
- def search_faiss(index, query_embedding, top_k=10):
- faiss.normalize_L2(query_embedding)
- distances, indices = index.search(query_embedding, top_k)
- return distances[0], indices[0]
- def ask_question_with_reranking(question, model, cross_encoder, index, chunks, chunk_urls, qa_chain, faiss_top_k=20, rerank_top_k=5):
- question_embedding = model.encode([question], convert_to_tensor=False)
- question_embedding = np.array(question_embedding).astype('float32')
- distances, indices = search_faiss(index, question_embedding, top_k=faiss_top_k)
- candidates = []
- for idx, dist in zip(indices, distances):
- candidates.append({
- "chunk": chunks[idx],
- "url": chunk_urls[idx],
- "index": idx,
- "faiss_score": float(dist)
- })
- # Re-ranking avec CrossEncoder
- pairs = [[question, c["chunk"]] for c in candidates]
- rerank_scores = cross_encoder.predict(pairs)
- for i in range(len(candidates)):
- candidates[i]["rerank_score"] = float(rerank_scores[i])
- # Tri final selon score CrossEncoder
- reranked = sorted(candidates, key=lambda x: x["rerank_score"], reverse=True)[:rerank_top_k]
- print("\n🔍 Résultats re-rankés :")
- df = pd.DataFrame([{
- "Index": r["index"],
- "URL": r["url"],
- "Score CrossEncoder": round(r["rerank_score"], 4),
- "Chunk": r["chunk"][:100] + "..." if len(r["chunk"]) > 100 else r["chunk"]
- } for r in reranked])
- print(df)
- context = "\n\n".join([r["chunk"] for r in reranked])
- result = qa_chain.run(context=context, question=question)
- print(f"\n🤖 Réponse :\n{result}\n")
- if __name__ == "__main__":
- if login_to_wiki("", ""):
- wiki_urls = [
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation_equipe",
- "https://wiki.intersel.fr/doku.php?id=intersel:presentation",
- "https://wiki.intersel.fr/doku.php?id=intersel:guides:entretiens:stagiaires",
- "https://wiki.intersel.fr/doku.php?id=projets:samsic:anticipia:start"
- ]
- url_texts = fetch_all_pages_content(wiki_urls)
- all_chunks = []
- chunk_urls = []
- for url, text in url_texts.items():
- print(f"\n--- Traitement de la page {url} ---")
- chunks = chunk_by_titles(text)
- print(f"Nombre de chunks par titres : {len(chunks)}")
- # Nettoyage + découpage en sous-chunks si trop long
- cleaned_chunks = []
- for chunk in chunks:
- cleaned = clean_text(chunk)
- if len(cleaned.split()) > 200:
- subchunks = semantic_chunking(cleaned, max_tokens=200)
- cleaned_chunks.extend(subchunks)
- else:
- cleaned_chunks.append(cleaned)
- for i, c in enumerate(cleaned_chunks):
- print(f"Chunk {i+1} ({len(c.split())} mots): {c[:100]}{'...' if len(c) > 100 else ''}")
- all_chunks.append(c)
- chunk_urls.append(url)
- import json
- chunk_data = [
- {"url": url, "chunk_index": i+1, "chunk": chunk}
- for i, (url, chunk) in enumerate(zip(chunk_urls, all_chunks))
- ]
- with open("chunks_pretty.json", "w", encoding="utf-8") as f:
- json.dump(chunk_data, f, ensure_ascii=False, indent=2)
- print("📁 Chunks enregistrés dans 'chunks_pretty.json'")
- model = SentenceTransformer('all-mpnet-base-v2')
- cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
- print("\n✅ Génération des embeddings des chunks...")
- chunk_embeddings = model.encode(all_chunks, convert_to_tensor=False, show_progress_bar=True)
- chunk_embeddings = np.array(chunk_embeddings).astype('float32')
-
- print("\n✅ Construction de l'index FAISS...")
- index = build_faiss_index(chunk_embeddings)
- print("✅ Index FAISS construit avec succès.")
-
- prompt_template = PromptTemplate(
- input_variables=["context", "question"],
- template="""
- Tu es un assistant expert du contenu du wiki interne d'Intersel. Ta tâche est de répondre à la question en utilisant uniquement les informations pertinentes du contexte fourni.
- Si l'information existe, donne une réponse claire, concise et précise.
- CONTEXTE :
- {context}
- QUESTION :
- {question}
- RÉPONSE :
- """
- )
- llm = Ollama(model="llama3")
- qa_chain = LLMChain(llm=llm, prompt=prompt_template)
- while True:
- question = input("❓ Posez votre question sur le Wiki ('exit' pour quitter) :\n> ")
- if question.lower() in ["exit", "quit"]:
- break
- ask_question_with_reranking(question, model, cross_encoder, index, all_chunks, chunk_urls, qa_chain)
-
|