import sys import os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import pytest from agent_ai.chunking import split_into_chunks, count_tokens import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) def test_empty_input(): chunks = split_into_chunks("", max_tokens=10, overlap=0) assert chunks == [] def test_max_token_limit(): text = "This is a simple sentence. " * 50 chunks = split_into_chunks(text, max_tokens=30, overlap=5) assert all(chunk["token_count"] <= 30 for chunk in chunks) def test_count_tokens_basic(): text = "Hello, this is a test." tokens = count_tokens(text) assert isinstance(tokens, int) assert tokens > 0 def test_split_into_chunks_basic(): text = "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five." chunks = split_into_chunks(text, max_tokens=15, overlap=0, with_metadata=True) assert isinstance(chunks, list) assert all("chunk_id" in chunk for chunk in chunks) assert all("text" in chunk for chunk in chunks) assert all("word_count" in chunk for chunk in chunks) assert all("token_count" in chunk for chunk in chunks) def test_split_into_chunks_no_metadata(): text = "Sentence one. Sentence two. Sentence three." chunks = split_into_chunks(text, max_tokens=10, overlap=0, with_metadata=False) assert isinstance(chunks, list) assert all(isinstance(chunk, str) for chunk in chunks)