| 1234567891011121314151617181920212223242526272829303132333435363738 |
- import sys
- import os
- sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
- import pytest
- from agent_ai.chunking import split_into_chunks, count_tokens
- import warnings
- warnings.filterwarnings("ignore", category=DeprecationWarning)
- def test_empty_input():
- chunks = split_into_chunks("", max_tokens=10, overlap=0)
- assert chunks == []
- def test_max_token_limit():
- text = "This is a simple sentence. " * 50
- chunks = split_into_chunks(text, max_tokens=30, overlap=5)
- assert all(chunk["token_count"] <= 30 for chunk in chunks)
- def test_count_tokens_basic():
- text = "Hello, this is a test."
- tokens = count_tokens(text)
- assert isinstance(tokens, int)
- assert tokens > 0
- def test_split_into_chunks_basic():
- text = "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five."
- chunks = split_into_chunks(text, max_tokens=15, overlap=0, with_metadata=True)
- assert isinstance(chunks, list)
- assert all("chunk_id" in chunk for chunk in chunks)
- assert all("text" in chunk for chunk in chunks)
- assert all("word_count" in chunk for chunk in chunks)
- assert all("token_count" in chunk for chunk in chunks)
- def test_split_into_chunks_no_metadata():
- text = "Sentence one. Sentence two. Sentence three."
- chunks = split_into_chunks(text, max_tokens=10, overlap=0, with_metadata=False)
- assert isinstance(chunks, list)
- assert all(isinstance(chunk, str) for chunk in chunks)
|