test_chunking.py 1.4 KB

1234567891011121314151617181920212223242526272829303132333435363738
  1. import sys
  2. import os
  3. sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
  4. import pytest
  5. from agent_ai.chunking import split_into_chunks, count_tokens
  6. import warnings
  7. warnings.filterwarnings("ignore", category=DeprecationWarning)
  8. def test_empty_input():
  9. chunks = split_into_chunks("", max_tokens=10, overlap=0)
  10. assert chunks == []
  11. def test_max_token_limit():
  12. text = "This is a simple sentence. " * 50
  13. chunks = split_into_chunks(text, max_tokens=30, overlap=5)
  14. assert all(chunk["token_count"] <= 30 for chunk in chunks)
  15. def test_count_tokens_basic():
  16. text = "Hello, this is a test."
  17. tokens = count_tokens(text)
  18. assert isinstance(tokens, int)
  19. assert tokens > 0
  20. def test_split_into_chunks_basic():
  21. text = "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five."
  22. chunks = split_into_chunks(text, max_tokens=15, overlap=0, with_metadata=True)
  23. assert isinstance(chunks, list)
  24. assert all("chunk_id" in chunk for chunk in chunks)
  25. assert all("text" in chunk for chunk in chunks)
  26. assert all("word_count" in chunk for chunk in chunks)
  27. assert all("token_count" in chunk for chunk in chunks)
  28. def test_split_into_chunks_no_metadata():
  29. text = "Sentence one. Sentence two. Sentence three."
  30. chunks = split_into_chunks(text, max_tokens=10, overlap=0, with_metadata=False)
  31. assert isinstance(chunks, list)
  32. assert all(isinstance(chunk, str) for chunk in chunks)