test_scraping.py 1.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243
  1. import sys
  2. import os
  3. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
  4. from unittest.mock import patch
  5. from agent_ai.scraping import fetch_all_pages_content
  6. @patch("agent_ai.scraping.session.get")
  7. def test_fetch_success(mock_get):
  8. mock_get.return_value.status_code = 200
  9. mock_get.return_value.text = '''
  10. <div id="dokuwiki__content">Sample content</div>
  11. '''
  12. urls = ["https://example.com/page"]
  13. url_texts, new_hashes = fetch_all_pages_content(urls, {})
  14. assert url_texts[urls[0]] == "Sample content"
  15. @patch("agent_ai.scraping.session.get")
  16. def test_fetch_no_textarea(mock_get):
  17. mock_get.return_value.status_code = 200
  18. mock_get.return_value.text = "<html>No textarea</html>"
  19. urls = ["https://example.com/no-textarea"]
  20. result = fetch_all_pages_content(urls, {})
  21. assert urls[0] not in result
  22. @patch("agent_ai.scraping.session.get")
  23. def test_fetch_error_status(mock_get):
  24. mock_get.return_value.status_code = 404
  25. urls = ["https://example.com/404"]
  26. result = fetch_all_pages_content(urls, {})
  27. assert urls[0] not in result
  28. @patch("agent_ai.scraping.session.get")
  29. def test_fetch_exception(mock_get):
  30. mock_get.side_effect = Exception("Network error")
  31. urls = ["https://example.com/error"]
  32. result = fetch_all_pages_content(urls, {})
  33. assert urls[0] not in result