test_scraping.py 1.2 KB

1234567891011121314151617181920212223242526272829303132333435363738
  1. from unittest.mock import patch
  2. from agent_ai.scraping import fetch_all_pages_content
  3. @patch("agent_ai.scraping.session.get")
  4. def test_fetch_success(mock_get):
  5. mock_get.return_value.status_code = 200
  6. mock_get.return_value.text = '''
  7. <textarea name="wikitext">Sample content</textarea>
  8. '''
  9. urls = ["https://example.com/page"]
  10. result = fetch_all_pages_content(urls)
  11. assert result[urls[0]] == "Sample content"
  12. @patch("agent_ai.scraping.session.get")
  13. def test_fetch_no_textarea(mock_get):
  14. mock_get.return_value.status_code = 200
  15. mock_get.return_value.text = "<html>No textarea</html>"
  16. urls = ["https://example.com/no-textarea"]
  17. result = fetch_all_pages_content(urls)
  18. assert urls[0] not in result
  19. @patch("agent_ai.scraping.session.get")
  20. def test_fetch_error_status(mock_get):
  21. mock_get.return_value.status_code = 404
  22. urls = ["https://example.com/404"]
  23. result = fetch_all_pages_content(urls)
  24. assert urls[0] not in result
  25. @patch("agent_ai.scraping.session.get")
  26. def test_fetch_exception(mock_get):
  27. mock_get.side_effect = Exception("Network error")
  28. urls = ["https://example.com/error"]
  29. result = fetch_all_pages_content(urls)
  30. assert urls[0] not in result