| 12345678910111213141516171819202122232425262728293031323334353637383940414243 |
- import sys
- import os
- sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
- from unittest.mock import patch
- from agent_ai.scraping import fetch_all_pages_content
- @patch("agent_ai.scraping.session.get")
- def test_fetch_success(mock_get):
- mock_get.return_value.status_code = 200
- mock_get.return_value.text = '''
- <div id="dokuwiki__content">Sample content</div>
- '''
- urls = ["https://example.com/page"]
- url_texts, new_hashes = fetch_all_pages_content(urls, {})
- assert url_texts[urls[0]] == "Sample content"
- @patch("agent_ai.scraping.session.get")
- def test_fetch_no_textarea(mock_get):
- mock_get.return_value.status_code = 200
- mock_get.return_value.text = "<html>No textarea</html>"
- urls = ["https://example.com/no-textarea"]
- result = fetch_all_pages_content(urls, {})
- assert urls[0] not in result
- @patch("agent_ai.scraping.session.get")
- def test_fetch_error_status(mock_get):
- mock_get.return_value.status_code = 404
- urls = ["https://example.com/404"]
- result = fetch_all_pages_content(urls, {})
- assert urls[0] not in result
- @patch("agent_ai.scraping.session.get")
- def test_fetch_exception(mock_get):
- mock_get.side_effect = Exception("Network error")
- urls = ["https://example.com/error"]
- result = fetch_all_pages_content(urls, {})
- assert urls[0] not in result
|