From b70f38626eec9c485139b1d7d2eba44b96d5562d Mon Sep 17 00:00:00 2001 From: Barrett Ruth Date: Wed, 24 Sep 2025 20:04:29 -0400 Subject: [PATCH] cleanup --- scrapers/__init__.py | 58 +------ scrapers/cses.py | 0 tests/scrapers/test_interface_compliance.py | 167 -------------------- 3 files changed, 4 insertions(+), 221 deletions(-) mode change 100755 => 100644 scrapers/cses.py delete mode 100644 tests/scrapers/test_interface_compliance.py diff --git a/scrapers/__init__.py b/scrapers/__init__.py index 01e594c..4749123 100644 --- a/scrapers/__init__.py +++ b/scrapers/__init__.py @@ -1,55 +1,5 @@ -def __getattr__(name): - if name == "AtCoderScraper": - from .atcoder import AtCoderScraper +from .atcoder import AtCoderScraper +from .codeforces import CodeforcesScraper +from .cses import CSESScraper - return AtCoderScraper - elif name == "BaseScraper": - from .base import BaseScraper - - return BaseScraper - elif name == "ScraperConfig": - from .base import ScraperConfig - - return ScraperConfig - elif name == "CodeforcesScraper": - from .codeforces import CodeforcesScraper - - return CodeforcesScraper - elif name == "CSESScraper": - from .cses import CSESScraper - - return CSESScraper - elif name in [ - "ContestListResult", - "ContestSummary", - "MetadataResult", - "ProblemSummary", - "TestCase", - "TestsResult", - ]: - from .models import ( - ContestListResult, # noqa: F401 - ContestSummary, # noqa: F401 - MetadataResult, # noqa: F401 - ProblemSummary, # noqa: F401 - TestCase, # noqa: F401 - TestsResult, # noqa: F401 - ) - - return locals()[name] - raise AttributeError(f"module 'scrapers' has no attribute '{name}'") - - -__all__ = [ - "AtCoderScraper", - "BaseScraper", - "CodeforcesScraper", - "CSESScraper", - "ScraperConfig", - "ContestListResult", - "ContestSummary", - "MetadataResult", - "ProblemSummary", - "TestCase", - "TestsResult", -] +__all__ = ["CodeforcesScraper", "CSESScraper", "AtCoderScraper"] diff --git a/scrapers/cses.py b/scrapers/cses.py old mode 100755 new mode 100644 diff --git a/tests/scrapers/test_interface_compliance.py b/tests/scrapers/test_interface_compliance.py deleted file mode 100644 index ab07ff2..0000000 --- a/tests/scrapers/test_interface_compliance.py +++ /dev/null @@ -1,167 +0,0 @@ -from unittest.mock import Mock - -import pytest - -import scrapers -from scrapers.base import BaseScraper -from scrapers.models import ContestListResult, MetadataResult, TestsResult - -SCRAPERS = [ - scrapers.AtCoderScraper, - scrapers.CodeforcesScraper, - scrapers.CSESScraper, -] - - -class TestScraperInterfaceCompliance: - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_implements_base_interface(self, scraper_class): - scraper = scraper_class() - - assert isinstance(scraper, BaseScraper) - assert hasattr(scraper, "platform_name") - assert hasattr(scraper, "scrape_contest_metadata") - assert hasattr(scraper, "scrape_problem_tests") - assert hasattr(scraper, "scrape_contest_list") - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_platform_name_is_string(self, scraper_class): - scraper = scraper_class() - platform_name = scraper.platform_name - - assert isinstance(platform_name, str) - assert len(platform_name) > 0 - assert platform_name.islower() # Convention: lowercase platform names - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_metadata_method_signature(self, scraper_class, mocker): - scraper = scraper_class() - - # Mock the underlying HTTP calls to avoid network requests - if scraper.platform_name == "codeforces": - mock_scraper = Mock() - mock_response = Mock() - mock_response.text = "A. Test" - mock_scraper.get.return_value = mock_response - mocker.patch( - "scrapers.codeforces.cloudscraper.create_scraper", - return_value=mock_scraper, - ) - - result = scraper.scrape_contest_metadata("test_contest") - - assert isinstance(result, MetadataResult) - assert hasattr(result, "success") - assert hasattr(result, "error") - assert hasattr(result, "problems") - assert hasattr(result, "contest_id") - assert isinstance(result.success, bool) - assert isinstance(result.error, str) - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_problem_tests_method_signature(self, scraper_class, mocker): - scraper = scraper_class() - - if scraper.platform_name == "codeforces": - mock_scraper = Mock() - mock_response = Mock() - mock_response.text = """ -
Time limit: 1 seconds
-
Memory limit: 256 megabytes
- 
3
- 
6
- """ - mock_scraper.get.return_value = mock_response - mocker.patch( - "scrapers.codeforces.cloudscraper.create_scraper", - return_value=mock_scraper, - ) - - result = scraper.scrape_problem_tests("test_contest", "A") - - assert isinstance(result, TestsResult) - assert hasattr(result, "success") - assert hasattr(result, "error") - assert hasattr(result, "tests") - assert hasattr(result, "problem_id") - assert hasattr(result, "url") - assert hasattr(result, "timeout_ms") - assert hasattr(result, "memory_mb") - assert isinstance(result.success, bool) - assert isinstance(result.error, str) - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_contest_list_method_signature(self, scraper_class, mocker): - scraper = scraper_class() - - if scraper.platform_name == "codeforces": - mock_scraper = Mock() - mock_response = Mock() - mock_response.json.return_value = { - "status": "OK", - "result": [{"id": 1900, "name": "Test Contest"}], - } - mock_scraper.get.return_value = mock_response - mocker.patch( - "scrapers.codeforces.cloudscraper.create_scraper", - return_value=mock_scraper, - ) - - result = scraper.scrape_contest_list() - - assert isinstance(result, ContestListResult) - assert hasattr(result, "success") - assert hasattr(result, "error") - assert hasattr(result, "contests") - assert isinstance(result.success, bool) - assert isinstance(result.error, str) - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_error_message_format(self, scraper_class, mocker): - scraper = scraper_class() - platform_name = scraper.platform_name - - # Force an error by mocking HTTP failure - if scraper.platform_name == "codeforces": - mock_scraper = Mock() - mock_scraper.get.side_effect = Exception("Network error") - mocker.patch( - "scrapers.codeforces.cloudscraper.create_scraper", - return_value=mock_scraper, - ) - elif scraper.platform_name == "atcoder": - mocker.patch( - "scrapers.atcoder.requests.get", side_effect=Exception("Network error") - ) - elif scraper.platform_name == "cses": - mocker.patch( - "scrapers.cses.make_request", side_effect=Exception("Network error") - ) - - # Test metadata error format - result = scraper.scrape_contest_metadata("test") - assert not result.success - assert result.error.startswith(f"{platform_name}: ") - - # Test problem tests error format - result = scraper.scrape_problem_tests("test", "A") - assert not result.success - assert result.error.startswith(f"{platform_name}: ") - - # Test contest list error format - result = scraper.scrape_contest_list() - assert not result.success - assert result.error.startswith(f"{platform_name}: ") - - @pytest.mark.parametrize("scraper_class", SCRAPERS) - def test_scraper_instantiation(self, scraper_class): - scraper1 = scraper_class() - assert isinstance(scraper1, BaseScraper) - assert scraper1.config is not None - - from scrapers.base import ScraperConfig - - custom_config = ScraperConfig(timeout_seconds=60) - scraper2 = scraper_class(custom_config) - assert isinstance(scraper2, BaseScraper) - assert scraper2.config.timeout_seconds == 60