feat(ci): pytest

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Install ruff
         run: uv tool install ruff
       - name: Check Python formatting with ruff
-        run: ruff format --check scrapers/
+        run: ruff format --check scrapers/ tests/scrapers/
 
   python-lint:
     name: Python Linting
@@ -75,4 +75,16 @@ jobs:
       - name: Install dependencies with mypy
         run: uv sync --dev
       - name: Type check Python files with mypy
-        run: uv run mypy scrapers/
+        run: uv run mypy scrapers/ tests/scrapers/
+
+  python-test:
+    name: Python Testing
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies with pytest
+        run: uv sync --dev
+      - name: Run Python tests
+        run: uv run pytest tests/scrapers/ -v

pyproject.toml

@@ -15,4 +15,9 @@ dev = [
     "mypy>=1.18.2",
     "types-beautifulsoup4>=4.12.0.20250516",
     "types-requests>=2.32.4.20250913",
+    "pytest>=8.0.0",
+    "pytest-mock>=3.12.0",
 ]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]

tests/scrapers/conftest.py

@@ -0,0 +1,41 @@
+import pytest
+
+
+@pytest.fixture
+def mock_codeforces_html():
+    return """
+    <div class="input">
+        <pre>
+            <div class="test-example-line-1">3</div>
+            <div class="test-example-line-1">1 2 3</div>
+        </pre>
+    </div>
+    <div class="output">
+        <pre>
+            <div class="test-example-line-1">6</div>
+        </pre>
+    </div>
+    """
+
+
+@pytest.fixture
+def mock_atcoder_html():
+    return """
+    <h3>Sample Input 1</h3>
+    <pre>3
+1 2 3</pre>
+    <h3>Sample Output 1</h3>
+    <pre>6</pre>
+    """
+
+
+@pytest.fixture
+def mock_cses_html():
+    return """
+    <h1>Example</h1>
+    <p>Input:</p>
+    <pre>3
+1 2 3</pre>
+    <p>Output:</p>
+    <pre>6</pre>
+    """

tests/scrapers/test_atcoder.py

@@ -0,0 +1,51 @@
+from unittest.mock import Mock
+import pytest
+from scrapers.atcoder import scrape, scrape_contest_problems
+
+
+def test_scrape_success(mocker, mock_atcoder_html):
+    mock_response = Mock()
+    mock_response.text = mock_atcoder_html
+
+    mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response)
+
+    result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a")
+
+    assert len(result) == 1
+    assert result[0][0] == "3\n1 2 3"
+    assert result[0][1] == "6"
+
+
+def test_scrape_contest_problems(mocker):
+    mock_response = Mock()
+    mock_response.text = """
+    <table class="table">
+        <tr><th>Task</th><th>Name</th></tr>
+        <tr>
+            <td></td>
+            <td><a href="/contests/abc350/tasks/abc350_a">A - Water Tank</a></td>
+        </tr>
+        <tr>
+            <td></td>
+            <td><a href="/contests/abc350/tasks/abc350_b">B - Dentist Aoki</a></td>
+        </tr>
+    </table>
+    """
+
+    mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response)
+
+    result = scrape_contest_problems("abc350")
+
+    assert len(result) == 2
+    assert result[0] == {"id": "a", "name": "A - Water Tank"}
+    assert result[1] == {"id": "b", "name": "B - Dentist Aoki"}
+
+
+def test_scrape_network_error(mocker):
+    mocker.patch(
+        "scrapers.atcoder.requests.get", side_effect=Exception("Network error")
+    )
+
+    result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a")
+
+    assert result == []

tests/scrapers/test_codeforces.py

@@ -0,0 +1,54 @@
+import json
+from unittest.mock import Mock
+import pytest
+from scrapers.codeforces import scrape, scrape_contest_problems
+
+
+def test_scrape_success(mocker, mock_codeforces_html):
+    mock_scraper = Mock()
+    mock_response = Mock()
+    mock_response.text = mock_codeforces_html
+    mock_scraper.get.return_value = mock_response
+
+    mocker.patch(
+        "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+    )
+
+    result = scrape("https://codeforces.com/contest/1900/problem/A")
+
+    assert len(result) == 1
+    assert result[0][0] == "1\n3\n1 2 3"
+    assert result[0][1] == "6"
+
+
+def test_scrape_contest_problems(mocker):
+    mock_scraper = Mock()
+    mock_response = Mock()
+    mock_response.text = """
+    <a href="/contest/1900/problem/A">A. Problem A</a>
+    <a href="/contest/1900/problem/B">B. Problem B</a>
+    """
+    mock_scraper.get.return_value = mock_response
+
+    mocker.patch(
+        "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+    )
+
+    result = scrape_contest_problems("1900")
+
+    assert len(result) == 2
+    assert result[0] == {"id": "a", "name": "A. Problem A"}
+    assert result[1] == {"id": "b", "name": "B. Problem B"}
+
+
+def test_scrape_network_error(mocker):
+    mock_scraper = Mock()
+    mock_scraper.get.side_effect = Exception("Network error")
+
+    mocker.patch(
+        "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+    )
+
+    result = scrape("https://codeforces.com/contest/1900/problem/A")
+
+    assert result == []

tests/scrapers/test_cses.py

@@ -0,0 +1,47 @@
+from unittest.mock import Mock
+import pytest
+from scrapers.cses import scrape, scrape_all_problems
+
+
+def test_scrape_success(mocker, mock_cses_html):
+    mock_response = Mock()
+    mock_response.text = mock_cses_html
+
+    mocker.patch("scrapers.cses.requests.get", return_value=mock_response)
+
+    result = scrape("https://cses.fi/problemset/task/1068")
+
+    assert len(result) == 1
+    assert result[0][0] == "3\n1 2 3"
+    assert result[0][1] == "6"
+
+
+def test_scrape_all_problems(mocker):
+    mock_response = Mock()
+    mock_response.text = """
+    <h1>Introductory Problems</h1>
+    <a href="/problemset/task/1068">Weird Algorithm</a>
+    <a href="/problemset/task/1083">Missing Number</a>
+    <h1>Sorting and Searching</h1>
+    <a href="/problemset/task/1084">Apartments</a>
+    """
+
+    mocker.patch("scrapers.cses.requests.get", return_value=mock_response)
+
+    result = scrape_all_problems()
+
+    assert "Introductory Problems" in result
+    assert "Sorting and Searching" in result
+    assert len(result["Introductory Problems"]) == 2
+    assert result["Introductory Problems"][0] == {
+        "id": "1068",
+        "name": "Weird Algorithm",
+    }
+
+
+def test_scrape_network_error(mocker):
+    mocker.patch("scrapers.cses.requests.get", side_effect=Exception("Network error"))
+
+    result = scrape("https://cses.fi/problemset/task/1068")
+
+    assert result == []

uv.lock

@@ -91,6 +91,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/97/fc88803a451029688dffd7eb446dc1b529657577aec13aceff1cc9628c5d/cloudscraper-1.2.71-py2.py3-none-any.whl", hash = "sha256:76f50ca529ed2279e220837befdec892626f9511708e200d48d5bb76ded679b0", size = 99652, upload-time = "2023-04-25T23:20:15.974Z" },
 ]
 
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
 [[package]]
 name = "idna"
 version = "3.10"
@@ -100,6 +109,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
 ]
 
+[[package]]
+name = "iniconfig"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
+]
+
 [[package]]
 name = "mypy"
 version = "1.18.2"
@@ -147,6 +165,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
 ]
 
+[[package]]
+name = "packaging"
+version = "25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
+]
+
 [[package]]
 name = "pathspec"
 version = "0.12.1"
@@ -156,6 +183,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
 ]
 
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.19.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
+]
+
 [[package]]
 name = "pyparsing"
 version = "3.2.3"
@@ -165,6 +210,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" },
 ]
 
+[[package]]
+name = "pytest"
+version = "8.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+    { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
+]
+
+[[package]]
+name = "pytest-mock"
+version = "3.15.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" },
+]
+
 [[package]]
 name = "requests"
 version = "2.32.5"
@@ -205,6 +278,8 @@ dependencies = [
 [package.dev-dependencies]
 dev = [
     { name = "mypy" },
+    { name = "pytest" },
+    { name = "pytest-mock" },
     { name = "types-beautifulsoup4" },
     { name = "types-requests" },
 ]
@@ -219,6 +294,8 @@ requires-dist = [
 [package.metadata.requires-dev]
 dev = [
     { name = "mypy", specifier = ">=1.18.2" },
+    { name = "pytest", specifier = ">=8.0.0" },
+    { name = "pytest-mock", specifier = ">=3.12.0" },
     { name = "types-beautifulsoup4", specifier = ">=4.12.0.20250516" },
     { name = "types-requests", specifier = ">=2.32.4.20250913" },
 ]