diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 50c7b00..e8a6034 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -51,7 +51,7 @@ jobs:
- name: Install ruff
run: uv tool install ruff
- name: Check Python formatting with ruff
- run: ruff format --check scrapers/
+ run: ruff format --check scrapers/ tests/scrapers/
python-lint:
name: Python Linting
@@ -75,4 +75,16 @@ jobs:
- name: Install dependencies with mypy
run: uv sync --dev
- name: Type check Python files with mypy
- run: uv run mypy scrapers/
+ run: uv run mypy scrapers/ tests/scrapers/
+
+ python-test:
+ name: Python Testing
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Install uv
+ uses: astral-sh/setup-uv@v4
+ - name: Install dependencies with pytest
+ run: uv sync --dev
+ - name: Run Python tests
+ run: uv run pytest tests/scrapers/ -v
diff --git a/pyproject.toml b/pyproject.toml
index 6c796e3..00b350b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,4 +15,9 @@ dev = [
"mypy>=1.18.2",
"types-beautifulsoup4>=4.12.0.20250516",
"types-requests>=2.32.4.20250913",
+ "pytest>=8.0.0",
+ "pytest-mock>=3.12.0",
]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]
diff --git a/scrapers/__init__.py b/scrapers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/scrapers/conftest.py b/tests/scrapers/conftest.py
new file mode 100644
index 0000000..3248ec2
--- /dev/null
+++ b/tests/scrapers/conftest.py
@@ -0,0 +1,41 @@
+import pytest
+
+
+@pytest.fixture
+def mock_codeforces_html():
+ return """
+
+
+ """
+
+
+@pytest.fixture
+def mock_atcoder_html():
+ return """
+ Sample Input 1
+ 3
+1 2 3
+ Sample Output 1
+ 6
+ """
+
+
+@pytest.fixture
+def mock_cses_html():
+ return """
+ Example
+ Input:
+ 3
+1 2 3
+ Output:
+ 6
+ """
diff --git a/tests/scrapers/test_atcoder.py b/tests/scrapers/test_atcoder.py
new file mode 100644
index 0000000..6b0fa40
--- /dev/null
+++ b/tests/scrapers/test_atcoder.py
@@ -0,0 +1,51 @@
+from unittest.mock import Mock
+import pytest
+from scrapers.atcoder import scrape, scrape_contest_problems
+
+
+def test_scrape_success(mocker, mock_atcoder_html):
+ mock_response = Mock()
+ mock_response.text = mock_atcoder_html
+
+ mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response)
+
+ result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a")
+
+ assert len(result) == 1
+ assert result[0][0] == "3\n1 2 3"
+ assert result[0][1] == "6"
+
+
+def test_scrape_contest_problems(mocker):
+ mock_response = Mock()
+ mock_response.text = """
+
+ """
+
+ mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response)
+
+ result = scrape_contest_problems("abc350")
+
+ assert len(result) == 2
+ assert result[0] == {"id": "a", "name": "A - Water Tank"}
+ assert result[1] == {"id": "b", "name": "B - Dentist Aoki"}
+
+
+def test_scrape_network_error(mocker):
+ mocker.patch(
+ "scrapers.atcoder.requests.get", side_effect=Exception("Network error")
+ )
+
+ result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a")
+
+ assert result == []
diff --git a/tests/scrapers/test_codeforces.py b/tests/scrapers/test_codeforces.py
new file mode 100644
index 0000000..381cc93
--- /dev/null
+++ b/tests/scrapers/test_codeforces.py
@@ -0,0 +1,54 @@
+import json
+from unittest.mock import Mock
+import pytest
+from scrapers.codeforces import scrape, scrape_contest_problems
+
+
+def test_scrape_success(mocker, mock_codeforces_html):
+ mock_scraper = Mock()
+ mock_response = Mock()
+ mock_response.text = mock_codeforces_html
+ mock_scraper.get.return_value = mock_response
+
+ mocker.patch(
+ "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+ )
+
+ result = scrape("https://codeforces.com/contest/1900/problem/A")
+
+ assert len(result) == 1
+ assert result[0][0] == "1\n3\n1 2 3"
+ assert result[0][1] == "6"
+
+
+def test_scrape_contest_problems(mocker):
+ mock_scraper = Mock()
+ mock_response = Mock()
+ mock_response.text = """
+ A. Problem A
+ B. Problem B
+ """
+ mock_scraper.get.return_value = mock_response
+
+ mocker.patch(
+ "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+ )
+
+ result = scrape_contest_problems("1900")
+
+ assert len(result) == 2
+ assert result[0] == {"id": "a", "name": "A. Problem A"}
+ assert result[1] == {"id": "b", "name": "B. Problem B"}
+
+
+def test_scrape_network_error(mocker):
+ mock_scraper = Mock()
+ mock_scraper.get.side_effect = Exception("Network error")
+
+ mocker.patch(
+ "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper
+ )
+
+ result = scrape("https://codeforces.com/contest/1900/problem/A")
+
+ assert result == []
diff --git a/tests/scrapers/test_cses.py b/tests/scrapers/test_cses.py
new file mode 100644
index 0000000..9df8281
--- /dev/null
+++ b/tests/scrapers/test_cses.py
@@ -0,0 +1,47 @@
+from unittest.mock import Mock
+import pytest
+from scrapers.cses import scrape, scrape_all_problems
+
+
+def test_scrape_success(mocker, mock_cses_html):
+ mock_response = Mock()
+ mock_response.text = mock_cses_html
+
+ mocker.patch("scrapers.cses.requests.get", return_value=mock_response)
+
+ result = scrape("https://cses.fi/problemset/task/1068")
+
+ assert len(result) == 1
+ assert result[0][0] == "3\n1 2 3"
+ assert result[0][1] == "6"
+
+
+def test_scrape_all_problems(mocker):
+ mock_response = Mock()
+ mock_response.text = """
+ Introductory Problems
+ Weird Algorithm
+ Missing Number
+ Sorting and Searching
+ Apartments
+ """
+
+ mocker.patch("scrapers.cses.requests.get", return_value=mock_response)
+
+ result = scrape_all_problems()
+
+ assert "Introductory Problems" in result
+ assert "Sorting and Searching" in result
+ assert len(result["Introductory Problems"]) == 2
+ assert result["Introductory Problems"][0] == {
+ "id": "1068",
+ "name": "Weird Algorithm",
+ }
+
+
+def test_scrape_network_error(mocker):
+ mocker.patch("scrapers.cses.requests.get", side_effect=Exception("Network error"))
+
+ result = scrape("https://cses.fi/problemset/task/1068")
+
+ assert result == []
diff --git a/uv.lock b/uv.lock
index dfef7b6..bfe4d6d 100644
--- a/uv.lock
+++ b/uv.lock
@@ -91,6 +91,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/81/97/fc88803a451029688dffd7eb446dc1b529657577aec13aceff1cc9628c5d/cloudscraper-1.2.71-py2.py3-none-any.whl", hash = "sha256:76f50ca529ed2279e220837befdec892626f9511708e200d48d5bb76ded679b0", size = 99652, upload-time = "2023-04-25T23:20:15.974Z" },
]
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
[[package]]
name = "idna"
version = "3.10"
@@ -100,6 +109,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
]
+[[package]]
+name = "iniconfig"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
+]
+
[[package]]
name = "mypy"
version = "1.18.2"
@@ -147,6 +165,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
]
+[[package]]
+name = "packaging"
+version = "25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
+]
+
[[package]]
name = "pathspec"
version = "0.12.1"
@@ -156,6 +183,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
]
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
+[[package]]
+name = "pygments"
+version = "2.19.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
+]
+
[[package]]
name = "pyparsing"
version = "3.2.3"
@@ -165,6 +210,34 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" },
]
+[[package]]
+name = "pytest"
+version = "8.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "iniconfig" },
+ { name = "packaging" },
+ { name = "pluggy" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
+]
+
+[[package]]
+name = "pytest-mock"
+version = "3.15.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" },
+]
+
[[package]]
name = "requests"
version = "2.32.5"
@@ -205,6 +278,8 @@ dependencies = [
[package.dev-dependencies]
dev = [
{ name = "mypy" },
+ { name = "pytest" },
+ { name = "pytest-mock" },
{ name = "types-beautifulsoup4" },
{ name = "types-requests" },
]
@@ -219,6 +294,8 @@ requires-dist = [
[package.metadata.requires-dev]
dev = [
{ name = "mypy", specifier = ">=1.18.2" },
+ { name = "pytest", specifier = ">=8.0.0" },
+ { name = "pytest-mock", specifier = ">=3.12.0" },
{ name = "types-beautifulsoup4", specifier = ">=4.12.0.20250516" },
{ name = "types-requests", specifier = ">=2.32.4.20250913" },
]