diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50c7b00..e8a6034 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - name: Install ruff run: uv tool install ruff - name: Check Python formatting with ruff - run: ruff format --check scrapers/ + run: ruff format --check scrapers/ tests/scrapers/ python-lint: name: Python Linting @@ -75,4 +75,16 @@ jobs: - name: Install dependencies with mypy run: uv sync --dev - name: Type check Python files with mypy - run: uv run mypy scrapers/ + run: uv run mypy scrapers/ tests/scrapers/ + + python-test: + name: Python Testing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Install dependencies with pytest + run: uv sync --dev + - name: Run Python tests + run: uv run pytest tests/scrapers/ -v diff --git a/pyproject.toml b/pyproject.toml index 6c796e3..00b350b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,4 +15,9 @@ dev = [ "mypy>=1.18.2", "types-beautifulsoup4>=4.12.0.20250516", "types-requests>=2.32.4.20250913", + "pytest>=8.0.0", + "pytest-mock>=3.12.0", ] + +[tool.pytest.ini_options] +pythonpath = ["."] diff --git a/scrapers/__init__.py b/scrapers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scrapers/conftest.py b/tests/scrapers/conftest.py new file mode 100644 index 0000000..3248ec2 --- /dev/null +++ b/tests/scrapers/conftest.py @@ -0,0 +1,41 @@ +import pytest + + +@pytest.fixture +def mock_codeforces_html(): + return """ +
+
+            
3
+
1 2 3
+
+
+
+
+            
6
+
+
+ """ + + +@pytest.fixture +def mock_atcoder_html(): + return """ +

Sample Input 1

+
3
+1 2 3
+

Sample Output 1

+
6
+ """ + + +@pytest.fixture +def mock_cses_html(): + return """ +

Example

+

Input:

+
3
+1 2 3
+

Output:

+
6
+ """ diff --git a/tests/scrapers/test_atcoder.py b/tests/scrapers/test_atcoder.py new file mode 100644 index 0000000..6b0fa40 --- /dev/null +++ b/tests/scrapers/test_atcoder.py @@ -0,0 +1,51 @@ +from unittest.mock import Mock +import pytest +from scrapers.atcoder import scrape, scrape_contest_problems + + +def test_scrape_success(mocker, mock_atcoder_html): + mock_response = Mock() + mock_response.text = mock_atcoder_html + + mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response) + + result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a") + + assert len(result) == 1 + assert result[0][0] == "3\n1 2 3" + assert result[0][1] == "6" + + +def test_scrape_contest_problems(mocker): + mock_response = Mock() + mock_response.text = """ + + + + + + + + + + +
TaskName
A - Water Tank
B - Dentist Aoki
+ """ + + mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response) + + result = scrape_contest_problems("abc350") + + assert len(result) == 2 + assert result[0] == {"id": "a", "name": "A - Water Tank"} + assert result[1] == {"id": "b", "name": "B - Dentist Aoki"} + + +def test_scrape_network_error(mocker): + mocker.patch( + "scrapers.atcoder.requests.get", side_effect=Exception("Network error") + ) + + result = scrape("https://atcoder.jp/contests/abc350/tasks/abc350_a") + + assert result == [] diff --git a/tests/scrapers/test_codeforces.py b/tests/scrapers/test_codeforces.py new file mode 100644 index 0000000..381cc93 --- /dev/null +++ b/tests/scrapers/test_codeforces.py @@ -0,0 +1,54 @@ +import json +from unittest.mock import Mock +import pytest +from scrapers.codeforces import scrape, scrape_contest_problems + + +def test_scrape_success(mocker, mock_codeforces_html): + mock_scraper = Mock() + mock_response = Mock() + mock_response.text = mock_codeforces_html + mock_scraper.get.return_value = mock_response + + mocker.patch( + "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper + ) + + result = scrape("https://codeforces.com/contest/1900/problem/A") + + assert len(result) == 1 + assert result[0][0] == "1\n3\n1 2 3" + assert result[0][1] == "6" + + +def test_scrape_contest_problems(mocker): + mock_scraper = Mock() + mock_response = Mock() + mock_response.text = """ + A. Problem A + B. Problem B + """ + mock_scraper.get.return_value = mock_response + + mocker.patch( + "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper + ) + + result = scrape_contest_problems("1900") + + assert len(result) == 2 + assert result[0] == {"id": "a", "name": "A. Problem A"} + assert result[1] == {"id": "b", "name": "B. Problem B"} + + +def test_scrape_network_error(mocker): + mock_scraper = Mock() + mock_scraper.get.side_effect = Exception("Network error") + + mocker.patch( + "scrapers.codeforces.cloudscraper.create_scraper", return_value=mock_scraper + ) + + result = scrape("https://codeforces.com/contest/1900/problem/A") + + assert result == [] diff --git a/tests/scrapers/test_cses.py b/tests/scrapers/test_cses.py new file mode 100644 index 0000000..9df8281 --- /dev/null +++ b/tests/scrapers/test_cses.py @@ -0,0 +1,47 @@ +from unittest.mock import Mock +import pytest +from scrapers.cses import scrape, scrape_all_problems + + +def test_scrape_success(mocker, mock_cses_html): + mock_response = Mock() + mock_response.text = mock_cses_html + + mocker.patch("scrapers.cses.requests.get", return_value=mock_response) + + result = scrape("https://cses.fi/problemset/task/1068") + + assert len(result) == 1 + assert result[0][0] == "3\n1 2 3" + assert result[0][1] == "6" + + +def test_scrape_all_problems(mocker): + mock_response = Mock() + mock_response.text = """ +

Introductory Problems

+ Weird Algorithm + Missing Number +

Sorting and Searching

+ Apartments + """ + + mocker.patch("scrapers.cses.requests.get", return_value=mock_response) + + result = scrape_all_problems() + + assert "Introductory Problems" in result + assert "Sorting and Searching" in result + assert len(result["Introductory Problems"]) == 2 + assert result["Introductory Problems"][0] == { + "id": "1068", + "name": "Weird Algorithm", + } + + +def test_scrape_network_error(mocker): + mocker.patch("scrapers.cses.requests.get", side_effect=Exception("Network error")) + + result = scrape("https://cses.fi/problemset/task/1068") + + assert result == [] diff --git a/uv.lock b/uv.lock index dfef7b6..bfe4d6d 100644 --- a/uv.lock +++ b/uv.lock @@ -91,6 +91,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/97/fc88803a451029688dffd7eb446dc1b529657577aec13aceff1cc9628c5d/cloudscraper-1.2.71-py2.py3-none-any.whl", hash = "sha256:76f50ca529ed2279e220837befdec892626f9511708e200d48d5bb76ded679b0", size = 99652, upload-time = "2023-04-25T23:20:15.974Z" }, ] +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + [[package]] name = "idna" version = "3.10" @@ -100,6 +109,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + [[package]] name = "mypy" version = "1.18.2" @@ -147,6 +165,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + [[package]] name = "pathspec" version = "0.12.1" @@ -156,6 +183,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + [[package]] name = "pyparsing" version = "3.2.3" @@ -165,6 +210,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, ] +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -205,6 +278,8 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-mock" }, { name = "types-beautifulsoup4" }, { name = "types-requests" }, ] @@ -219,6 +294,8 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "mypy", specifier = ">=1.18.2" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-mock", specifier = ">=3.12.0" }, { name = "types-beautifulsoup4", specifier = ">=4.12.0.20250516" }, { name = "types-requests", specifier = ">=2.32.4.20250913" }, ]