Barrett Ruth
297c71e7c7
Problem: `codeforces.py` used `curl_cffi` to bypass Cloudflare when fetching contest problem HTML, making it unavailable in the nix python env and requiring an extra dependency. Solution: rewrite `_fetch_problems_html` to use scrapling `StealthySession` with `solve_cloudflare=True`, matching the existing CF submit pattern. Extend `needs_browser` in `scraper.lua` to route CF `metadata` and `tests` through the FHS env on NixOS. Remove `curl-cffi` from `pyproject.toml`, `flake.nix`, and test mocks.
33 lines
714 B
TOML
33 lines
714 B
TOML
[project]
|
|
name = "scrapers"
|
|
version = "0.1.0"
|
|
description = "Competitive programming scrapers for a variety of web platforms."
|
|
readme = "README.md"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"backoff>=2.2.1",
|
|
"beautifulsoup4>=4.13.5",
|
|
"scrapling[fetchers]>=0.4",
|
|
"httpx>=0.28.1",
|
|
"ndjson>=0.3.1",
|
|
"pydantic>=2.11.10",
|
|
"requests>=2.32.5",
|
|
]
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"types-beautifulsoup4>=4.12.0.20250516",
|
|
"types-requests>=2.32.4.20250913",
|
|
"pytest>=8.0.0",
|
|
"pytest-mock>=3.12.0",
|
|
"pre-commit>=4.3.0",
|
|
"basedpyright>=1.31.6",
|
|
"ruff>=0.14.2",
|
|
"ty>=0.0.1a32",
|
|
]
|
|
|
|
[tool.pytest.ini_options]
|
|
pythonpath = ["."]
|
|
|
|
[tool.mypy]
|
|
ignore_missing_imports = true
|