barrett/cp.nvim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: replace curl_cffi with scrapling in codeforces metadata (#334)

Browse source 
## Problem

`codeforces.py` used `curl_cffi` to bypass Cloudflare when fetching
contest problem HTML, making it unavailable in the nix python env and
requiring an extra dependency across `pyproject.toml` and `flake.nix`.

## Solution

Rewrite `_fetch_problems_html` to use scrapling `StealthySession` with
`solve_cloudflare=True`, matching the existing CF submit pattern. Extend
`needs_browser` in `scraper.lua` to route CF `metadata` and `tests`
through the FHS env on NixOS. Remove `curl-cffi` from `pyproject.toml`,
`flake.nix`, and test mocks.
This commit is contained in:
Barrett Ruth 2026-03-06 13:25:44 -05:00 • committed by GitHub
parent 6a3d2fe4f8
commit 0b40e0f33e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 32 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

34
tests/conftest.py
View file

@ -10,7 +10,6 @@ from typing import Any
import httpx
import pytest
import requests
from curl_cffi import requests as curl_requests
ROOT = Path(__file__).resolve().parent.parent
FIX = Path(__file__).resolve().parent / "fixtures"
@ -136,15 +135,10 @@ def run_scraper_offline(fixture_text):
case "codeforces":
class MockCurlResponse:
def __init__(self, html: str):
self.text = html
def raise_for_status(self):
pass
def _mock_curl_get(url: str, **kwargs):
return MockCurlResponse(_router_codeforces(url=url))
def _mock_fetch_problems_html(cid: str) -> str:
return _router_codeforces(
url=f"https://codeforces.com/contest/{cid}/problems"
)
def _mock_requests_get(url: str, **kwargs):
if "api/contest.list" in url:
@ -175,7 +169,7 @@ def run_scraper_offline(fixture_text):
raise AssertionError(f"Unexpected requests.get call: {url}")
return {
"curl_requests.get": _mock_curl_get,
"_fetch_problems_html": _mock_fetch_problems_html,
"requests.get": _mock_requests_get,
}
@ -215,23 +209,8 @@ def run_scraper_offline(fixture_text):
return MockResponse(data)
raise AssertionError(f"No fixture for CodeChef url={url!r}")
class MockCodeChefCurlResponse:
def __init__(self, html: str):
self.text = html
def raise_for_status(self):
pass
def _mock_curl_get(url: str, **kwargs):
if "/problems/" in url:
problem_id = url.rstrip("/").split("/")[-1]
html = fixture_text(f"codechef/{problem_id}.html")
return MockCodeChefCurlResponse(html)
raise AssertionError(f"No fixture for CodeChef url={url!r}")
return {
"__offline_get_async": __offline_get_async,
"curl_requests.get": _mock_curl_get,
}
case _:
@ -250,7 +229,7 @@ def run_scraper_offline(fixture_text):
offline_fetches = _make_offline_fetches(scraper_name)
if scraper_name == "codeforces":
curl_requests.get = offline_fetches["curl_requests.get"]
ns._fetch_problems_html = offline_fetches["_fetch_problems_html"]
requests.get = offline_fetches["requests.get"]
elif scraper_name == "atcoder":
ns._fetch = offline_fetches["_fetch"]
@ -259,7 +238,6 @@ def run_scraper_offline(fixture_text):
httpx.AsyncClient.get = offline_fetches["__offline_fetch_text"]
elif scraper_name == "codechef":
httpx.AsyncClient.get = offline_fetches["__offline_get_async"]
curl_requests.get = offline_fetches["curl_requests.get"]
scraper_class = getattr(ns, scraper_classes[scraper_name])
scraper = scraper_class()