c194f12eee
## Problem `_submit_sync` was a 170-line nested closure with `_solve_turnstile` and the browser-install block further nested inside it. Status events went to stderr, which `run_scraper()` silently discards, leaving the user with a 10–30s silent hang after credential entry. The NDJSON spawn path also lacked stdin support, so submit had no streaming path at all. ## Solution Extract `_TURNSTILE_JS`, `_solve_turnstile`, `_ensure_browser`, and `_submit_headless` to module level in `atcoder.py`; status events (`installing_browser`, `checking_login`, `logging_in`, `submitting`) now print to stdout as NDJSON. Add stdin pipe support to the NDJSON spawn path in `scraper.lua` and switch `M.submit` to streaming with an `on_status` callback. Wire `on_status` in `submit.lua` to fire `vim.notify` for each phase transition.
34 lines
739 B
TOML
34 lines
739 B
TOML
[project]
|
|
name = "scrapers"
|
|
version = "0.1.0"
|
|
description = "Competitive programming scrapers for a variety of web platforms."
|
|
readme = "README.md"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"backoff>=2.2.1",
|
|
"beautifulsoup4>=4.13.5",
|
|
"scrapling[fetchers]>=0.4",
|
|
"curl-cffi>=0.13.0",
|
|
"httpx>=0.28.1",
|
|
"ndjson>=0.3.1",
|
|
"pydantic>=2.11.10",
|
|
"requests>=2.32.5",
|
|
]
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"types-beautifulsoup4>=4.12.0.20250516",
|
|
"types-requests>=2.32.4.20250913",
|
|
"pytest>=8.0.0",
|
|
"pytest-mock>=3.12.0",
|
|
"pre-commit>=4.3.0",
|
|
"basedpyright>=1.31.6",
|
|
"ruff>=0.14.2",
|
|
"ty>=0.0.1a32",
|
|
]
|
|
|
|
[tool.pytest.ini_options]
|
|
pythonpath = ["."]
|
|
|
|
[tool.mypy]
|
|
ignore_missing_imports = true
|