feat(codeforces): implement submit; cache CSES token (#300)
## Problem Codeforces submit was a stub. CSES submit re-ran the full login flow on every invocation (~1.5s overhead). ## Solution **Codeforces**: headless browser submit via StealthySession (same pattern as AtCoder). Solves Cloudflare Turnstile on login, uploads source via file input, caches cookies at `~/.cache/cp-nvim/codeforces-cookies.json` so repeat submits skip login. **CSES**: persist the API token in credentials via a `credentials` ndjson event. Subsequent submits validate the cached token with a single GET before falling back to full login. Also includes a vimdoc table of contents.
This commit is contained in:
Barrett Ruth
GitHub
parent
e9f72dfbbc
commit
6fcb5d1bbc
9 changed files with 307 additions and 96 deletions
|
|
@ -3,6 +3,37 @@
|
|||
Author: Barrett Ruth <br.barrettruth@gmail.com>
|
||||
License: Same terms as Vim itself (see |license|)
|
||||
|
||||
==============================================================================
|
||||
CONTENTS *cp-contents*
|
||||
|
||||
1. Introduction .................................................. |cp.nvim|
|
||||
2. Requirements ........................................ |cp-requirements|
|
||||
3. Setup ........................................................ |cp-setup|
|
||||
4. Configuration ................................................ |cp-config|
|
||||
5. Commands .................................................. |cp-commands|
|
||||
6. Mappings .................................................. |cp-mappings|
|
||||
7. Language Selection .................................. |cp-lang-selection|
|
||||
8. Workflow .................................................. |cp-workflow|
|
||||
9. Workflow Example ............................................ |cp-example|
|
||||
10. Verdict Formatting ................................. |cp-verdict-format|
|
||||
11. Picker Integration .......................................... |cp-picker|
|
||||
12. Picker Keymaps ........................................ |cp-picker-keys|
|
||||
13. Panel ........................................................ |cp-panel|
|
||||
14. Interactive Mode .......................................... |cp-interact|
|
||||
15. Stress Testing .............................................. |cp-stress|
|
||||
16. Race .......................................................... |cp-race|
|
||||
17. Credentials ............................................ |cp-credentials|
|
||||
18. Submit ...................................................... |cp-submit|
|
||||
19. ANSI Colors ................................................... |cp-ansi|
|
||||
20. Highlight Groups ........................................ |cp-highlights|
|
||||
21. Terminal Colors .................................... |cp-terminal-colors|
|
||||
22. Highlight Customization .......................... |cp-highlight-custom|
|
||||
23. Helpers .................................................... |cp-helpers|
|
||||
24. Statusline Integration .................................. |cp-statusline|
|
||||
25. Panel Keymaps .......................................... |cp-panel-keys|
|
||||
26. File Structure ................................................ |cp-files|
|
||||
27. Health Check ................................................ |cp-health|
|
||||
|
||||
==============================================================================
|
||||
INTRODUCTION *cp.nvim*
|
||||
|
||||
|
|
|
|||
|
|
@ -29,11 +29,18 @@ from .models import (
|
|||
TestCase,
|
||||
TestsResult,
|
||||
)
|
||||
from .timeouts import (
|
||||
BROWSER_ELEMENT_WAIT,
|
||||
BROWSER_NAV_TIMEOUT,
|
||||
BROWSER_SESSION_TIMEOUT,
|
||||
BROWSER_SETTLE_DELAY,
|
||||
BROWSER_TURNSTILE_POLL,
|
||||
HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
MIB_TO_MB = 1.048576
|
||||
BASE_URL = "https://atcoder.jp"
|
||||
ARCHIVE_URL = f"{BASE_URL}/contests/archive"
|
||||
TIMEOUT_SECONDS = 30
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
|
@ -76,7 +83,7 @@ def _retry_after_requests(details):
|
|||
on_backoff=_retry_after_requests,
|
||||
)
|
||||
def _fetch(url: str) -> str:
|
||||
r = _session.get(url, headers=HEADERS, timeout=TIMEOUT_SECONDS)
|
||||
r = _session.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
if r.status_code in RETRY_STATUS:
|
||||
raise requests.HTTPError(response=r)
|
||||
r.raise_for_status()
|
||||
|
|
@ -99,7 +106,7 @@ def _giveup_httpx(exc: Exception) -> bool:
|
|||
giveup=_giveup_httpx,
|
||||
)
|
||||
async def _get_async(client: httpx.AsyncClient, url: str) -> str:
|
||||
r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_SECONDS)
|
||||
r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
|
@ -239,14 +246,14 @@ _TURNSTILE_JS = "() => { const el = document.querySelector('[name=\"cf-turnstile
|
|||
|
||||
|
||||
def _solve_turnstile(page) -> None:
|
||||
if page.evaluate(_TURNSTILE_JS):
|
||||
return
|
||||
iframe_loc = page.locator('iframe[src*="challenges.cloudflare.com"]')
|
||||
if not iframe_loc.count():
|
||||
return
|
||||
for _ in range(6):
|
||||
has_token = page.evaluate(_TURNSTILE_JS)
|
||||
if has_token:
|
||||
return
|
||||
try:
|
||||
box = page.locator(
|
||||
'iframe[src*="challenges.cloudflare.com"]'
|
||||
).first.bounding_box()
|
||||
box = iframe_loc.first.bounding_box()
|
||||
if box:
|
||||
page.mouse.click(
|
||||
box["x"] + box["width"] * 0.15,
|
||||
|
|
@ -255,7 +262,7 @@ def _solve_turnstile(page) -> None:
|
|||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.wait_for_function(_TURNSTILE_JS, timeout=5000)
|
||||
page.wait_for_function(_TURNSTILE_JS, timeout=BROWSER_TURNSTILE_POLL)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -331,7 +338,9 @@ def _submit_headless(
|
|||
page.fill('input[name="username"]', credentials.get("username", ""))
|
||||
page.fill('input[name="password"]', credentials.get("password", ""))
|
||||
page.click("#submit")
|
||||
page.wait_for_url(lambda url: "/login" not in url, timeout=60000)
|
||||
page.wait_for_url(
|
||||
lambda url: "/login" not in url, timeout=BROWSER_NAV_TIMEOUT
|
||||
)
|
||||
except Exception as e:
|
||||
login_error = str(e)
|
||||
|
||||
|
|
@ -345,7 +354,7 @@ def _submit_headless(
|
|||
)
|
||||
page.locator(
|
||||
f'select[name="data.LanguageId"] option[value="{language_id}"]'
|
||||
).wait_for(state="attached", timeout=15000)
|
||||
).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT)
|
||||
page.select_option('select[name="data.LanguageId"]', language_id)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".cpp", delete=False, prefix="atcoder_"
|
||||
|
|
@ -354,18 +363,20 @@ def _submit_headless(
|
|||
tmp_path = tf.name
|
||||
try:
|
||||
page.set_input_files("#input-open-file", tmp_path)
|
||||
page.wait_for_timeout(500)
|
||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
page.locator('button[type="submit"]').click()
|
||||
page.wait_for_url(lambda url: "/submissions/me" in url, timeout=60000)
|
||||
page.wait_for_url(
|
||||
lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT
|
||||
)
|
||||
except Exception as e:
|
||||
submit_error = str(e)
|
||||
|
||||
try:
|
||||
with StealthySession(
|
||||
headless=True,
|
||||
timeout=60000,
|
||||
timeout=BROWSER_SESSION_TIMEOUT,
|
||||
google_search=False,
|
||||
cookies=saved_cookies,
|
||||
) as session:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import httpx
|
|||
from curl_cffi import requests as curl_requests
|
||||
|
||||
from .base import BaseScraper, extract_precision
|
||||
from .timeouts import HTTP_TIMEOUT
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
|
|
@ -26,7 +27,6 @@ PROBLEM_URL = "https://www.codechef.com/problems/{problem_id}"
|
|||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
TIMEOUT_S = 15.0
|
||||
CONNECTIONS = 8
|
||||
MEMORY_LIMIT_RE = re.compile(
|
||||
r"Memory\s+[Ll]imit.*?([0-9.]+)\s*(MB|GB)", re.IGNORECASE | re.DOTALL
|
||||
|
|
@ -34,7 +34,7 @@ MEMORY_LIMIT_RE = re.compile(
|
|||
|
||||
|
||||
async def fetch_json(client: httpx.AsyncClient, path: str) -> dict:
|
||||
r = await client.get(BASE_URL + path, headers=HEADERS, timeout=TIMEOUT_S)
|
||||
r = await client.get(BASE_URL + path, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
|
@ -51,7 +51,7 @@ def _extract_memory_limit(html: str) -> float:
|
|||
|
||||
|
||||
def _fetch_html_sync(url: str) -> str:
|
||||
response = curl_requests.get(url, impersonate="chrome", timeout=TIMEOUT_S)
|
||||
response = curl_requests.get(url, impersonate="chrome", timeout=HTTP_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
|
@ -18,10 +20,16 @@ from .models import (
|
|||
SubmitResult,
|
||||
TestCase,
|
||||
)
|
||||
from .timeouts import (
|
||||
BROWSER_ELEMENT_WAIT,
|
||||
BROWSER_NAV_TIMEOUT,
|
||||
BROWSER_SESSION_TIMEOUT,
|
||||
BROWSER_SETTLE_DELAY,
|
||||
HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
BASE_URL = "https://codeforces.com"
|
||||
API_CONTEST_LIST_URL = f"{BASE_URL}/api/contest.list"
|
||||
TIMEOUT_SECONDS = 30
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
|
@ -136,7 +144,7 @@ def _is_interactive(block: Tag) -> bool:
|
|||
|
||||
def _fetch_problems_html(contest_id: str) -> str:
|
||||
url = f"{BASE_URL}/contest/{contest_id}/problems"
|
||||
response = curl_requests.get(url, impersonate="chrome", timeout=TIMEOUT_SECONDS)
|
||||
response = curl_requests.get(url, impersonate="chrome", timeout=HTTP_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
|
|
@ -223,7 +231,7 @@ class CodeforcesScraper(BaseScraper):
|
|||
|
||||
async def scrape_contest_list(self) -> ContestListResult:
|
||||
try:
|
||||
r = requests.get(API_CONTEST_LIST_URL, timeout=TIMEOUT_SECONDS)
|
||||
r = requests.get(API_CONTEST_LIST_URL, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
if data.get("status") != "OK":
|
||||
|
|
@ -289,13 +297,187 @@ class CodeforcesScraper(BaseScraper):
|
|||
language_id: str,
|
||||
credentials: dict[str, str],
|
||||
) -> SubmitResult:
|
||||
return await asyncio.to_thread(
|
||||
_submit_headless,
|
||||
contest_id,
|
||||
problem_id,
|
||||
source_code,
|
||||
language_id,
|
||||
credentials,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_gate_reload(page, wait_selector: str) -> None:
|
||||
from .atcoder import _solve_turnstile
|
||||
|
||||
if "Verification" not in page.title():
|
||||
return
|
||||
_solve_turnstile(page)
|
||||
page.wait_for_function(
|
||||
f"() => !!document.querySelector('{wait_selector}')",
|
||||
timeout=BROWSER_ELEMENT_WAIT,
|
||||
)
|
||||
|
||||
|
||||
def _submit_headless(
|
||||
contest_id: str,
|
||||
problem_id: str,
|
||||
source_code: str,
|
||||
language_id: str,
|
||||
credentials: dict[str, str],
|
||||
) -> SubmitResult:
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from scrapling.fetchers import StealthySession # type: ignore[import-untyped,unresolved-import]
|
||||
except ImportError:
|
||||
return SubmitResult(
|
||||
success=False,
|
||||
error="Codeforces submit not yet implemented",
|
||||
submission_id="",
|
||||
verdict="",
|
||||
error="scrapling is required for Codeforces submit",
|
||||
)
|
||||
|
||||
from .atcoder import _ensure_browser, _solve_turnstile
|
||||
|
||||
_ensure_browser()
|
||||
|
||||
cookie_cache = Path.home() / ".cache" / "cp-nvim" / "codeforces-cookies.json"
|
||||
cookie_cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
saved_cookies: list[dict[str, Any]] = []
|
||||
if cookie_cache.exists():
|
||||
try:
|
||||
saved_cookies = json.loads(cookie_cache.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logged_in = False
|
||||
login_error: str | None = None
|
||||
submit_error: str | None = None
|
||||
|
||||
def check_login(page):
|
||||
nonlocal logged_in
|
||||
logged_in = page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('a'))"
|
||||
".some(a => a.textContent.includes('Logout'))"
|
||||
)
|
||||
|
||||
def login_action(page):
|
||||
nonlocal login_error
|
||||
try:
|
||||
_wait_for_gate_reload(page, "#enterForm")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.fill(
|
||||
'input[name="handleOrEmail"]',
|
||||
credentials.get("username", ""),
|
||||
)
|
||||
page.fill(
|
||||
'input[name="password"]',
|
||||
credentials.get("password", ""),
|
||||
)
|
||||
page.locator('#enterForm input[type="submit"]').click()
|
||||
page.wait_for_url(
|
||||
lambda url: "/enter" not in url, timeout=BROWSER_NAV_TIMEOUT
|
||||
)
|
||||
except Exception as e:
|
||||
login_error = str(e)
|
||||
|
||||
def submit_action(page):
|
||||
nonlocal submit_error
|
||||
try:
|
||||
_solve_turnstile(page)
|
||||
except Exception:
|
||||
pass
|
||||
tmp_path: str | None = None
|
||||
try:
|
||||
page.select_option(
|
||||
'select[name="submittedProblemIndex"]',
|
||||
problem_id.upper(),
|
||||
)
|
||||
page.select_option('select[name="programTypeId"]', language_id)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".cpp", delete=False, prefix="cf_"
|
||||
) as tf:
|
||||
tf.write(source_code)
|
||||
tmp_path = tf.name
|
||||
try:
|
||||
page.set_input_files('input[name="sourceFile"]', tmp_path)
|
||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||
except Exception:
|
||||
page.fill('textarea[name="source"]', source_code)
|
||||
page.locator("form.submit-form input.submit").click(no_wait_after=True)
|
||||
try:
|
||||
page.wait_for_url(
|
||||
lambda url: "/my" in url or "/status" in url,
|
||||
timeout=BROWSER_NAV_TIMEOUT * 2,
|
||||
)
|
||||
except Exception:
|
||||
err_el = page.query_selector("span.error")
|
||||
if err_el:
|
||||
submit_error = err_el.inner_text().strip()
|
||||
else:
|
||||
submit_error = "Submit failed: page did not navigate"
|
||||
except Exception as e:
|
||||
submit_error = str(e)
|
||||
finally:
|
||||
if tmp_path:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
with StealthySession(
|
||||
headless=True,
|
||||
timeout=BROWSER_SESSION_TIMEOUT,
|
||||
google_search=False,
|
||||
cookies=saved_cookies,
|
||||
) as session:
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/",
|
||||
page_action=check_login,
|
||||
network_idle=True,
|
||||
)
|
||||
|
||||
if not logged_in:
|
||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/enter",
|
||||
page_action=login_action,
|
||||
solve_cloudflare=True,
|
||||
)
|
||||
if login_error:
|
||||
return SubmitResult(
|
||||
success=False, error=f"Login failed: {login_error}"
|
||||
)
|
||||
|
||||
print(json.dumps({"status": "submitting"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/contest/{contest_id}/submit",
|
||||
page_action=submit_action,
|
||||
solve_cloudflare=True,
|
||||
)
|
||||
|
||||
try:
|
||||
browser_cookies = session.context.cookies()
|
||||
if any(c["name"] == "JSESSIONID" for c in browser_cookies):
|
||||
cookie_cache.write_text(json.dumps(browser_cookies))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if submit_error:
|
||||
return SubmitResult(success=False, error=submit_error)
|
||||
|
||||
return SubmitResult(
|
||||
success=True,
|
||||
error="",
|
||||
submission_id="",
|
||||
verdict="submitted",
|
||||
)
|
||||
except Exception as e:
|
||||
return SubmitResult(success=False, error=str(e))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
CodeforcesScraper().run_cli()
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import Any
|
|||
import httpx
|
||||
|
||||
from .base import BaseScraper, extract_precision
|
||||
from .timeouts import HTTP_TIMEOUT, SUBMIT_POLL_TIMEOUT
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
|
|
@ -26,7 +27,6 @@ TASK_PATH = "/problemset/task/{id}"
|
|||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
TIMEOUT_S = 15.0
|
||||
CONNECTIONS = 8
|
||||
|
||||
CSES_LANGUAGES: dict[str, dict[str, str]] = {
|
||||
|
|
@ -78,7 +78,7 @@ def snake_to_title(name: str) -> str:
|
|||
|
||||
|
||||
async def fetch_text(client: httpx.AsyncClient, path: str) -> str:
|
||||
r = await client.get(BASE_URL + path, headers=HEADERS, timeout=TIMEOUT_S)
|
||||
r = await client.get(BASE_URL + path, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
|
@ -290,7 +290,7 @@ class CSESScraper(BaseScraper):
|
|||
password: str,
|
||||
) -> str | None:
|
||||
login_page = await client.get(
|
||||
f"{BASE_URL}/login", headers=HEADERS, timeout=TIMEOUT_S
|
||||
f"{BASE_URL}/login", headers=HEADERS, timeout=HTTP_TIMEOUT
|
||||
)
|
||||
csrf_match = re.search(r'name="csrf_token" value="([^"]+)"', login_page.text)
|
||||
if not csrf_match:
|
||||
|
|
@ -304,20 +304,20 @@ class CSESScraper(BaseScraper):
|
|||
"pass": password,
|
||||
},
|
||||
headers=HEADERS,
|
||||
timeout=TIMEOUT_S,
|
||||
timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
if "Invalid username or password" in login_resp.text:
|
||||
return None
|
||||
|
||||
api_resp = await client.post(
|
||||
f"{API_URL}/login", headers=HEADERS, timeout=TIMEOUT_S
|
||||
f"{API_URL}/login", headers=HEADERS, timeout=HTTP_TIMEOUT
|
||||
)
|
||||
api_data = api_resp.json()
|
||||
token: str = api_data["X-Auth-Token"]
|
||||
auth_url: str = api_data["authentication_url"]
|
||||
|
||||
auth_page = await client.get(auth_url, headers=HEADERS, timeout=TIMEOUT_S)
|
||||
auth_page = await client.get(auth_url, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
auth_csrf = re.search(r'name="csrf_token" value="([^"]+)"', auth_page.text)
|
||||
form_token = re.search(r'name="token" value="([^"]+)"', auth_page.text)
|
||||
if not auth_csrf or not form_token:
|
||||
|
|
@ -330,18 +330,29 @@ class CSESScraper(BaseScraper):
|
|||
"token": form_token.group(1),
|
||||
},
|
||||
headers=HEADERS,
|
||||
timeout=TIMEOUT_S,
|
||||
timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
check = await client.get(
|
||||
f"{API_URL}/login",
|
||||
headers={"X-Auth-Token": token, **HEADERS},
|
||||
timeout=TIMEOUT_S,
|
||||
timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
if check.status_code != 200:
|
||||
return None
|
||||
return token
|
||||
|
||||
async def _check_token(self, client: httpx.AsyncClient, token: str) -> bool:
|
||||
try:
|
||||
r = await client.get(
|
||||
f"{API_URL}/login",
|
||||
headers={"X-Auth-Token": token, **HEADERS},
|
||||
timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
return r.status_code == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def submit(
|
||||
self,
|
||||
contest_id: str,
|
||||
|
|
@ -356,11 +367,30 @@ class CSESScraper(BaseScraper):
|
|||
return self._submit_error("Missing credentials. Use :CP login cses")
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||
token = credentials.get("token")
|
||||
|
||||
if token:
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
if not await self._check_token(client, token):
|
||||
token = None
|
||||
|
||||
token = await self._web_login(client, username, password)
|
||||
if not token:
|
||||
return self._submit_error("Login failed (bad credentials?)")
|
||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||
token = await self._web_login(client, username, password)
|
||||
if not token:
|
||||
return self._submit_error("Login failed (bad credentials?)")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"credentials": {
|
||||
"username": username,
|
||||
"password": password,
|
||||
"token": token,
|
||||
}
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print(json.dumps({"status": "submitting"}), flush=True)
|
||||
|
||||
|
|
@ -383,7 +413,7 @@ class CSESScraper(BaseScraper):
|
|||
"Content-Type": "application/json",
|
||||
**HEADERS,
|
||||
},
|
||||
timeout=TIMEOUT_S,
|
||||
timeout=HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
if r.status_code not in range(200, 300):
|
||||
|
|
@ -406,7 +436,7 @@ class CSESScraper(BaseScraper):
|
|||
"X-Auth-Token": token,
|
||||
**HEADERS,
|
||||
},
|
||||
timeout=30.0,
|
||||
timeout=SUBMIT_POLL_TIMEOUT,
|
||||
)
|
||||
if r.status_code == 200:
|
||||
info = r.json()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from datetime import datetime
|
|||
import httpx
|
||||
|
||||
from .base import BaseScraper
|
||||
from .timeouts import HTTP_TIMEOUT
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
|
|
@ -23,7 +24,6 @@ BASE_URL = "https://open.kattis.com"
|
|||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
TIMEOUT_S = 15.0
|
||||
CONNECTIONS = 8
|
||||
|
||||
TIME_RE = re.compile(
|
||||
|
|
@ -37,13 +37,13 @@ MEM_RE = re.compile(
|
|||
|
||||
|
||||
async def _fetch_text(client: httpx.AsyncClient, url: str) -> str:
|
||||
r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S)
|
||||
r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
||||
async def _fetch_bytes(client: httpx.AsyncClient, url: str) -> bytes:
|
||||
r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S)
|
||||
r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT)
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
|
||||
|
|
|
|||
9
scrapers/timeouts.py
Normal file
9
scrapers/timeouts.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
HTTP_TIMEOUT = 15.0
|
||||
|
||||
BROWSER_SESSION_TIMEOUT = 15000
|
||||
BROWSER_NAV_TIMEOUT = 10000
|
||||
BROWSER_TURNSTILE_POLL = 5000
|
||||
BROWSER_ELEMENT_WAIT = 10000
|
||||
BROWSER_SETTLE_DELAY = 500
|
||||
|
||||
SUBMIT_POLL_TIMEOUT = 30.0
|
||||
|
|
@ -8,6 +8,7 @@ from typing import Any, cast
|
|||
import httpx
|
||||
|
||||
from .base import BaseScraper
|
||||
from .timeouts import HTTP_TIMEOUT
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
|
|
@ -21,7 +22,6 @@ BASE_URL = "http://www.usaco.org"
|
|||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
TIMEOUT_S = 15.0
|
||||
CONNECTIONS = 4
|
||||
|
||||
MONTHS = [
|
||||
|
|
@ -58,7 +58,9 @@ RESULTS_PAGE_RE = re.compile(
|
|||
|
||||
|
||||
async def _fetch_text(client: httpx.AsyncClient, url: str) -> str:
|
||||
r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S, follow_redirects=True)
|
||||
r = await client.get(
|
||||
url, headers=HEADERS, timeout=HTTP_TIMEOUT, follow_redirects=True
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
|
|
|||
54
t/1068.cc
54
t/1068.cc
|
|
@ -1,54 +0,0 @@
|
|||
#include <bits/stdc++.h> // {{{
|
||||
|
||||
#include <version>
|
||||
#ifdef __cpp_lib_ranges_enumerate
|
||||
#include <ranges>
|
||||
namespace rv = std::views;
|
||||
namespace rs = std::ranges;
|
||||
#endif
|
||||
|
||||
#pragma GCC optimize("O2,unroll-loops")
|
||||
#pragma GCC target("avx2,bmi,bmi2,lzcnt,popcnt")
|
||||
|
||||
using namespace std;
|
||||
|
||||
using i32 = int32_t;
|
||||
using u32 = uint32_t;
|
||||
using i64 = int64_t;
|
||||
using u64 = uint64_t;
|
||||
using f64 = double;
|
||||
using f128 = long double;
|
||||
|
||||
#if __cplusplus >= 202002L
|
||||
template <typename T>
|
||||
constexpr T MIN = std::numeric_limits<T>::min();
|
||||
|
||||
template <typename T>
|
||||
constexpr T MAX = std::numeric_limits<T>::max();
|
||||
#endif
|
||||
|
||||
#ifdef LOCAL
|
||||
#define db(...) std::print(__VA_ARGS__)
|
||||
#define dbln(...) std::println(__VA_ARGS__)
|
||||
#else
|
||||
#define db(...)
|
||||
#define dbln(...)
|
||||
#endif
|
||||
// }}}
|
||||
|
||||
void solve() {
|
||||
cout << "hi\n";
|
||||
}
|
||||
|
||||
int main() { // {{{
|
||||
std::cin.exceptions(std::cin.failbit);
|
||||
#ifdef LOCAL
|
||||
std::cerr.rdbuf(std::cout.rdbuf());
|
||||
std::cout.setf(std::ios::unitbuf);
|
||||
std::cerr.setf(std::ios::unitbuf);
|
||||
#else
|
||||
std::cin.tie(nullptr)->sync_with_stdio(false);
|
||||
#endif
|
||||
solve();
|
||||
return 0;
|
||||
} // }}}
|
||||
Loading…
Add table
Add a link
Reference in a new issue