fix(submit): harden atcoder and codeforces submit flow (#304)
## Problem AtCoder file upload always wrote a `.cpp` temp file regardless of language. CF submit used `solve_cloudflare=True` on the submit page, causing a spurious "No Cloudflare challenge found" error; `_wait_for_gate_reload` in `login_action` was dead code. Stale cookies caused silent auth failures with no recovery path. The `uv.spawn` ndjson path for submit had no overall timeout. ## Solution Replace AtCoder's temp file with `page.set_input_files` using an in-memory buffer and correct extension via `_LANGUAGE_ID_EXTENSION`. Replace CF's temp-file/fallback dance with a direct `textarea[name="source"]` fill and set `solve_cloudflare=False` on the submit fetch. Add a login fast-path that skips the homepage check when cookies exist, with automatic stale-cookie recovery via `_retried` flag on redirect-to-login detection. Remove `_wait_for_gate_reload`. Fix `_ensure_browser` to propagate install errors. Add a 120s kill timer to the ndjson `uv.spawn` submit path in `scraper.lua`.
This commit is contained in:
Barrett Ruth
GitHub
parent
6fcb5d1bbc
commit
127089c57f
3 changed files with 122 additions and 96 deletions
|
|
@ -85,6 +85,7 @@ local function run_scraper(platform, subcommand, args, opts)
|
|||
local stderr = uv.new_pipe(false)
|
||||
local buf = ''
|
||||
|
||||
local timer = nil
|
||||
local handle
|
||||
handle = uv.spawn(cmd[1], {
|
||||
args = vim.list_slice(cmd, 2),
|
||||
|
|
@ -92,6 +93,10 @@ local function run_scraper(platform, subcommand, args, opts)
|
|||
env = spawn_env_list(env),
|
||||
cwd = plugin_path,
|
||||
}, function(code, signal)
|
||||
if timer and not timer:is_closing() then
|
||||
timer:stop()
|
||||
timer:close()
|
||||
end
|
||||
if buf ~= '' and opts.on_event then
|
||||
local ok_tail, ev_tail = pcall(vim.json.decode, buf)
|
||||
if ok_tail then
|
||||
|
|
@ -124,6 +129,30 @@ local function run_scraper(platform, subcommand, args, opts)
|
|||
return { success = false, error = 'spawn failed' }
|
||||
end
|
||||
|
||||
if subcommand == 'submit' then
|
||||
timer = uv.new_timer()
|
||||
timer:start(120000, 0, function()
|
||||
timer:stop()
|
||||
timer:close()
|
||||
if stdin_pipe and not stdin_pipe:is_closing() then
|
||||
stdin_pipe:close()
|
||||
end
|
||||
if not stdout:is_closing() then
|
||||
stdout:close()
|
||||
end
|
||||
if not stderr:is_closing() then
|
||||
stderr:close()
|
||||
end
|
||||
if handle and not handle:is_closing() then
|
||||
handle:kill(15)
|
||||
handle:close()
|
||||
end
|
||||
if opts.on_exit then
|
||||
opts.on_exit({ success = false, error = 'submit timed out' })
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
if stdin_pipe then
|
||||
uv.write(stdin_pipe, opts.stdin, function()
|
||||
uv.shutdown(stdin_pipe, function()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import os
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -38,6 +37,11 @@ from .timeouts import (
|
|||
HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
_LANGUAGE_ID_EXTENSION = {
|
||||
"6017": "cc",
|
||||
"6082": "py",
|
||||
}
|
||||
|
||||
MIB_TO_MB = 1.048576
|
||||
BASE_URL = "https://atcoder.jp"
|
||||
ARCHIVE_URL = f"{BASE_URL}/contests/archive"
|
||||
|
|
@ -274,23 +278,20 @@ def _ensure_browser() -> None:
|
|||
from patchright._impl._driver import compute_driver_executable # type: ignore[import-untyped,unresolved-import]
|
||||
|
||||
node, cli = compute_driver_executable()
|
||||
browser_info = subprocess.run(
|
||||
[node, cli, "install", "--dry-run", "chromium"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for line in browser_info.stdout.splitlines():
|
||||
if "Install location:" in line:
|
||||
install_dir = line.split(":", 1)[1].strip()
|
||||
if not os.path.isdir(install_dir):
|
||||
print(json.dumps({"status": "installing_browser"}), flush=True)
|
||||
subprocess.run(
|
||||
[node, cli, "install", "chromium"],
|
||||
check=True,
|
||||
)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
browser_info = subprocess.run(
|
||||
[node, cli, "install", "--dry-run", "chromium"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
for line in browser_info.stdout.splitlines():
|
||||
if "Install location:" in line:
|
||||
install_dir = line.split(":", 1)[1].strip()
|
||||
if not os.path.isdir(install_dir):
|
||||
print(json.dumps({"status": "installing_browser"}), flush=True)
|
||||
subprocess.run([node, cli, "install", "chromium"], check=True)
|
||||
break
|
||||
|
||||
|
||||
def _submit_headless(
|
||||
|
|
@ -299,6 +300,7 @@ def _submit_headless(
|
|||
source_code: str,
|
||||
language_id: str,
|
||||
credentials: dict[str, str],
|
||||
_retried: bool = False,
|
||||
) -> "SubmitResult":
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -321,9 +323,10 @@ def _submit_headless(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
logged_in = False
|
||||
logged_in = cookie_cache.exists() and not _retried
|
||||
login_error: str | None = None
|
||||
submit_error: str | None = None
|
||||
needs_relogin = False
|
||||
|
||||
def check_login(page):
|
||||
nonlocal logged_in
|
||||
|
|
@ -345,7 +348,10 @@ def _submit_headless(
|
|||
login_error = str(e)
|
||||
|
||||
def submit_action(page):
|
||||
nonlocal submit_error
|
||||
nonlocal submit_error, needs_relogin
|
||||
if "/login" in page.url:
|
||||
needs_relogin = True
|
||||
return
|
||||
try:
|
||||
_solve_turnstile(page)
|
||||
page.select_option(
|
||||
|
|
@ -356,16 +362,16 @@ def _submit_headless(
|
|||
f'select[name="data.LanguageId"] option[value="{language_id}"]'
|
||||
).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT)
|
||||
page.select_option('select[name="data.LanguageId"]', language_id)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".cpp", delete=False, prefix="atcoder_"
|
||||
) as tf:
|
||||
tf.write(source_code)
|
||||
tmp_path = tf.name
|
||||
try:
|
||||
page.set_input_files("#input-open-file", tmp_path)
|
||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
ext = _LANGUAGE_ID_EXTENSION.get(language_id, "txt")
|
||||
page.set_input_files(
|
||||
"#input-open-file",
|
||||
{
|
||||
"name": f"solution.{ext}",
|
||||
"mimeType": "text/plain",
|
||||
"buffer": source_code.encode(),
|
||||
},
|
||||
)
|
||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||
page.locator('button[type="submit"]').click()
|
||||
page.wait_for_url(
|
||||
lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT
|
||||
|
|
@ -378,14 +384,13 @@ def _submit_headless(
|
|||
headless=True,
|
||||
timeout=BROWSER_SESSION_TIMEOUT,
|
||||
google_search=False,
|
||||
cookies=saved_cookies,
|
||||
cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [],
|
||||
) as session:
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/home",
|
||||
page_action=check_login,
|
||||
network_idle=True,
|
||||
)
|
||||
if not (cookie_cache.exists() and not _retried):
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/home", page_action=check_login, network_idle=True
|
||||
)
|
||||
|
||||
if not logged_in:
|
||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||
|
|
@ -413,12 +418,23 @@ def _submit_headless(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
if submit_error:
|
||||
return SubmitResult(success=False, error=submit_error)
|
||||
|
||||
return SubmitResult(
|
||||
success=True, error="", submission_id="", verdict="submitted"
|
||||
if needs_relogin and not _retried:
|
||||
cookie_cache.unlink(missing_ok=True)
|
||||
return _submit_headless(
|
||||
contest_id,
|
||||
problem_id,
|
||||
source_code,
|
||||
language_id,
|
||||
credentials,
|
||||
_retried=True,
|
||||
)
|
||||
|
||||
if submit_error:
|
||||
return SubmitResult(success=False, error=submit_error)
|
||||
|
||||
return SubmitResult(
|
||||
success=True, error="", submission_id="", verdict="submitted"
|
||||
)
|
||||
except Exception as e:
|
||||
return SubmitResult(success=False, error=str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,7 @@
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
|
@ -21,10 +19,8 @@ from .models import (
|
|||
TestCase,
|
||||
)
|
||||
from .timeouts import (
|
||||
BROWSER_ELEMENT_WAIT,
|
||||
BROWSER_NAV_TIMEOUT,
|
||||
BROWSER_SESSION_TIMEOUT,
|
||||
BROWSER_SETTLE_DELAY,
|
||||
HTTP_TIMEOUT,
|
||||
)
|
||||
|
||||
|
|
@ -307,24 +303,13 @@ class CodeforcesScraper(BaseScraper):
|
|||
)
|
||||
|
||||
|
||||
def _wait_for_gate_reload(page, wait_selector: str) -> None:
|
||||
from .atcoder import _solve_turnstile
|
||||
|
||||
if "Verification" not in page.title():
|
||||
return
|
||||
_solve_turnstile(page)
|
||||
page.wait_for_function(
|
||||
f"() => !!document.querySelector('{wait_selector}')",
|
||||
timeout=BROWSER_ELEMENT_WAIT,
|
||||
)
|
||||
|
||||
|
||||
def _submit_headless(
|
||||
contest_id: str,
|
||||
problem_id: str,
|
||||
source_code: str,
|
||||
language_id: str,
|
||||
credentials: dict[str, str],
|
||||
_retried: bool = False,
|
||||
) -> SubmitResult:
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -349,9 +334,10 @@ def _submit_headless(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
logged_in = False
|
||||
logged_in = cookie_cache.exists() and not _retried
|
||||
login_error: str | None = None
|
||||
submit_error: str | None = None
|
||||
needs_relogin = False
|
||||
|
||||
def check_login(page):
|
||||
nonlocal logged_in
|
||||
|
|
@ -362,10 +348,6 @@ def _submit_headless(
|
|||
|
||||
def login_action(page):
|
||||
nonlocal login_error
|
||||
try:
|
||||
_wait_for_gate_reload(page, "#enterForm")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.fill(
|
||||
'input[name="handleOrEmail"]',
|
||||
|
|
@ -383,28 +365,21 @@ def _submit_headless(
|
|||
login_error = str(e)
|
||||
|
||||
def submit_action(page):
|
||||
nonlocal submit_error
|
||||
nonlocal submit_error, needs_relogin
|
||||
if "/enter" in page.url or "/login" in page.url:
|
||||
needs_relogin = True
|
||||
return
|
||||
try:
|
||||
_solve_turnstile(page)
|
||||
except Exception:
|
||||
pass
|
||||
tmp_path: str | None = None
|
||||
try:
|
||||
page.select_option(
|
||||
'select[name="submittedProblemIndex"]',
|
||||
problem_id.upper(),
|
||||
)
|
||||
page.select_option('select[name="programTypeId"]', language_id)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".cpp", delete=False, prefix="cf_"
|
||||
) as tf:
|
||||
tf.write(source_code)
|
||||
tmp_path = tf.name
|
||||
try:
|
||||
page.set_input_files('input[name="sourceFile"]', tmp_path)
|
||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||
except Exception:
|
||||
page.fill('textarea[name="source"]', source_code)
|
||||
page.fill('textarea[name="source"]', source_code)
|
||||
page.locator("form.submit-form input.submit").click(no_wait_after=True)
|
||||
try:
|
||||
page.wait_for_url(
|
||||
|
|
@ -419,26 +394,21 @@ def _submit_headless(
|
|||
submit_error = "Submit failed: page did not navigate"
|
||||
except Exception as e:
|
||||
submit_error = str(e)
|
||||
finally:
|
||||
if tmp_path:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
with StealthySession(
|
||||
headless=True,
|
||||
timeout=BROWSER_SESSION_TIMEOUT,
|
||||
google_search=False,
|
||||
cookies=saved_cookies,
|
||||
cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [],
|
||||
) as session:
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/",
|
||||
page_action=check_login,
|
||||
network_idle=True,
|
||||
)
|
||||
if not (cookie_cache.exists() and not _retried):
|
||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||
session.fetch(
|
||||
f"{BASE_URL}/",
|
||||
page_action=check_login,
|
||||
network_idle=True,
|
||||
)
|
||||
|
||||
if not logged_in:
|
||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||
|
|
@ -456,7 +426,7 @@ def _submit_headless(
|
|||
session.fetch(
|
||||
f"{BASE_URL}/contest/{contest_id}/submit",
|
||||
page_action=submit_action,
|
||||
solve_cloudflare=True,
|
||||
solve_cloudflare=False,
|
||||
)
|
||||
|
||||
try:
|
||||
|
|
@ -466,15 +436,26 @@ def _submit_headless(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
if submit_error:
|
||||
return SubmitResult(success=False, error=submit_error)
|
||||
|
||||
return SubmitResult(
|
||||
success=True,
|
||||
error="",
|
||||
submission_id="",
|
||||
verdict="submitted",
|
||||
if needs_relogin and not _retried:
|
||||
cookie_cache.unlink(missing_ok=True)
|
||||
return _submit_headless(
|
||||
contest_id,
|
||||
problem_id,
|
||||
source_code,
|
||||
language_id,
|
||||
credentials,
|
||||
_retried=True,
|
||||
)
|
||||
|
||||
if submit_error:
|
||||
return SubmitResult(success=False, error=submit_error)
|
||||
|
||||
return SubmitResult(
|
||||
success=True,
|
||||
error="",
|
||||
submission_id="",
|
||||
verdict="submitted",
|
||||
)
|
||||
except Exception as e:
|
||||
return SubmitResult(success=False, error=str(e))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue