fix(submit): harden atcoder and codeforces submit flow
Problem: AtCoder file upload always used a `.cpp` temp file regardless of language. CF submit used `solve_cloudflare=True` causing a spurious "No Cloudflare challenge found" error, and `_wait_for_gate_reload` in `login_action` was dead code. Stale or expired cookies caused silent auth failures with no recovery. The `uv.spawn` ndjson path for submit had no overall timeout, so a hung browser process would live forever. Solution: Replace AtCoder's temp file with `page.set_input_files` using an in-memory buffer and correct extension via `_LANGUAGE_ID_EXTENSION`. Replace CF's temp-file/fallback dance with a direct `textarea[name="source"]` fill and set `solve_cloudflare=False` on the submit fetch. Add a fast-path that skips the homepage login check when cookies exist, with automatic stale-cookie recovery via `_retried` flag on redirect-to-login detection. Remove `_wait_for_gate_reload`. Fix `_ensure_browser` to propagate install errors instead of swallowing them. Add a 120s kill timer to the ndjson `uv.spawn` submit path.
This commit is contained in:
parent
6fcb5d1bbc
commit
6923301562
3 changed files with 105 additions and 99 deletions
|
|
@ -92,6 +92,10 @@ local function run_scraper(platform, subcommand, args, opts)
|
||||||
env = spawn_env_list(env),
|
env = spawn_env_list(env),
|
||||||
cwd = plugin_path,
|
cwd = plugin_path,
|
||||||
}, function(code, signal)
|
}, function(code, signal)
|
||||||
|
if timer and not timer:is_closing() then
|
||||||
|
timer:stop()
|
||||||
|
timer:close()
|
||||||
|
end
|
||||||
if buf ~= '' and opts.on_event then
|
if buf ~= '' and opts.on_event then
|
||||||
local ok_tail, ev_tail = pcall(vim.json.decode, buf)
|
local ok_tail, ev_tail = pcall(vim.json.decode, buf)
|
||||||
if ok_tail then
|
if ok_tail then
|
||||||
|
|
@ -124,6 +128,31 @@ local function run_scraper(platform, subcommand, args, opts)
|
||||||
return { success = false, error = 'spawn failed' }
|
return { success = false, error = 'spawn failed' }
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local timer = nil
|
||||||
|
if subcommand == 'submit' then
|
||||||
|
timer = uv.new_timer()
|
||||||
|
timer:start(120000, 0, function()
|
||||||
|
timer:stop()
|
||||||
|
timer:close()
|
||||||
|
if stdin_pipe and not stdin_pipe:is_closing() then
|
||||||
|
stdin_pipe:close()
|
||||||
|
end
|
||||||
|
if not stdout:is_closing() then
|
||||||
|
stdout:close()
|
||||||
|
end
|
||||||
|
if not stderr:is_closing() then
|
||||||
|
stderr:close()
|
||||||
|
end
|
||||||
|
if handle and not handle:is_closing() then
|
||||||
|
handle:kill(15)
|
||||||
|
handle:close()
|
||||||
|
end
|
||||||
|
if opts.on_exit then
|
||||||
|
opts.on_exit({ success = false, error = 'submit timed out' })
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
if stdin_pipe then
|
if stdin_pipe then
|
||||||
uv.write(stdin_pipe, opts.stdin, function()
|
uv.write(stdin_pipe, opts.stdin, function()
|
||||||
uv.shutdown(stdin_pipe, function()
|
uv.shutdown(stdin_pipe, function()
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,6 @@ import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
|
||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
@ -38,6 +37,11 @@ from .timeouts import (
|
||||||
HTTP_TIMEOUT,
|
HTTP_TIMEOUT,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_LANGUAGE_ID_EXTENSION = {
|
||||||
|
"6017": "cc",
|
||||||
|
"6082": "py",
|
||||||
|
}
|
||||||
|
|
||||||
MIB_TO_MB = 1.048576
|
MIB_TO_MB = 1.048576
|
||||||
BASE_URL = "https://atcoder.jp"
|
BASE_URL = "https://atcoder.jp"
|
||||||
ARCHIVE_URL = f"{BASE_URL}/contests/archive"
|
ARCHIVE_URL = f"{BASE_URL}/contests/archive"
|
||||||
|
|
@ -274,23 +278,20 @@ def _ensure_browser() -> None:
|
||||||
from patchright._impl._driver import compute_driver_executable # type: ignore[import-untyped,unresolved-import]
|
from patchright._impl._driver import compute_driver_executable # type: ignore[import-untyped,unresolved-import]
|
||||||
|
|
||||||
node, cli = compute_driver_executable()
|
node, cli = compute_driver_executable()
|
||||||
browser_info = subprocess.run(
|
|
||||||
[node, cli, "install", "--dry-run", "chromium"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
for line in browser_info.stdout.splitlines():
|
|
||||||
if "Install location:" in line:
|
|
||||||
install_dir = line.split(":", 1)[1].strip()
|
|
||||||
if not os.path.isdir(install_dir):
|
|
||||||
print(json.dumps({"status": "installing_browser"}), flush=True)
|
|
||||||
subprocess.run(
|
|
||||||
[node, cli, "install", "chromium"],
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
return
|
||||||
|
browser_info = subprocess.run(
|
||||||
|
[node, cli, "install", "--dry-run", "chromium"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
for line in browser_info.stdout.splitlines():
|
||||||
|
if "Install location:" in line:
|
||||||
|
install_dir = line.split(":", 1)[1].strip()
|
||||||
|
if not os.path.isdir(install_dir):
|
||||||
|
print(json.dumps({"status": "installing_browser"}), flush=True)
|
||||||
|
subprocess.run([node, cli, "install", "chromium"], check=True)
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
def _submit_headless(
|
def _submit_headless(
|
||||||
|
|
@ -299,6 +300,7 @@ def _submit_headless(
|
||||||
source_code: str,
|
source_code: str,
|
||||||
language_id: str,
|
language_id: str,
|
||||||
credentials: dict[str, str],
|
credentials: dict[str, str],
|
||||||
|
_retried: bool = False,
|
||||||
) -> "SubmitResult":
|
) -> "SubmitResult":
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
@ -321,9 +323,10 @@ def _submit_headless(
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
logged_in = False
|
logged_in = cookie_cache.exists() and not _retried
|
||||||
login_error: str | None = None
|
login_error: str | None = None
|
||||||
submit_error: str | None = None
|
submit_error: str | None = None
|
||||||
|
needs_relogin = False
|
||||||
|
|
||||||
def check_login(page):
|
def check_login(page):
|
||||||
nonlocal logged_in
|
nonlocal logged_in
|
||||||
|
|
@ -345,7 +348,10 @@ def _submit_headless(
|
||||||
login_error = str(e)
|
login_error = str(e)
|
||||||
|
|
||||||
def submit_action(page):
|
def submit_action(page):
|
||||||
nonlocal submit_error
|
nonlocal submit_error, needs_relogin
|
||||||
|
if "/login" in page.url:
|
||||||
|
needs_relogin = True
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
_solve_turnstile(page)
|
_solve_turnstile(page)
|
||||||
page.select_option(
|
page.select_option(
|
||||||
|
|
@ -356,16 +362,12 @@ def _submit_headless(
|
||||||
f'select[name="data.LanguageId"] option[value="{language_id}"]'
|
f'select[name="data.LanguageId"] option[value="{language_id}"]'
|
||||||
).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT)
|
).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT)
|
||||||
page.select_option('select[name="data.LanguageId"]', language_id)
|
page.select_option('select[name="data.LanguageId"]', language_id)
|
||||||
with tempfile.NamedTemporaryFile(
|
ext = _LANGUAGE_ID_EXTENSION.get(language_id, "txt")
|
||||||
mode="w", suffix=".cpp", delete=False, prefix="atcoder_"
|
page.set_input_files(
|
||||||
) as tf:
|
"#input-open-file",
|
||||||
tf.write(source_code)
|
{"name": f"solution.{ext}", "mimeType": "text/plain", "buffer": source_code.encode()},
|
||||||
tmp_path = tf.name
|
)
|
||||||
try:
|
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
||||||
page.set_input_files("#input-open-file", tmp_path)
|
|
||||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
|
||||||
finally:
|
|
||||||
os.unlink(tmp_path)
|
|
||||||
page.locator('button[type="submit"]').click()
|
page.locator('button[type="submit"]').click()
|
||||||
page.wait_for_url(
|
page.wait_for_url(
|
||||||
lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT
|
lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT
|
||||||
|
|
@ -378,14 +380,11 @@ def _submit_headless(
|
||||||
headless=True,
|
headless=True,
|
||||||
timeout=BROWSER_SESSION_TIMEOUT,
|
timeout=BROWSER_SESSION_TIMEOUT,
|
||||||
google_search=False,
|
google_search=False,
|
||||||
cookies=saved_cookies,
|
cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [],
|
||||||
) as session:
|
) as session:
|
||||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
if not (cookie_cache.exists() and not _retried):
|
||||||
session.fetch(
|
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||||
f"{BASE_URL}/home",
|
session.fetch(f"{BASE_URL}/home", page_action=check_login, network_idle=True)
|
||||||
page_action=check_login,
|
|
||||||
network_idle=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not logged_in:
|
if not logged_in:
|
||||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||||
|
|
@ -395,9 +394,7 @@ def _submit_headless(
|
||||||
solve_cloudflare=True,
|
solve_cloudflare=True,
|
||||||
)
|
)
|
||||||
if login_error:
|
if login_error:
|
||||||
return SubmitResult(
|
return SubmitResult(success=False, error=f"Login failed: {login_error}")
|
||||||
success=False, error=f"Login failed: {login_error}"
|
|
||||||
)
|
|
||||||
|
|
||||||
print(json.dumps({"status": "submitting"}), flush=True)
|
print(json.dumps({"status": "submitting"}), flush=True)
|
||||||
session.fetch(
|
session.fetch(
|
||||||
|
|
@ -413,12 +410,16 @@ def _submit_headless(
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if submit_error:
|
if needs_relogin and not _retried:
|
||||||
return SubmitResult(success=False, error=submit_error)
|
cookie_cache.unlink(missing_ok=True)
|
||||||
|
return _submit_headless(
|
||||||
return SubmitResult(
|
contest_id, problem_id, source_code, language_id, credentials, _retried=True
|
||||||
success=True, error="", submission_id="", verdict="submitted"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if submit_error:
|
||||||
|
return SubmitResult(success=False, error=submit_error)
|
||||||
|
|
||||||
|
return SubmitResult(success=True, error="", submission_id="", verdict="submitted")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return SubmitResult(success=False, error=str(e))
|
return SubmitResult(success=False, error=str(e))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,7 @@
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
import tempfile
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
@ -21,10 +19,8 @@ from .models import (
|
||||||
TestCase,
|
TestCase,
|
||||||
)
|
)
|
||||||
from .timeouts import (
|
from .timeouts import (
|
||||||
BROWSER_ELEMENT_WAIT,
|
|
||||||
BROWSER_NAV_TIMEOUT,
|
BROWSER_NAV_TIMEOUT,
|
||||||
BROWSER_SESSION_TIMEOUT,
|
BROWSER_SESSION_TIMEOUT,
|
||||||
BROWSER_SETTLE_DELAY,
|
|
||||||
HTTP_TIMEOUT,
|
HTTP_TIMEOUT,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -307,24 +303,13 @@ class CodeforcesScraper(BaseScraper):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _wait_for_gate_reload(page, wait_selector: str) -> None:
|
|
||||||
from .atcoder import _solve_turnstile
|
|
||||||
|
|
||||||
if "Verification" not in page.title():
|
|
||||||
return
|
|
||||||
_solve_turnstile(page)
|
|
||||||
page.wait_for_function(
|
|
||||||
f"() => !!document.querySelector('{wait_selector}')",
|
|
||||||
timeout=BROWSER_ELEMENT_WAIT,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _submit_headless(
|
def _submit_headless(
|
||||||
contest_id: str,
|
contest_id: str,
|
||||||
problem_id: str,
|
problem_id: str,
|
||||||
source_code: str,
|
source_code: str,
|
||||||
language_id: str,
|
language_id: str,
|
||||||
credentials: dict[str, str],
|
credentials: dict[str, str],
|
||||||
|
_retried: bool = False,
|
||||||
) -> SubmitResult:
|
) -> SubmitResult:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
@ -349,9 +334,10 @@ def _submit_headless(
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
logged_in = False
|
logged_in = cookie_cache.exists() and not _retried
|
||||||
login_error: str | None = None
|
login_error: str | None = None
|
||||||
submit_error: str | None = None
|
submit_error: str | None = None
|
||||||
|
needs_relogin = False
|
||||||
|
|
||||||
def check_login(page):
|
def check_login(page):
|
||||||
nonlocal logged_in
|
nonlocal logged_in
|
||||||
|
|
@ -362,10 +348,6 @@ def _submit_headless(
|
||||||
|
|
||||||
def login_action(page):
|
def login_action(page):
|
||||||
nonlocal login_error
|
nonlocal login_error
|
||||||
try:
|
|
||||||
_wait_for_gate_reload(page, "#enterForm")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
try:
|
||||||
page.fill(
|
page.fill(
|
||||||
'input[name="handleOrEmail"]',
|
'input[name="handleOrEmail"]',
|
||||||
|
|
@ -383,28 +365,21 @@ def _submit_headless(
|
||||||
login_error = str(e)
|
login_error = str(e)
|
||||||
|
|
||||||
def submit_action(page):
|
def submit_action(page):
|
||||||
nonlocal submit_error
|
nonlocal submit_error, needs_relogin
|
||||||
|
if "/enter" in page.url or "/login" in page.url:
|
||||||
|
needs_relogin = True
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
_solve_turnstile(page)
|
_solve_turnstile(page)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
tmp_path: str | None = None
|
|
||||||
try:
|
try:
|
||||||
page.select_option(
|
page.select_option(
|
||||||
'select[name="submittedProblemIndex"]',
|
'select[name="submittedProblemIndex"]',
|
||||||
problem_id.upper(),
|
problem_id.upper(),
|
||||||
)
|
)
|
||||||
page.select_option('select[name="programTypeId"]', language_id)
|
page.select_option('select[name="programTypeId"]', language_id)
|
||||||
with tempfile.NamedTemporaryFile(
|
page.fill('textarea[name="source"]', source_code)
|
||||||
mode="w", suffix=".cpp", delete=False, prefix="cf_"
|
|
||||||
) as tf:
|
|
||||||
tf.write(source_code)
|
|
||||||
tmp_path = tf.name
|
|
||||||
try:
|
|
||||||
page.set_input_files('input[name="sourceFile"]', tmp_path)
|
|
||||||
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
|
|
||||||
except Exception:
|
|
||||||
page.fill('textarea[name="source"]', source_code)
|
|
||||||
page.locator("form.submit-form input.submit").click(no_wait_after=True)
|
page.locator("form.submit-form input.submit").click(no_wait_after=True)
|
||||||
try:
|
try:
|
||||||
page.wait_for_url(
|
page.wait_for_url(
|
||||||
|
|
@ -419,26 +394,21 @@ def _submit_headless(
|
||||||
submit_error = "Submit failed: page did not navigate"
|
submit_error = "Submit failed: page did not navigate"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
submit_error = str(e)
|
submit_error = str(e)
|
||||||
finally:
|
|
||||||
if tmp_path:
|
|
||||||
try:
|
|
||||||
os.unlink(tmp_path)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with StealthySession(
|
with StealthySession(
|
||||||
headless=True,
|
headless=True,
|
||||||
timeout=BROWSER_SESSION_TIMEOUT,
|
timeout=BROWSER_SESSION_TIMEOUT,
|
||||||
google_search=False,
|
google_search=False,
|
||||||
cookies=saved_cookies,
|
cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [],
|
||||||
) as session:
|
) as session:
|
||||||
print(json.dumps({"status": "checking_login"}), flush=True)
|
if not (cookie_cache.exists() and not _retried):
|
||||||
session.fetch(
|
print(json.dumps({"status": "checking_login"}), flush=True)
|
||||||
f"{BASE_URL}/",
|
session.fetch(
|
||||||
page_action=check_login,
|
f"{BASE_URL}/",
|
||||||
network_idle=True,
|
page_action=check_login,
|
||||||
)
|
network_idle=True,
|
||||||
|
)
|
||||||
|
|
||||||
if not logged_in:
|
if not logged_in:
|
||||||
print(json.dumps({"status": "logging_in"}), flush=True)
|
print(json.dumps({"status": "logging_in"}), flush=True)
|
||||||
|
|
@ -456,7 +426,7 @@ def _submit_headless(
|
||||||
session.fetch(
|
session.fetch(
|
||||||
f"{BASE_URL}/contest/{contest_id}/submit",
|
f"{BASE_URL}/contest/{contest_id}/submit",
|
||||||
page_action=submit_action,
|
page_action=submit_action,
|
||||||
solve_cloudflare=True,
|
solve_cloudflare=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -466,15 +436,21 @@ def _submit_headless(
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if submit_error:
|
if needs_relogin and not _retried:
|
||||||
return SubmitResult(success=False, error=submit_error)
|
cookie_cache.unlink(missing_ok=True)
|
||||||
|
return _submit_headless(
|
||||||
return SubmitResult(
|
contest_id, problem_id, source_code, language_id, credentials, _retried=True
|
||||||
success=True,
|
|
||||||
error="",
|
|
||||||
submission_id="",
|
|
||||||
verdict="submitted",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if submit_error:
|
||||||
|
return SubmitResult(success=False, error=submit_error)
|
||||||
|
|
||||||
|
return SubmitResult(
|
||||||
|
success=True,
|
||||||
|
error="",
|
||||||
|
submission_id="",
|
||||||
|
verdict="submitted",
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return SubmitResult(success=False, error=str(e))
|
return SubmitResult(success=False, error=str(e))
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue