diff --git a/lua/cp/scraper.lua b/lua/cp/scraper.lua index fc8ba69..a600e95 100644 --- a/lua/cp/scraper.lua +++ b/lua/cp/scraper.lua @@ -92,6 +92,10 @@ local function run_scraper(platform, subcommand, args, opts) env = spawn_env_list(env), cwd = plugin_path, }, function(code, signal) + if timer and not timer:is_closing() then + timer:stop() + timer:close() + end if buf ~= '' and opts.on_event then local ok_tail, ev_tail = pcall(vim.json.decode, buf) if ok_tail then @@ -124,6 +128,31 @@ local function run_scraper(platform, subcommand, args, opts) return { success = false, error = 'spawn failed' } end + local timer = nil + if subcommand == 'submit' then + timer = uv.new_timer() + timer:start(120000, 0, function() + timer:stop() + timer:close() + if stdin_pipe and not stdin_pipe:is_closing() then + stdin_pipe:close() + end + if not stdout:is_closing() then + stdout:close() + end + if not stderr:is_closing() then + stderr:close() + end + if handle and not handle:is_closing() then + handle:kill(15) + handle:close() + end + if opts.on_exit then + opts.on_exit({ success = false, error = 'submit timed out' }) + end + end) + end + if stdin_pipe then uv.write(stdin_pipe, opts.stdin, function() uv.shutdown(stdin_pipe, function() diff --git a/scrapers/atcoder.py b/scrapers/atcoder.py index 45a2195..f8a1ace 100644 --- a/scrapers/atcoder.py +++ b/scrapers/atcoder.py @@ -6,7 +6,6 @@ import os import re import subprocess import sys -import tempfile import time from typing import Any @@ -38,6 +37,11 @@ from .timeouts import ( HTTP_TIMEOUT, ) +_LANGUAGE_ID_EXTENSION = { + "6017": "cc", + "6082": "py", +} + MIB_TO_MB = 1.048576 BASE_URL = "https://atcoder.jp" ARCHIVE_URL = f"{BASE_URL}/contests/archive" @@ -274,23 +278,20 @@ def _ensure_browser() -> None: from patchright._impl._driver import compute_driver_executable # type: ignore[import-untyped,unresolved-import] node, cli = compute_driver_executable() - browser_info = subprocess.run( - [node, cli, "install", "--dry-run", "chromium"], - capture_output=True, - text=True, - ) - for line in browser_info.stdout.splitlines(): - if "Install location:" in line: - install_dir = line.split(":", 1)[1].strip() - if not os.path.isdir(install_dir): - print(json.dumps({"status": "installing_browser"}), flush=True) - subprocess.run( - [node, cli, "install", "chromium"], - check=True, - ) - break except Exception: - pass + return + browser_info = subprocess.run( + [node, cli, "install", "--dry-run", "chromium"], + capture_output=True, + text=True, + ) + for line in browser_info.stdout.splitlines(): + if "Install location:" in line: + install_dir = line.split(":", 1)[1].strip() + if not os.path.isdir(install_dir): + print(json.dumps({"status": "installing_browser"}), flush=True) + subprocess.run([node, cli, "install", "chromium"], check=True) + break def _submit_headless( @@ -299,6 +300,7 @@ def _submit_headless( source_code: str, language_id: str, credentials: dict[str, str], + _retried: bool = False, ) -> "SubmitResult": from pathlib import Path @@ -321,9 +323,10 @@ def _submit_headless( except Exception: pass - logged_in = False + logged_in = cookie_cache.exists() and not _retried login_error: str | None = None submit_error: str | None = None + needs_relogin = False def check_login(page): nonlocal logged_in @@ -345,7 +348,10 @@ def _submit_headless( login_error = str(e) def submit_action(page): - nonlocal submit_error + nonlocal submit_error, needs_relogin + if "/login" in page.url: + needs_relogin = True + return try: _solve_turnstile(page) page.select_option( @@ -356,16 +362,12 @@ def _submit_headless( f'select[name="data.LanguageId"] option[value="{language_id}"]' ).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT) page.select_option('select[name="data.LanguageId"]', language_id) - with tempfile.NamedTemporaryFile( - mode="w", suffix=".cpp", delete=False, prefix="atcoder_" - ) as tf: - tf.write(source_code) - tmp_path = tf.name - try: - page.set_input_files("#input-open-file", tmp_path) - page.wait_for_timeout(BROWSER_SETTLE_DELAY) - finally: - os.unlink(tmp_path) + ext = _LANGUAGE_ID_EXTENSION.get(language_id, "txt") + page.set_input_files( + "#input-open-file", + {"name": f"solution.{ext}", "mimeType": "text/plain", "buffer": source_code.encode()}, + ) + page.wait_for_timeout(BROWSER_SETTLE_DELAY) page.locator('button[type="submit"]').click() page.wait_for_url( lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT @@ -378,14 +380,11 @@ def _submit_headless( headless=True, timeout=BROWSER_SESSION_TIMEOUT, google_search=False, - cookies=saved_cookies, + cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [], ) as session: - print(json.dumps({"status": "checking_login"}), flush=True) - session.fetch( - f"{BASE_URL}/home", - page_action=check_login, - network_idle=True, - ) + if not (cookie_cache.exists() and not _retried): + print(json.dumps({"status": "checking_login"}), flush=True) + session.fetch(f"{BASE_URL}/home", page_action=check_login, network_idle=True) if not logged_in: print(json.dumps({"status": "logging_in"}), flush=True) @@ -395,9 +394,7 @@ def _submit_headless( solve_cloudflare=True, ) if login_error: - return SubmitResult( - success=False, error=f"Login failed: {login_error}" - ) + return SubmitResult(success=False, error=f"Login failed: {login_error}") print(json.dumps({"status": "submitting"}), flush=True) session.fetch( @@ -413,12 +410,16 @@ def _submit_headless( except Exception: pass - if submit_error: - return SubmitResult(success=False, error=submit_error) - - return SubmitResult( - success=True, error="", submission_id="", verdict="submitted" + if needs_relogin and not _retried: + cookie_cache.unlink(missing_ok=True) + return _submit_headless( + contest_id, problem_id, source_code, language_id, credentials, _retried=True ) + + if submit_error: + return SubmitResult(success=False, error=submit_error) + + return SubmitResult(success=True, error="", submission_id="", verdict="submitted") except Exception as e: return SubmitResult(success=False, error=str(e)) diff --git a/scrapers/codeforces.py b/scrapers/codeforces.py index fd0c129..f3fc30a 100644 --- a/scrapers/codeforces.py +++ b/scrapers/codeforces.py @@ -2,9 +2,7 @@ import asyncio import json -import os import re -import tempfile from typing import Any import requests @@ -21,10 +19,8 @@ from .models import ( TestCase, ) from .timeouts import ( - BROWSER_ELEMENT_WAIT, BROWSER_NAV_TIMEOUT, BROWSER_SESSION_TIMEOUT, - BROWSER_SETTLE_DELAY, HTTP_TIMEOUT, ) @@ -307,24 +303,13 @@ class CodeforcesScraper(BaseScraper): ) -def _wait_for_gate_reload(page, wait_selector: str) -> None: - from .atcoder import _solve_turnstile - - if "Verification" not in page.title(): - return - _solve_turnstile(page) - page.wait_for_function( - f"() => !!document.querySelector('{wait_selector}')", - timeout=BROWSER_ELEMENT_WAIT, - ) - - def _submit_headless( contest_id: str, problem_id: str, source_code: str, language_id: str, credentials: dict[str, str], + _retried: bool = False, ) -> SubmitResult: from pathlib import Path @@ -349,9 +334,10 @@ def _submit_headless( except Exception: pass - logged_in = False + logged_in = cookie_cache.exists() and not _retried login_error: str | None = None submit_error: str | None = None + needs_relogin = False def check_login(page): nonlocal logged_in @@ -362,10 +348,6 @@ def _submit_headless( def login_action(page): nonlocal login_error - try: - _wait_for_gate_reload(page, "#enterForm") - except Exception: - pass try: page.fill( 'input[name="handleOrEmail"]', @@ -383,28 +365,21 @@ def _submit_headless( login_error = str(e) def submit_action(page): - nonlocal submit_error + nonlocal submit_error, needs_relogin + if "/enter" in page.url or "/login" in page.url: + needs_relogin = True + return try: _solve_turnstile(page) except Exception: pass - tmp_path: str | None = None try: page.select_option( 'select[name="submittedProblemIndex"]', problem_id.upper(), ) page.select_option('select[name="programTypeId"]', language_id) - with tempfile.NamedTemporaryFile( - mode="w", suffix=".cpp", delete=False, prefix="cf_" - ) as tf: - tf.write(source_code) - tmp_path = tf.name - try: - page.set_input_files('input[name="sourceFile"]', tmp_path) - page.wait_for_timeout(BROWSER_SETTLE_DELAY) - except Exception: - page.fill('textarea[name="source"]', source_code) + page.fill('textarea[name="source"]', source_code) page.locator("form.submit-form input.submit").click(no_wait_after=True) try: page.wait_for_url( @@ -419,26 +394,21 @@ def _submit_headless( submit_error = "Submit failed: page did not navigate" except Exception as e: submit_error = str(e) - finally: - if tmp_path: - try: - os.unlink(tmp_path) - except OSError: - pass try: with StealthySession( headless=True, timeout=BROWSER_SESSION_TIMEOUT, google_search=False, - cookies=saved_cookies, + cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [], ) as session: - print(json.dumps({"status": "checking_login"}), flush=True) - session.fetch( - f"{BASE_URL}/", - page_action=check_login, - network_idle=True, - ) + if not (cookie_cache.exists() and not _retried): + print(json.dumps({"status": "checking_login"}), flush=True) + session.fetch( + f"{BASE_URL}/", + page_action=check_login, + network_idle=True, + ) if not logged_in: print(json.dumps({"status": "logging_in"}), flush=True) @@ -456,7 +426,7 @@ def _submit_headless( session.fetch( f"{BASE_URL}/contest/{contest_id}/submit", page_action=submit_action, - solve_cloudflare=True, + solve_cloudflare=False, ) try: @@ -466,15 +436,21 @@ def _submit_headless( except Exception: pass - if submit_error: - return SubmitResult(success=False, error=submit_error) - - return SubmitResult( - success=True, - error="", - submission_id="", - verdict="submitted", + if needs_relogin and not _retried: + cookie_cache.unlink(missing_ok=True) + return _submit_headless( + contest_id, problem_id, source_code, language_id, credentials, _retried=True ) + + if submit_error: + return SubmitResult(success=False, error=submit_error) + + return SubmitResult( + success=True, + error="", + submission_id="", + verdict="submitted", + ) except Exception as e: return SubmitResult(success=False, error=str(e))