barrett/cp.nvim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: expand language IDs, fix AtCoder submit, normalize logging (#353)

Browse source 
## Problem

Language version coverage was incomplete across all platforms, AtCoder
submit used a stale cookie fast-path that caused silent failures, and
raw
`vim.notify` calls throughout the codebase produced inconsistent or
missing `[cp.nvim]:` prefixes.

## Solution

Remove cookie persistence from AtCoder login/submit (always fresh
login),
increase the submit nav timeout to 40s, and switch to in-memory buffer
upload with the correct per-language extension from a full
`_LANGUAGE_ID_EXTENSION`
map covering all 116 AtCoder languages. Expand `LANGUAGE_VERSIONS` in
`constants.lua` with all AtCoder languages, 15 new CF languages with
full
version variants, and 50+ Kattis languages. Fix AtCoder `prolog` ID
(`6079`→`6081`, was Pony) and remove the non-existent `racket` entry.
Replace all raw `vim.notify` calls with `logger.log`. Simplify the
submit
language doc to point at `constants.lua` rather than maintaining a
static table.
This commit is contained in:
Barrett Ruth 2026-03-06 21:35:13 -05:00 • committed by GitHub
parent 1ac521a126
commit 291de4e137
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 356 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

179
scrapers/atcoder.py
View file

@ -36,9 +36,96 @@ from .timeouts import (
HTTP_TIMEOUT,
)
_LANGUAGE_ID_EXTENSION = {
_LANGUAGE_ID_EXTENSION: dict[str, str] = {
"6002": "ada",
"6003": "apl",
"6004": "asm",
"6005": "asm",
"6006": "awk",
"6008": "sh",
"6009": "bas",
"6010": "bc",
"6012": "bf",
"6013": "c",
"6014": "c",
"6015": "cs",
"6016": "cs",
"6017": "cc",
"6021": "clj",
"6022": "clj",
"6023": "clj",
"6025": "cljs",
"6026": "cob",
"6027": "lisp",
"6028": "cr",
"6030": "d",
"6031": "d",
"6032": "d",
"6033": "dart",
"6038": "ex",
"6039": "el",
"6041": "erl",
"6042": "fs",
"6043": "factor",
"6044": "fish",
"6045": "fth",
"6046": "f90",
"6047": "f90",
"6048": "f",
"6049": "gleam",
"6050": "go",
"6051": "go",
"6052": "hs",
"6053": "hx",
"6054": "cc",
"6056": "java",
"6057": "js",
"6058": "js",
"6059": "js",
"6060": "jule",
"6061": "kk",
"6062": "kt",
"6065": "lean",
"6066": "ll",
"6067": "lua",
"6068": "lua",
"6071": "nim",
"6072": "nim",
"6073": "ml",
"6074": "m",
"6075": "pas",
"6076": "pl",
"6077": "php",
"6079": "pony",
"6080": "ps1",
"6081": "pro",
"6082": "py",
"6083": "py",
"6084": "r",
"6085": "re",
"6086": "rb",
"6087": "rb",
"6088": "rs",
"6089": "py",
"6090": "scala",
"6091": "scala",
"6092": "scm",
"6093": "scm",
"6094": "sd7",
"6095": "swift",
"6096": "tcl",
"6100": "ts",
"6101": "ts",
"6102": "ts",
"6105": "v",
"6106": "vala",
"6107": "v",
"6109": "wat",
"6111": "zig",
"6114": "jl",
"6115": "py",
"6116": "cc",
"6118": "sql",
}
MIB_TO_MB = 1.048576
@ -304,9 +391,6 @@ def _login_headless(credentials: dict[str, str]) -> LoginResult:
_ensure_browser()
cookie_cache = Path.home() / ".cache" / "cp-nvim" / "atcoder-cookies.json"
cookie_cache.parent.mkdir(parents=True, exist_ok=True)
logged_in = False
login_error: str | None = None
@ -352,13 +436,6 @@ def _login_headless(credentials: dict[str, str]) -> LoginResult:
success=False, error="Login failed (bad credentials?)"
)
try:
browser_cookies = session.context.cookies()
if any(c["name"] == "REVEL_SESSION" for c in browser_cookies):
cookie_cache.write_text(json.dumps(browser_cookies))
except Exception:
pass
return LoginResult(success=True, error="")
except Exception as e:
return LoginResult(success=False, error=str(e))
@ -370,7 +447,6 @@ def _submit_headless(
file_path: str,
language_id: str,
credentials: dict[str, str],
_retried: bool = False,
) -> "SubmitResult":
try:
from scrapling.fetchers import StealthySession # type: ignore[import-untyped,unresolved-import]
@ -382,25 +458,8 @@ def _submit_headless(
_ensure_browser()
cookie_cache = Path.home() / ".cache" / "cp-nvim" / "atcoder-cookies.json"
cookie_cache.parent.mkdir(parents=True, exist_ok=True)
saved_cookies: list[dict[str, Any]] = []
if cookie_cache.exists():
try:
saved_cookies = json.loads(cookie_cache.read_text())
except Exception:
pass
logged_in = cookie_cache.exists() and not _retried
login_error: str | None = None
submit_error: str | None = None
needs_relogin = False
def check_login(page):
nonlocal logged_in
logged_in = page.evaluate(
"() => Array.from(document.querySelectorAll('a')).some(a => a.textContent.trim() === 'Sign Out')"
)
def login_action(page):
nonlocal login_error
@ -416,9 +475,9 @@ def _submit_headless(
login_error = str(e)
def submit_action(page):
nonlocal submit_error, needs_relogin
nonlocal submit_error
if "/login" in page.url:
needs_relogin = True
submit_error = "Not logged in after login step"
return
try:
_solve_turnstile(page)
@ -430,9 +489,19 @@ def _submit_headless(
f'select[name="data.LanguageId"] option[value="{language_id}"]'
).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT)
page.select_option('select[name="data.LanguageId"]', language_id)
page.set_input_files("#input-open-file", file_path)
ext = _LANGUAGE_ID_EXTENSION.get(
language_id, Path(file_path).suffix.lstrip(".") or "txt"
)
page.set_input_files(
"#input-open-file",
{
"name": f"solution.{ext}",
"mimeType": "text/plain",
"buffer": Path(file_path).read_bytes(),
},
)
page.wait_for_timeout(BROWSER_SETTLE_DELAY)
page.locator('button[type="submit"]').click()
page.locator('button[type="submit"]').click(no_wait_after=True)
page.wait_for_url(
lambda url: "/submissions/me" in url,
timeout=BROWSER_SUBMIT_NAV_TIMEOUT["atcoder"],
@ -445,25 +514,15 @@ def _submit_headless(
headless=True,
timeout=BROWSER_SESSION_TIMEOUT,
google_search=False,
cookies=saved_cookies if (cookie_cache.exists() and not _retried) else [],
) as session:
if not (cookie_cache.exists() and not _retried):
print(json.dumps({"status": "checking_login"}), flush=True)
session.fetch(
f"{BASE_URL}/home", page_action=check_login, network_idle=True
)
if not logged_in:
print(json.dumps({"status": "logging_in"}), flush=True)
session.fetch(
f"{BASE_URL}/login",
page_action=login_action,
solve_cloudflare=True,
)
if login_error:
return SubmitResult(
success=False, error=f"Login failed: {login_error}"
)
print(json.dumps({"status": "logging_in"}), flush=True)
session.fetch(
f"{BASE_URL}/login",
page_action=login_action,
solve_cloudflare=True,
)
if login_error:
return SubmitResult(success=False, error=f"Login failed: {login_error}")
print(json.dumps({"status": "submitting"}), flush=True)
session.fetch(
@ -472,24 +531,6 @@ def _submit_headless(
solve_cloudflare=True,
)
try:
browser_cookies = session.context.cookies()
if any(c["name"] == "REVEL_SESSION" for c in browser_cookies):
cookie_cache.write_text(json.dumps(browser_cookies))
except Exception:
pass
if needs_relogin and not _retried:
cookie_cache.unlink(missing_ok=True)
return _submit_headless(
contest_id,
problem_id,
file_path,
language_id,
credentials,
_retried=True,
)
if submit_error:
return SubmitResult(success=False, error=submit_error)

7
scrapers/kattis.py
View file

@ -338,13 +338,14 @@ class KattisScraper(BaseScraper):
await _save_kattis_cookies(client)
print(json.dumps({"status": "submitting"}), flush=True)
ext = "py" if "python" in language_id.lower() else "cpp"
lang_lower = language_id.lower()
mainclass = Path(file_path).stem if "java" in lang_lower else ""
data: dict[str, str] = {
"submit": "true",
"script": "true",
"language": language_id,
"problem": problem_id,
"mainclass": "",
"mainclass": mainclass,
"submit_ctr": "2",
}
if contest_id != problem_id:
@ -354,7 +355,7 @@ class KattisScraper(BaseScraper):
return await client.post(
f"{BASE_URL}/submit",
data=data,
files={"sub_file[]": (f"solution.{ext}", source, "text/plain")},
files={"sub_file[]": (Path(file_path).name, source, "text/plain")},
headers=HEADERS,
timeout=HTTP_TIMEOUT,
)

99
scrapers/language_ids.py
View file

@ -4,10 +4,60 @@ LANGUAGE_IDS = {
"python": "6082",
"java": "6056",
"rust": "6088",
"c": "6014",
"go": "6051",
"haskell": "6052",
"csharp": "6015",
"kotlin": "6062",
"ruby": "6087",
"javascript": "6059",
"typescript": "6100",
"scala": "6090",
"ocaml": "6073",
"dart": "6033",
"elixir": "6038",
"erlang": "6041",
"fsharp": "6042",
"swift": "6095",
"zig": "6111",
"nim": "6072",
"lua": "6067",
"perl": "6076",
"php": "6077",
"pascal": "6075",
"crystal": "6028",
"d": "6030",
"julia": "6114",
"r": "6084",
"commonlisp": "6027",
"scheme": "6092",
"clojure": "6022",
"ada": "6002",
"bash": "6008",
"fortran": "6047",
"gleam": "6049",
"lean": "6065",
"vala": "6106",
"v": "6105",
},
"codeforces": {
"cpp": "89",
"python": "70",
"java": "87",
"kotlin": "99",
"rust": "75",
"go": "32",
"csharp": "96",
"haskell": "12",
"javascript": "55",
"ruby": "67",
"scala": "20",
"ocaml": "19",
"d": "28",
"perl": "13",
"php": "6",
"pascal": "4",
"fsharp": "97",
},
"cses": {
"cpp": "C++17",
@ -25,6 +75,55 @@ LANGUAGE_IDS = {
"python": "Python 3",
"java": "Java",
"rust": "Rust",
"ada": "Ada",
"algol60": "Algol 60",
"algol68": "Algol 68",
"apl": "APL",
"bash": "Bash",
"bcpl": "BCPL",
"bqn": "BQN",
"c": "C",
"cobol": "COBOL",
"commonlisp": "Common Lisp",
"crystal": "Crystal",
"csharp": "C#",
"d": "D",
"dart": "Dart",
"elixir": "Elixir",
"erlang": "Erlang",
"forth": "Forth",
"fortran": "Fortran",
"fortran77": "Fortran 77",
"fsharp": "F#",
"gerbil": "Gerbil",
"go": "Go",
"haskell": "Haskell",
"icon": "Icon",
"javascript": "JavaScript (Node.js)",
"julia": "Julia",
"kotlin": "Kotlin",
"lua": "Lua",
"modula2": "Modula-2",
"nim": "Nim",
"objectivec": "Objective-C",
"ocaml": "OCaml",
"octave": "Octave",
"odin": "Odin",
"pascal": "Pascal",
"perl": "Perl",
"php": "PHP",
"pli": "PL/I",
"prolog": "Prolog",
"racket": "Racket",
"ruby": "Ruby",
"scala": "Scala",
"simula": "Simula 67",
"smalltalk": "Smalltalk",
"snobol": "SNOBOL",
"swift": "Swift",
"typescript": "TypeScript",
"visualbasic": "Visual Basic",
"zig": "Zig",
},
"codechef": {
"cpp": "C++",

1
scrapers/timeouts.py
View file

@ -7,6 +7,7 @@ BROWSER_NAV_TIMEOUT = 10000
BROWSER_SUBMIT_NAV_TIMEOUT: defaultdict[str, int] = defaultdict(
lambda: BROWSER_NAV_TIMEOUT
)
BROWSER_SUBMIT_NAV_TIMEOUT["atcoder"] = BROWSER_NAV_TIMEOUT * 2
BROWSER_SUBMIT_NAV_TIMEOUT["codeforces"] = BROWSER_NAV_TIMEOUT * 2
BROWSER_TURNSTILE_POLL = 5000
BROWSER_ELEMENT_WAIT = 10000