fix(utils): redraw before submit env notify; simplify message

Problem: In a dev checkout, `_nix_submit_cmd` is nil (only baked in by
the nix derivation). The uv fallback fails with code 127 on NixOS because
`uv` is not a bare system binary — it's only available via the FHS-wrapped
`cp-nvim-submit` script produced by `mkSubmitEnv`.

Solution: Add `discover_nix_submit_cmd` mirroring `discover_nix_python`:
runs `nix build #submitEnv --no-link --print-out-paths`, caches the result
in `stdpath('cache')/cp-nvim/nix-submit`, and sets `_nix_submit_cmd`.
`run_scraper` calls `setup_nix_submit_env()` before spawning submit.

flake.nix

@@ -26,6 +26,20 @@
           ps.requests
         ]);
 
+      mkDevPythonEnv =
+        pkgs:
+        pkgs.python312.withPackages (ps: [
+          ps.backoff
+          ps.beautifulsoup4
+          ps.curl-cffi
+          ps.httpx
+          ps.ndjson
+          ps.pydantic
+          ps.requests
+          ps.pytest
+          ps.pytest-mock
+        ]);
+
       mkSubmitEnv =
         pkgs:
         pkgs.buildFHSEnv {
@@ -44,19 +58,24 @@
               glib
               gtk3
               libdrm
-              libGL
               libxkbcommon
               mesa
+              libGL
               nspr
               nss
               pango
-              xorg.libX11
+              libx11
-              xorg.libXcomposite
+              libxcomposite
-              xorg.libXdamage
+              libxdamage
-              xorg.libXext
+              libxext
-              xorg.libXfixes
+              libxfixes
-              xorg.libXrandr
+              libxrandr
-              xorg.libxcb
+              libxcb
+              at-spi2-core
+              expat
+              libgbm
+              systemdLibs
+              zlib
             ];
           runScript = "${pkgs.uv}/bin/uv";
         };
@@ -100,15 +119,19 @@
         submitEnv = mkSubmitEnv (pkgsFor system);
       });
 
+      formatter = eachSystem (system: (pkgsFor system).nixfmt-tree);
+
       devShells = eachSystem (system: {
         default = (pkgsFor system).mkShell {
           packages = with (pkgsFor system); [
             uv
-            python312
+            (mkDevPythonEnv (pkgsFor system))
             prettier
+            ruff
             stylua
             selene
             lua-language-server
+            ty
           ];
         };
       });

lua/cp/scraper.lua

@@ -44,6 +44,10 @@ local function run_scraper(platform, subcommand, args, opts)
     return { success = false, error = msg }
   end
 
+  if subcommand == 'submit' then
+    utils.setup_nix_submit_env()
+  end
+
   local plugin_path = utils.get_plugin_path()
   local cmd
   if subcommand == 'submit' then
@@ -73,6 +77,10 @@ local function run_scraper(platform, subcommand, args, opts)
 
   if opts and opts.ndjson then
     local uv = vim.uv
+    local stdin_pipe = nil
+    if opts.stdin then
+      stdin_pipe = uv.new_pipe(false)
+    end
     local stdout = uv.new_pipe(false)
     local stderr = uv.new_pipe(false)
     local buf = ''
@@ -80,7 +88,7 @@ local function run_scraper(platform, subcommand, args, opts)
     local handle
     handle = uv.spawn(cmd[1], {
       args = vim.list_slice(cmd, 2),
-      stdio = { nil, stdout, stderr },
+      stdio = { stdin_pipe, stdout, stderr },
       env = spawn_env_list(env),
       cwd = plugin_path,
     }, function(code, signal)
@@ -94,6 +102,9 @@ local function run_scraper(platform, subcommand, args, opts)
       if opts.on_exit then
         opts.on_exit({ success = (code == 0), code = code, signal = signal })
       end
+      if stdin_pipe and not stdin_pipe:is_closing() then
+        stdin_pipe:close()
+      end
       if not stdout:is_closing() then
         stdout:close()
       end
@@ -106,10 +117,21 @@ local function run_scraper(platform, subcommand, args, opts)
     end)
 
     if not handle then
+      if stdin_pipe and not stdin_pipe:is_closing() then
+        stdin_pipe:close()
+      end
       logger.log('Failed to start scraper process', vim.log.levels.ERROR)
       return { success = false, error = 'spawn failed' }
     end
 
+    if stdin_pipe then
+      uv.write(stdin_pipe, opts.stdin, function()
+        uv.shutdown(stdin_pipe, function()
+          stdin_pipe:close()
+        end)
+      end)
+    end
+
     uv.read_start(stdout, function(_, data)
       if data == nil then
         if buf ~= '' and opts.on_event then
@@ -260,18 +282,39 @@ function M.scrape_all_tests(platform, contest_id, callback)
   })
 end
 
-function M.submit(platform, contest_id, problem_id, language, source_code, credentials, callback)
+function M.submit(
-  local creds_json = vim.json.encode(credentials)
+  platform,
+  contest_id,
+  problem_id,
+  language,
+  source_code,
+  credentials,
+  on_status,
+  callback
+)
+  local done = false
   run_scraper(platform, 'submit', { contest_id, problem_id, language }, {
+    ndjson = true,
     stdin = source_code,
-    env_extra = { CP_CREDENTIALS = creds_json },
+    env_extra = { CP_CREDENTIALS = vim.json.encode(credentials) },
-    on_exit = function(result)
+    on_event = function(ev)
-      if type(callback) == 'function' then
+      if ev.status ~= nil then
-        if result and result.success then
+        if type(on_status) == 'function' then
-          callback(result.data or { success = true })
+          on_status(ev.status)
-        else
-          callback({ success = false, error = result and result.error or 'unknown' })
         end
+      elseif ev.success ~= nil then
+        done = true
+        if type(callback) == 'function' then
+          callback(ev)
+        end
+      end
+    end,
+    on_exit = function(proc)
+      if not done and type(callback) == 'function' then
+        callback({
+          success = false,
+          error = 'submit process exited (code=' .. tostring(proc.code) .. ')',
+        })
       end
     end,
   })

lua/cp/submit.lua

@@ -4,6 +4,13 @@ local cache = require('cp.cache')
 local logger = require('cp.log')
 local state = require('cp.state')
 
+local STATUS_MSGS = {
+  installing_browser = 'Installing browser (first time setup)...',
+  checking_login = 'Checking login...',
+  logging_in = 'Logging in...',
+  submitting = 'Submitting...',
+}
+
 local function prompt_credentials(platform, callback)
   local saved = cache.get_credentials(platform)
   if saved and saved.username and saved.password then
@@ -48,6 +55,8 @@ function M.submit(opts)
     local source_lines = vim.fn.readfile(source_file)
     local source_code = table.concat(source_lines, '\n')
 
+    vim.notify('[cp.nvim] Submitting...', vim.log.levels.INFO)
+
     require('cp.scraper').submit(
       platform,
       contest_id,
@@ -55,6 +64,11 @@ function M.submit(opts)
       language,
       source_code,
       creds,
+      function(status)
+        vim.schedule(function()
+          vim.notify('[cp.nvim] ' .. (STATUS_MSGS[status] or status), vim.log.levels.INFO)
+        end)
+      end,
       function(result)
         vim.schedule(function()
           if result and result.success then

lua/cp/utils.lua

@@ -119,10 +119,76 @@ function M.get_python_submit_cmd(module, plugin_path)
   if _nix_submit_cmd then
     return { _nix_submit_cmd, 'run', '--directory', plugin_path, '-m', 'scrapers.' .. module }
   end
-  return M.get_python_cmd(module, plugin_path)
+  return { 'uv', 'run', '--directory', plugin_path, '-m', 'scrapers.' .. module }
 end
 
 local python_env_setup = false
+local _nix_submit_attempted = false
+
+---@return boolean
+local function discover_nix_submit_cmd()
+  local cache_dir = vim.fn.stdpath('cache') .. '/cp-nvim'
+  local cache_file = cache_dir .. '/nix-submit'
+
+  local f = io.open(cache_file, 'r')
+  if f then
+    local cached = f:read('*l')
+    f:close()
+    if cached and vim.fn.executable(cached) == 1 then
+      _nix_submit_cmd = cached
+      return true
+    end
+  end
+
+  local plugin_path = M.get_plugin_path()
+  vim.cmd.redraw()
+  vim.notify('Building submit environment...', vim.log.levels.INFO)
+  vim.cmd.redraw()
+  local result = vim
+    .system(
+      { 'nix', 'build', plugin_path .. '#submitEnv', '--no-link', '--print-out-paths' },
+      { text = true }
+    )
+    :wait()
+
+  if result.code ~= 0 then
+    logger.log('nix build #submitEnv failed: ' .. (result.stderr or ''), vim.log.levels.WARN)
+    return false
+  end
+
+  local store_path = result.stdout:gsub('%s+$', '')
+  local submit_cmd = store_path .. '/bin/cp-nvim-submit'
+
+  if vim.fn.executable(submit_cmd) ~= 1 then
+    logger.log('nix submit cmd not executable at ' .. submit_cmd, vim.log.levels.WARN)
+    return false
+  end
+
+  vim.fn.mkdir(cache_dir, 'p')
+  f = io.open(cache_file, 'w')
+  if f then
+    f:write(submit_cmd)
+    f:close()
+  end
+
+  _nix_submit_cmd = submit_cmd
+  return true
+end
+
+---@return boolean
+function M.setup_nix_submit_env()
+  if _nix_submit_cmd then
+    return true
+  end
+  if _nix_submit_attempted then
+    return false
+  end
+  _nix_submit_attempted = true
+  if vim.fn.executable('nix') == 1 then
+    return discover_nix_submit_cmd()
+  end
+  return false
+end
 
 ---@return boolean
 local function discover_nix_python()

pyproject.toml

@@ -7,7 +7,7 @@ requires-python = ">=3.11"
 dependencies = [
     "backoff>=2.2.1",
     "beautifulsoup4>=4.13.5",
-    "camoufox[geoip]",
+    "scrapling[fetchers]>=0.4",
     "curl-cffi>=0.13.0",
     "httpx>=0.28.1",
     "ndjson>=0.3.1",

scrapers/atcoder.py

@@ -4,7 +4,9 @@ import asyncio
 import json
 import os
 import re
+import subprocess
 import sys
+import tempfile
 import time
 from typing import Any
 
@@ -233,6 +235,183 @@ def _extract_samples(html: str) -> list[TestCase]:
     return cases
 
 
+_TURNSTILE_JS = "() => { const el = document.querySelector('[name=\"cf-turnstile-response\"]'); return el && el.value.length > 0; }"
+
+
+def _solve_turnstile(page) -> None:
+    for _ in range(6):
+        has_token = page.evaluate(_TURNSTILE_JS)
+        if has_token:
+            return
+        try:
+            box = page.locator(
+                'iframe[src*="challenges.cloudflare.com"]'
+            ).first.bounding_box()
+            if box:
+                page.mouse.click(
+                    box["x"] + box["width"] * 0.15,
+                    box["y"] + box["height"] * 0.5,
+                )
+        except Exception:
+            pass
+        try:
+            page.wait_for_function(_TURNSTILE_JS, timeout=5000)
+            return
+        except Exception:
+            pass
+    raise RuntimeError("Turnstile not solved after multiple attempts")
+
+
+def _ensure_browser() -> None:
+    try:
+        from patchright._impl._driver import compute_driver_executable  # type: ignore[import-untyped,unresolved-import]
+
+        node, cli = compute_driver_executable()
+        browser_info = subprocess.run(
+            [node, cli, "install", "--dry-run", "chromium"],
+            capture_output=True,
+            text=True,
+        )
+        for line in browser_info.stdout.splitlines():
+            if "Install location:" in line:
+                install_dir = line.split(":", 1)[1].strip()
+                if not os.path.isdir(install_dir):
+                    print(json.dumps({"status": "installing_browser"}), flush=True)
+                    subprocess.run(
+                        [node, cli, "install", "chromium"],
+                        check=True,
+                    )
+                break
+    except Exception:
+        pass
+
+
+def _submit_headless(
+    contest_id: str,
+    problem_id: str,
+    source_code: str,
+    language_id: str,
+    credentials: dict[str, str],
+) -> "SubmitResult":
+    from pathlib import Path
+
+    try:
+        from scrapling.fetchers import StealthySession  # type: ignore[import-untyped,unresolved-import]
+    except ImportError:
+        return SubmitResult(
+            success=False,
+            error="scrapling is required for AtCoder submit. Install it: uv add 'scrapling[fetchers]>=0.4'",
+        )
+
+    _ensure_browser()
+
+    cookie_cache = Path.home() / ".cache" / "cp-nvim" / "atcoder-cookies.json"
+    cookie_cache.parent.mkdir(parents=True, exist_ok=True)
+    saved_cookies: list[dict[str, Any]] = []
+    if cookie_cache.exists():
+        try:
+            saved_cookies = json.loads(cookie_cache.read_text())
+        except Exception:
+            pass
+
+    logged_in = False
+    login_error: str | None = None
+    submit_error: str | None = None
+
+    def check_login(page):
+        nonlocal logged_in
+        logged_in = page.evaluate(
+            "() => Array.from(document.querySelectorAll('a')).some(a => a.textContent.trim() === 'Sign Out')"
+        )
+
+    def login_action(page):
+        nonlocal login_error
+        try:
+            _solve_turnstile(page)
+            page.fill('input[name="username"]', credentials.get("username", ""))
+            page.fill('input[name="password"]', credentials.get("password", ""))
+            page.click("#submit")
+            page.wait_for_url(lambda url: "/login" not in url, timeout=60000)
+        except Exception as e:
+            login_error = str(e)
+
+    def submit_action(page):
+        nonlocal submit_error
+        try:
+            _solve_turnstile(page)
+            page.select_option(
+                'select[name="data.TaskScreenName"]',
+                f"{contest_id}_{problem_id}",
+            )
+            page.locator(
+                f'select[name="data.LanguageId"] option[value="{language_id}"]'
+            ).wait_for(state="attached", timeout=15000)
+            page.select_option('select[name="data.LanguageId"]', language_id)
+            with tempfile.NamedTemporaryFile(
+                mode="w", suffix=".cpp", delete=False, prefix="atcoder_"
+            ) as tf:
+                tf.write(source_code)
+                tmp_path = tf.name
+            try:
+                page.set_input_files("#input-open-file", tmp_path)
+                page.wait_for_timeout(500)
+            finally:
+                os.unlink(tmp_path)
+            page.locator('button[type="submit"]').click()
+            page.wait_for_url(lambda url: "/submissions/me" in url, timeout=60000)
+        except Exception as e:
+            submit_error = str(e)
+
+    try:
+        with StealthySession(
+            headless=True,
+            timeout=60000,
+            google_search=False,
+            cookies=saved_cookies,
+        ) as session:
+            print(json.dumps({"status": "checking_login"}), flush=True)
+            session.fetch(
+                f"{BASE_URL}/home",
+                page_action=check_login,
+                network_idle=True,
+            )
+
+            if not logged_in:
+                print(json.dumps({"status": "logging_in"}), flush=True)
+                session.fetch(
+                    f"{BASE_URL}/login",
+                    page_action=login_action,
+                    solve_cloudflare=True,
+                )
+                if login_error:
+                    return SubmitResult(
+                        success=False, error=f"Login failed: {login_error}"
+                    )
+
+            print(json.dumps({"status": "submitting"}), flush=True)
+            session.fetch(
+                f"{BASE_URL}/contests/{contest_id}/submit",
+                page_action=submit_action,
+                solve_cloudflare=True,
+            )
+
+            try:
+                browser_cookies = session.context.cookies()
+                if any(c["name"] == "REVEL_SESSION" for c in browser_cookies):
+                    cookie_cache.write_text(json.dumps(browser_cookies))
+            except Exception:
+                pass
+
+            if submit_error:
+                return SubmitResult(success=False, error=submit_error)
+
+            return SubmitResult(
+                success=True, error="", submission_id="", verdict="submitted"
+            )
+    except Exception as e:
+        return SubmitResult(success=False, error=str(e))
+
+
 def _scrape_tasks_sync(contest_id: str) -> list[dict[str, str]]:
     html = _fetch(f"{BASE_URL}/contests/{contest_id}/tasks")
     return _parse_tasks_list(html)
@@ -379,83 +558,14 @@ class AtcoderScraper(BaseScraper):
         language_id: str,
         credentials: dict[str, str],
     ) -> SubmitResult:
-        def _submit_sync() -> SubmitResult:
+        return await asyncio.to_thread(
-            try:
+            _submit_headless,
-                from camoufox.sync_api import Camoufox
+            contest_id,
-            except ImportError:
+            problem_id,
-                return SubmitResult(
+            source_code,
-                    success=False,
+            language_id,
-                    error="camoufox is required for AtCoder login. Install it: uv add 'camoufox[geoip]'",
+            credentials,
-                )
+        )
-
-            from curl_cffi import requests as curl_requests
-
-            try:
-                with Camoufox(headless=True) as browser:
-                    page = browser.new_page()
-                    page.goto(f"{BASE_URL}/login", wait_until="domcontentloaded")
-                    page.wait_for_load_state("networkidle")
-                    page.fill('input[name="username"]', credentials.get("username", ""))
-                    page.fill('input[name="password"]', credentials.get("password", ""))
-                    page.click('#submit')
-                    page.wait_for_url(lambda url: "/login" not in url, timeout=30000)
-                    cookies = page.context.cookies()
-
-                session = curl_requests.Session(impersonate="chrome")
-                for cookie in cookies:
-                    session.cookies.set(
-                        cookie["name"],
-                        cookie["value"],
-                        domain=cookie.get("domain", ""),
-                    )
-
-                submit_page = session.get(
-                    f"{BASE_URL}/contests/{contest_id}/submit",
-                    timeout=TIMEOUT_SECONDS,
-                )
-                submit_page.raise_for_status()
-                soup = BeautifulSoup(submit_page.text, "html.parser")
-                csrf_input = soup.find("input", {"name": "csrf_token"})
-                if not csrf_input or not hasattr(csrf_input, "get"):
-                    return SubmitResult(
-                        success=False, error="Could not find CSRF token on submit page"
-                    )
-                csrf_token = csrf_input.get("value", "") or ""  # type: ignore[union-attr]
-
-                task_screen_name = f"{contest_id}_{problem_id}"
-                submit_resp = session.post(
-                    f"{BASE_URL}/contests/{contest_id}/submit",
-                    data={
-                        "data.TaskScreenName": task_screen_name,
-                        "data.LanguageId": language_id,
-                        "sourceCode": source_code,
-                        "csrf_token": csrf_token,
-                    },
-                    timeout=TIMEOUT_SECONDS,
-                    allow_redirects=False,
-                )
-                if submit_resp.status_code in (301, 302):
-                    location = submit_resp.headers.get("Location", "")
-                    if "/submissions/me" in location:
-                        return SubmitResult(
-                            success=True,
-                            error="",
-                            submission_id="",
-                            verdict="submitted",
-                        )
-                    return SubmitResult(
-                        success=False,
-                        error=f"Submit may have failed: redirected to {location}",
-                    )
-                submit_resp.raise_for_status()
-                return SubmitResult(
-                    success=False,
-                    error="Unexpected response from submit (expected redirect)",
-                )
-            except Exception as e:
-                return SubmitResult(success=False, error=str(e))
-
-        return await asyncio.to_thread(_submit_sync)
 
 
 async def main_async() -> int:

scrapers/language_ids.py

@@ -1,7 +1,7 @@
 LANGUAGE_IDS = {
     "atcoder": {
-        "cpp": "5028",
+        "cpp": "6017",
-        "python": "5078",
+        "python": "6082",
     },
     "codeforces": {
         "cpp": "89",

scripts/ci.sh

@@ -4,8 +4,10 @@ set -eu
 nix develop --command stylua --check .
 git ls-files '*.lua' | xargs nix develop --command selene --display-style quiet
 nix develop --command prettier --check .
+nix fmt
+git diff --exit-code -- '*.nix'
 nix develop --command lua-language-server --check . --checklevel=Warning
-nix develop --command uvx ruff format --check .
+nix develop --command ruff format --check .
-nix develop --command uvx ruff check .
+nix develop --command ruff check .
-nix develop --command uvx ty check .
+nix develop --command ty check .
-nix develop --command uv run pytest tests/ -v
+nix develop --command python -m pytest tests/ -v