diff --git a/doc/cp.nvim.txt b/doc/cp.nvim.txt index f61d39e..2c0cc5c 100644 --- a/doc/cp.nvim.txt +++ b/doc/cp.nvim.txt @@ -3,6 +3,37 @@ Author: Barrett Ruth License: Same terms as Vim itself (see |license|) +============================================================================== +CONTENTS *cp-contents* + + 1. Introduction .................................................. |cp.nvim| + 2. Requirements ........................................ |cp-requirements| + 3. Setup ........................................................ |cp-setup| + 4. Configuration ................................................ |cp-config| + 5. Commands .................................................. |cp-commands| + 6. Mappings .................................................. |cp-mappings| + 7. Language Selection .................................. |cp-lang-selection| + 8. Workflow .................................................. |cp-workflow| + 9. Workflow Example ............................................ |cp-example| + 10. Verdict Formatting ................................. |cp-verdict-format| + 11. Picker Integration .......................................... |cp-picker| + 12. Picker Keymaps ........................................ |cp-picker-keys| + 13. Panel ........................................................ |cp-panel| + 14. Interactive Mode .......................................... |cp-interact| + 15. Stress Testing .............................................. |cp-stress| + 16. Race .......................................................... |cp-race| + 17. Credentials ............................................ |cp-credentials| + 18. Submit ...................................................... |cp-submit| + 19. ANSI Colors ................................................... |cp-ansi| + 20. Highlight Groups ........................................ |cp-highlights| + 21. Terminal Colors .................................... |cp-terminal-colors| + 22. Highlight Customization .......................... |cp-highlight-custom| + 23. Helpers .................................................... |cp-helpers| + 24. Statusline Integration .................................. |cp-statusline| + 25. Panel Keymaps .......................................... |cp-panel-keys| + 26. File Structure ................................................ |cp-files| + 27. Health Check ................................................ |cp-health| + ============================================================================== INTRODUCTION *cp.nvim* diff --git a/scrapers/atcoder.py b/scrapers/atcoder.py index 8be75ff..45a2195 100644 --- a/scrapers/atcoder.py +++ b/scrapers/atcoder.py @@ -29,11 +29,18 @@ from .models import ( TestCase, TestsResult, ) +from .timeouts import ( + BROWSER_ELEMENT_WAIT, + BROWSER_NAV_TIMEOUT, + BROWSER_SESSION_TIMEOUT, + BROWSER_SETTLE_DELAY, + BROWSER_TURNSTILE_POLL, + HTTP_TIMEOUT, +) MIB_TO_MB = 1.048576 BASE_URL = "https://atcoder.jp" ARCHIVE_URL = f"{BASE_URL}/contests/archive" -TIMEOUT_SECONDS = 30 HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" } @@ -76,7 +83,7 @@ def _retry_after_requests(details): on_backoff=_retry_after_requests, ) def _fetch(url: str) -> str: - r = _session.get(url, headers=HEADERS, timeout=TIMEOUT_SECONDS) + r = _session.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT) if r.status_code in RETRY_STATUS: raise requests.HTTPError(response=r) r.raise_for_status() @@ -99,7 +106,7 @@ def _giveup_httpx(exc: Exception) -> bool: giveup=_giveup_httpx, ) async def _get_async(client: httpx.AsyncClient, url: str) -> str: - r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_SECONDS) + r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT) r.raise_for_status() return r.text @@ -239,14 +246,14 @@ _TURNSTILE_JS = "() => { const el = document.querySelector('[name=\"cf-turnstile def _solve_turnstile(page) -> None: + if page.evaluate(_TURNSTILE_JS): + return + iframe_loc = page.locator('iframe[src*="challenges.cloudflare.com"]') + if not iframe_loc.count(): + return for _ in range(6): - has_token = page.evaluate(_TURNSTILE_JS) - if has_token: - return try: - box = page.locator( - 'iframe[src*="challenges.cloudflare.com"]' - ).first.bounding_box() + box = iframe_loc.first.bounding_box() if box: page.mouse.click( box["x"] + box["width"] * 0.15, @@ -255,7 +262,7 @@ def _solve_turnstile(page) -> None: except Exception: pass try: - page.wait_for_function(_TURNSTILE_JS, timeout=5000) + page.wait_for_function(_TURNSTILE_JS, timeout=BROWSER_TURNSTILE_POLL) return except Exception: pass @@ -331,7 +338,9 @@ def _submit_headless( page.fill('input[name="username"]', credentials.get("username", "")) page.fill('input[name="password"]', credentials.get("password", "")) page.click("#submit") - page.wait_for_url(lambda url: "/login" not in url, timeout=60000) + page.wait_for_url( + lambda url: "/login" not in url, timeout=BROWSER_NAV_TIMEOUT + ) except Exception as e: login_error = str(e) @@ -345,7 +354,7 @@ def _submit_headless( ) page.locator( f'select[name="data.LanguageId"] option[value="{language_id}"]' - ).wait_for(state="attached", timeout=15000) + ).wait_for(state="attached", timeout=BROWSER_ELEMENT_WAIT) page.select_option('select[name="data.LanguageId"]', language_id) with tempfile.NamedTemporaryFile( mode="w", suffix=".cpp", delete=False, prefix="atcoder_" @@ -354,18 +363,20 @@ def _submit_headless( tmp_path = tf.name try: page.set_input_files("#input-open-file", tmp_path) - page.wait_for_timeout(500) + page.wait_for_timeout(BROWSER_SETTLE_DELAY) finally: os.unlink(tmp_path) page.locator('button[type="submit"]').click() - page.wait_for_url(lambda url: "/submissions/me" in url, timeout=60000) + page.wait_for_url( + lambda url: "/submissions/me" in url, timeout=BROWSER_NAV_TIMEOUT + ) except Exception as e: submit_error = str(e) try: with StealthySession( headless=True, - timeout=60000, + timeout=BROWSER_SESSION_TIMEOUT, google_search=False, cookies=saved_cookies, ) as session: diff --git a/scrapers/codechef.py b/scrapers/codechef.py index 57ce33e..c4b9d37 100644 --- a/scrapers/codechef.py +++ b/scrapers/codechef.py @@ -9,6 +9,7 @@ import httpx from curl_cffi import requests as curl_requests from .base import BaseScraper, extract_precision +from .timeouts import HTTP_TIMEOUT from .models import ( ContestListResult, ContestSummary, @@ -26,7 +27,6 @@ PROBLEM_URL = "https://www.codechef.com/problems/{problem_id}" HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } -TIMEOUT_S = 15.0 CONNECTIONS = 8 MEMORY_LIMIT_RE = re.compile( r"Memory\s+[Ll]imit.*?([0-9.]+)\s*(MB|GB)", re.IGNORECASE | re.DOTALL @@ -34,7 +34,7 @@ MEMORY_LIMIT_RE = re.compile( async def fetch_json(client: httpx.AsyncClient, path: str) -> dict: - r = await client.get(BASE_URL + path, headers=HEADERS, timeout=TIMEOUT_S) + r = await client.get(BASE_URL + path, headers=HEADERS, timeout=HTTP_TIMEOUT) r.raise_for_status() return r.json() @@ -51,7 +51,7 @@ def _extract_memory_limit(html: str) -> float: def _fetch_html_sync(url: str) -> str: - response = curl_requests.get(url, impersonate="chrome", timeout=TIMEOUT_S) + response = curl_requests.get(url, impersonate="chrome", timeout=HTTP_TIMEOUT) response.raise_for_status() return response.text diff --git a/scrapers/codeforces.py b/scrapers/codeforces.py index c0495d8..fd0c129 100644 --- a/scrapers/codeforces.py +++ b/scrapers/codeforces.py @@ -2,7 +2,9 @@ import asyncio import json +import os import re +import tempfile from typing import Any import requests @@ -18,10 +20,16 @@ from .models import ( SubmitResult, TestCase, ) +from .timeouts import ( + BROWSER_ELEMENT_WAIT, + BROWSER_NAV_TIMEOUT, + BROWSER_SESSION_TIMEOUT, + BROWSER_SETTLE_DELAY, + HTTP_TIMEOUT, +) BASE_URL = "https://codeforces.com" API_CONTEST_LIST_URL = f"{BASE_URL}/api/contest.list" -TIMEOUT_SECONDS = 30 HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" } @@ -136,7 +144,7 @@ def _is_interactive(block: Tag) -> bool: def _fetch_problems_html(contest_id: str) -> str: url = f"{BASE_URL}/contest/{contest_id}/problems" - response = curl_requests.get(url, impersonate="chrome", timeout=TIMEOUT_SECONDS) + response = curl_requests.get(url, impersonate="chrome", timeout=HTTP_TIMEOUT) response.raise_for_status() return response.text @@ -223,7 +231,7 @@ class CodeforcesScraper(BaseScraper): async def scrape_contest_list(self) -> ContestListResult: try: - r = requests.get(API_CONTEST_LIST_URL, timeout=TIMEOUT_SECONDS) + r = requests.get(API_CONTEST_LIST_URL, timeout=HTTP_TIMEOUT) r.raise_for_status() data = r.json() if data.get("status") != "OK": @@ -289,13 +297,187 @@ class CodeforcesScraper(BaseScraper): language_id: str, credentials: dict[str, str], ) -> SubmitResult: + return await asyncio.to_thread( + _submit_headless, + contest_id, + problem_id, + source_code, + language_id, + credentials, + ) + + +def _wait_for_gate_reload(page, wait_selector: str) -> None: + from .atcoder import _solve_turnstile + + if "Verification" not in page.title(): + return + _solve_turnstile(page) + page.wait_for_function( + f"() => !!document.querySelector('{wait_selector}')", + timeout=BROWSER_ELEMENT_WAIT, + ) + + +def _submit_headless( + contest_id: str, + problem_id: str, + source_code: str, + language_id: str, + credentials: dict[str, str], +) -> SubmitResult: + from pathlib import Path + + try: + from scrapling.fetchers import StealthySession # type: ignore[import-untyped,unresolved-import] + except ImportError: return SubmitResult( success=False, - error="Codeforces submit not yet implemented", - submission_id="", - verdict="", + error="scrapling is required for Codeforces submit", ) + from .atcoder import _ensure_browser, _solve_turnstile + + _ensure_browser() + + cookie_cache = Path.home() / ".cache" / "cp-nvim" / "codeforces-cookies.json" + cookie_cache.parent.mkdir(parents=True, exist_ok=True) + saved_cookies: list[dict[str, Any]] = [] + if cookie_cache.exists(): + try: + saved_cookies = json.loads(cookie_cache.read_text()) + except Exception: + pass + + logged_in = False + login_error: str | None = None + submit_error: str | None = None + + def check_login(page): + nonlocal logged_in + logged_in = page.evaluate( + "() => Array.from(document.querySelectorAll('a'))" + ".some(a => a.textContent.includes('Logout'))" + ) + + def login_action(page): + nonlocal login_error + try: + _wait_for_gate_reload(page, "#enterForm") + except Exception: + pass + try: + page.fill( + 'input[name="handleOrEmail"]', + credentials.get("username", ""), + ) + page.fill( + 'input[name="password"]', + credentials.get("password", ""), + ) + page.locator('#enterForm input[type="submit"]').click() + page.wait_for_url( + lambda url: "/enter" not in url, timeout=BROWSER_NAV_TIMEOUT + ) + except Exception as e: + login_error = str(e) + + def submit_action(page): + nonlocal submit_error + try: + _solve_turnstile(page) + except Exception: + pass + tmp_path: str | None = None + try: + page.select_option( + 'select[name="submittedProblemIndex"]', + problem_id.upper(), + ) + page.select_option('select[name="programTypeId"]', language_id) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".cpp", delete=False, prefix="cf_" + ) as tf: + tf.write(source_code) + tmp_path = tf.name + try: + page.set_input_files('input[name="sourceFile"]', tmp_path) + page.wait_for_timeout(BROWSER_SETTLE_DELAY) + except Exception: + page.fill('textarea[name="source"]', source_code) + page.locator("form.submit-form input.submit").click(no_wait_after=True) + try: + page.wait_for_url( + lambda url: "/my" in url or "/status" in url, + timeout=BROWSER_NAV_TIMEOUT * 2, + ) + except Exception: + err_el = page.query_selector("span.error") + if err_el: + submit_error = err_el.inner_text().strip() + else: + submit_error = "Submit failed: page did not navigate" + except Exception as e: + submit_error = str(e) + finally: + if tmp_path: + try: + os.unlink(tmp_path) + except OSError: + pass + + try: + with StealthySession( + headless=True, + timeout=BROWSER_SESSION_TIMEOUT, + google_search=False, + cookies=saved_cookies, + ) as session: + print(json.dumps({"status": "checking_login"}), flush=True) + session.fetch( + f"{BASE_URL}/", + page_action=check_login, + network_idle=True, + ) + + if not logged_in: + print(json.dumps({"status": "logging_in"}), flush=True) + session.fetch( + f"{BASE_URL}/enter", + page_action=login_action, + solve_cloudflare=True, + ) + if login_error: + return SubmitResult( + success=False, error=f"Login failed: {login_error}" + ) + + print(json.dumps({"status": "submitting"}), flush=True) + session.fetch( + f"{BASE_URL}/contest/{contest_id}/submit", + page_action=submit_action, + solve_cloudflare=True, + ) + + try: + browser_cookies = session.context.cookies() + if any(c["name"] == "JSESSIONID" for c in browser_cookies): + cookie_cache.write_text(json.dumps(browser_cookies)) + except Exception: + pass + + if submit_error: + return SubmitResult(success=False, error=submit_error) + + return SubmitResult( + success=True, + error="", + submission_id="", + verdict="submitted", + ) + except Exception as e: + return SubmitResult(success=False, error=str(e)) + if __name__ == "__main__": CodeforcesScraper().run_cli() diff --git a/scrapers/cses.py b/scrapers/cses.py index b2e845a..7d9f4f0 100644 --- a/scrapers/cses.py +++ b/scrapers/cses.py @@ -9,6 +9,7 @@ from typing import Any import httpx from .base import BaseScraper, extract_precision +from .timeouts import HTTP_TIMEOUT, SUBMIT_POLL_TIMEOUT from .models import ( ContestListResult, ContestSummary, @@ -26,7 +27,6 @@ TASK_PATH = "/problemset/task/{id}" HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } -TIMEOUT_S = 15.0 CONNECTIONS = 8 CSES_LANGUAGES: dict[str, dict[str, str]] = { @@ -78,7 +78,7 @@ def snake_to_title(name: str) -> str: async def fetch_text(client: httpx.AsyncClient, path: str) -> str: - r = await client.get(BASE_URL + path, headers=HEADERS, timeout=TIMEOUT_S) + r = await client.get(BASE_URL + path, headers=HEADERS, timeout=HTTP_TIMEOUT) r.raise_for_status() return r.text @@ -290,7 +290,7 @@ class CSESScraper(BaseScraper): password: str, ) -> str | None: login_page = await client.get( - f"{BASE_URL}/login", headers=HEADERS, timeout=TIMEOUT_S + f"{BASE_URL}/login", headers=HEADERS, timeout=HTTP_TIMEOUT ) csrf_match = re.search(r'name="csrf_token" value="([^"]+)"', login_page.text) if not csrf_match: @@ -304,20 +304,20 @@ class CSESScraper(BaseScraper): "pass": password, }, headers=HEADERS, - timeout=TIMEOUT_S, + timeout=HTTP_TIMEOUT, ) if "Invalid username or password" in login_resp.text: return None api_resp = await client.post( - f"{API_URL}/login", headers=HEADERS, timeout=TIMEOUT_S + f"{API_URL}/login", headers=HEADERS, timeout=HTTP_TIMEOUT ) api_data = api_resp.json() token: str = api_data["X-Auth-Token"] auth_url: str = api_data["authentication_url"] - auth_page = await client.get(auth_url, headers=HEADERS, timeout=TIMEOUT_S) + auth_page = await client.get(auth_url, headers=HEADERS, timeout=HTTP_TIMEOUT) auth_csrf = re.search(r'name="csrf_token" value="([^"]+)"', auth_page.text) form_token = re.search(r'name="token" value="([^"]+)"', auth_page.text) if not auth_csrf or not form_token: @@ -330,18 +330,29 @@ class CSESScraper(BaseScraper): "token": form_token.group(1), }, headers=HEADERS, - timeout=TIMEOUT_S, + timeout=HTTP_TIMEOUT, ) check = await client.get( f"{API_URL}/login", headers={"X-Auth-Token": token, **HEADERS}, - timeout=TIMEOUT_S, + timeout=HTTP_TIMEOUT, ) if check.status_code != 200: return None return token + async def _check_token(self, client: httpx.AsyncClient, token: str) -> bool: + try: + r = await client.get( + f"{API_URL}/login", + headers={"X-Auth-Token": token, **HEADERS}, + timeout=HTTP_TIMEOUT, + ) + return r.status_code == 200 + except Exception: + return False + async def submit( self, contest_id: str, @@ -356,11 +367,30 @@ class CSESScraper(BaseScraper): return self._submit_error("Missing credentials. Use :CP login cses") async with httpx.AsyncClient(follow_redirects=True) as client: - print(json.dumps({"status": "logging_in"}), flush=True) + token = credentials.get("token") + + if token: + print(json.dumps({"status": "checking_login"}), flush=True) + if not await self._check_token(client, token): + token = None - token = await self._web_login(client, username, password) if not token: - return self._submit_error("Login failed (bad credentials?)") + print(json.dumps({"status": "logging_in"}), flush=True) + token = await self._web_login(client, username, password) + if not token: + return self._submit_error("Login failed (bad credentials?)") + print( + json.dumps( + { + "credentials": { + "username": username, + "password": password, + "token": token, + } + } + ), + flush=True, + ) print(json.dumps({"status": "submitting"}), flush=True) @@ -383,7 +413,7 @@ class CSESScraper(BaseScraper): "Content-Type": "application/json", **HEADERS, }, - timeout=TIMEOUT_S, + timeout=HTTP_TIMEOUT, ) if r.status_code not in range(200, 300): @@ -406,7 +436,7 @@ class CSESScraper(BaseScraper): "X-Auth-Token": token, **HEADERS, }, - timeout=30.0, + timeout=SUBMIT_POLL_TIMEOUT, ) if r.status_code == 200: info = r.json() diff --git a/scrapers/kattis.py b/scrapers/kattis.py index d1675bf..2bfd2d6 100644 --- a/scrapers/kattis.py +++ b/scrapers/kattis.py @@ -10,6 +10,7 @@ from datetime import datetime import httpx from .base import BaseScraper +from .timeouts import HTTP_TIMEOUT from .models import ( ContestListResult, ContestSummary, @@ -23,7 +24,6 @@ BASE_URL = "https://open.kattis.com" HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } -TIMEOUT_S = 15.0 CONNECTIONS = 8 TIME_RE = re.compile( @@ -37,13 +37,13 @@ MEM_RE = re.compile( async def _fetch_text(client: httpx.AsyncClient, url: str) -> str: - r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S) + r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT) r.raise_for_status() return r.text async def _fetch_bytes(client: httpx.AsyncClient, url: str) -> bytes: - r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S) + r = await client.get(url, headers=HEADERS, timeout=HTTP_TIMEOUT) r.raise_for_status() return r.content diff --git a/scrapers/timeouts.py b/scrapers/timeouts.py new file mode 100644 index 0000000..a21ad0d --- /dev/null +++ b/scrapers/timeouts.py @@ -0,0 +1,9 @@ +HTTP_TIMEOUT = 15.0 + +BROWSER_SESSION_TIMEOUT = 15000 +BROWSER_NAV_TIMEOUT = 10000 +BROWSER_TURNSTILE_POLL = 5000 +BROWSER_ELEMENT_WAIT = 10000 +BROWSER_SETTLE_DELAY = 500 + +SUBMIT_POLL_TIMEOUT = 30.0 diff --git a/scrapers/usaco.py b/scrapers/usaco.py index 565f1b5..9e4d7da 100644 --- a/scrapers/usaco.py +++ b/scrapers/usaco.py @@ -8,6 +8,7 @@ from typing import Any, cast import httpx from .base import BaseScraper +from .timeouts import HTTP_TIMEOUT from .models import ( ContestListResult, ContestSummary, @@ -21,7 +22,6 @@ BASE_URL = "http://www.usaco.org" HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } -TIMEOUT_S = 15.0 CONNECTIONS = 4 MONTHS = [ @@ -58,7 +58,9 @@ RESULTS_PAGE_RE = re.compile( async def _fetch_text(client: httpx.AsyncClient, url: str) -> str: - r = await client.get(url, headers=HEADERS, timeout=TIMEOUT_S, follow_redirects=True) + r = await client.get( + url, headers=HEADERS, timeout=HTTP_TIMEOUT, follow_redirects=True + ) r.raise_for_status() return r.text diff --git a/t/1068.cc b/t/1068.cc deleted file mode 100644 index 5d3fe37..0000000 --- a/t/1068.cc +++ /dev/null @@ -1,54 +0,0 @@ -#include // {{{ - -#include -#ifdef __cpp_lib_ranges_enumerate -#include -namespace rv = std::views; -namespace rs = std::ranges; -#endif - -#pragma GCC optimize("O2,unroll-loops") -#pragma GCC target("avx2,bmi,bmi2,lzcnt,popcnt") - -using namespace std; - -using i32 = int32_t; -using u32 = uint32_t; -using i64 = int64_t; -using u64 = uint64_t; -using f64 = double; -using f128 = long double; - -#if __cplusplus >= 202002L -template -constexpr T MIN = std::numeric_limits::min(); - -template -constexpr T MAX = std::numeric_limits::max(); -#endif - -#ifdef LOCAL -#define db(...) std::print(__VA_ARGS__) -#define dbln(...) std::println(__VA_ARGS__) -#else -#define db(...) -#define dbln(...) -#endif -// }}} - -void solve() { - cout << "hi\n"; -} - -int main() { // {{{ - std::cin.exceptions(std::cin.failbit); -#ifdef LOCAL - std::cerr.rdbuf(std::cout.rdbuf()); - std::cout.setf(std::ios::unitbuf); - std::cerr.setf(std::ios::unitbuf); -#else - std::cin.tie(nullptr)->sync_with_stdio(false); -#endif - solve(); - return 0; -} // }}}