diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index a608648..97b7786 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -27,8 +27,7 @@ jobs: - 'ftdetect/**' - '*.lua' - '.luarc.json' - - 'stylua.toml' - - 'selene.toml' + - '*.toml' python: - 'scrapers/**' - 'tests/scrapers/**' diff --git a/doc/cp.txt b/doc/cp.txt index 1ad6d93..67493f8 100644 --- a/doc/cp.txt +++ b/doc/cp.txt @@ -9,8 +9,7 @@ INTRODUCTION *cp* *cp.nvim* cp.nvim is a competitive programming plugin that automates problem setup, compilation, and testing workflow for online judges. -Supported platforms: AtCoder, Codeforces, CSES -Supported languages: C++, Python +Supported platforms (for now!): AtCoder, Codeforces, CSES ============================================================================== REQUIREMENTS *cp-requirements* @@ -72,6 +71,9 @@ COMMANDS *cp-commands* Use --debug flag to compile with debug flags. Requires contest setup first. + :CP pick Launch configured picker for interactive + platform/contest/problem selection. + Navigation Commands ~ :CP next Navigate to next problem in current contest. Stops at last problem (no wrapping). @@ -131,10 +133,11 @@ Here's an example configuration with lazy.nvim: >lua default = { cpp = { compile = { 'g++', '{source}', '-o', '{binary}', - '-std=c++17' }, + '-std=c++17', '-fdiagnostic-colors=always' }, test = { '{binary}' }, debug = { 'g++', '{source}', '-o', '{binary}', '-std=c++17', '-g', + '-fdiagnostic-colors=always' '-fsanitize=address,undefined' }, }, python = { @@ -143,14 +146,8 @@ Here's an example configuration with lazy.nvim: >lua }, }, snippets = {}, - hooks = { - before_run = nil, - before_debug = nil, - setup_code = nil, - }, debug = false, scrapers = { 'atcoder', 'codeforces', 'cses' }, - filename = default_filename, -- + run_panel = { ansi = true, diff_mode = 'vim', @@ -165,6 +162,7 @@ Here's an example configuration with lazy.nvim: >lua '--word-diff-regex=.', '--no-prefix' }, }, }, + picker = 'telescope', -- 'telescope', 'fzf-lua', or nil (disabled) } } < @@ -180,6 +178,9 @@ Here's an example configuration with lazy.nvim: >lua Default: all scrapers enabled {run_panel} (|RunPanelConfig|) Test panel behavior configuration. {diff} (|DiffConfig|) Diff backend configuration. + {picker} (string, optional) Picker integration: "telescope", + "fzf-lua", or nil to disable. When enabled, provides + :CP pick for interactive platform/contest/problem selection. {filename} (function, optional) Custom filename generation. function(contest, contest_id, problem_id, config, language) Should return full filename with extension. @@ -279,21 +280,6 @@ AtCoder ~ *cp-atcoder* URL format: https://atcoder.jp/contests/abc123/tasks/abc123_a -AtCoder contests use consistent naming patterns where contest ID and problem -ID are combined to form the task name. - -Platform characteristics: -• Contest types: ABC (Beginner), ARC (Regular), AGC (Grand), etc. -• Problem naming: Contest ID + problem letter (e.g. "abc324_a") -• Multi-test problems: Handled with conditional compilation directives -• Template features: Includes fast I/O and common competitive programming - headers - -In terms of cp.nvim, this corresponds to: -- Platform: atcoder -- Contest ID: abc123 (from URL path segment) -- Problem ID: a (single letter, extracted from task name) - Usage examples: > :CP atcoder abc324 a " Full setup: problem A from contest ABC324 :CP atcoder abc324 " Contest setup: load contest metadata only @@ -308,21 +294,6 @@ Codeforces ~ *cp-codeforces* URL format: https://codeforces.com/contest/1234/problem/A -Codeforces uses numeric contest IDs with letter-based problem identifiers. -Educational rounds, gym contests, and regular contests all follow this pattern. - -Platform characteristics: -• Contest types: Regular, Educational, Div. 1/2/3, Global rounds -• Problem naming: Numeric contest + problem letter -• Multi-test support: Template includes test case loop structure -• Interactive problems: Supported with flush handling -• Time/memory limits: Typically 1-2 seconds, 256 MB - -In terms of cp.nvim, this corresponds to: -- Platform: codeforces -- Contest ID: 1234 (numeric, from URL) -- Problem ID: a (lowercase letter, normalized from URL) - Usage examples: > :CP codeforces 1934 a " Full setup: problem A from contest 1934 :CP codeforces 1934 " Contest setup: load contest metadata only @@ -336,21 +307,6 @@ CSES ~ *cp-cses* URL format: https://cses.fi/problemset/task/1068 -CSES (Code Submission Evaluation System) is organized by problem categories -rather than traditional contests. Problems are grouped by topic and difficulty. - -Platform characteristics: -• Organization: Category-based (Introductory, Sorting, Dynamic Programming) -• Problem numbering: Sequential numeric IDs (1001, 1068, etc.) -• Difficulty progression: Problems increase in complexity within categories -• No time pressure: Educational focus rather than contest environment -• Cache expiry: 30 days (problems may be updated periodically) - -In terms of cp.nvim, this corresponds to: -- Platform: cses -- Contest ID: Category name (introductory_problems, sorting_and_searching) -- Problem ID: Problem number (1068, 1640) - Usage examples: > :CP cses dynamic_programming 1633 " Set up problem 1633 from DP category :CP cses dynamic_programming " Set up ALL problems from DP category @@ -361,7 +317,7 @@ Usage examples: > ============================================================================== -COMPLETE WORKFLOW EXAMPLE *cp-example* +COMPLETE WORKFLOW EXAMPLE *cp-example* Example: Setting up and solving AtCoder contest ABC324 @@ -398,6 +354,39 @@ Example: Setting up and solving AtCoder contest ABC324 9. Submit solutions on AtCoder website +============================================================================== +PICKER INTEGRATION *cp-picker* + +When picker integration is enabled in configuration, cp.nvim provides interactive +platform, contest, and problem selection using telescope.nvim or fzf-lua. + +:CP pick *:CP-pick* + Launch configured picker for interactive problem selection. + Control Flow: Select Platform → Contest → Problem → Code! + + Requires picker = 'telescope' or picker = 'fzf-lua' in configuration. + Requires corresponding plugin (telescope.nvim or fzf-lua) to be installed. + +Picker Controls ~ + *cp-picker-controls* + The picker interface provides several keyboard shortcuts for enhanced control: + + Force refresh contest list, bypassing cache + Useful when contest lists are outdated or incomplete + Shows loading indicator during refresh operation + + Standard picker controls (telescope.nvim/fzf-lua): + Select current item and proceed to next step + / Cancel picker and return to editor + / Navigate to next item + / Navigate to previous item + / Start filtering/searching items + +Notes ~ + • Contest lists are fetched dynamically using scrapers with a TTL of 1 day + • Use to force refresh + • Large contest lists may take time to load + ============================================================================== RUN PANEL *cp-run* @@ -448,13 +437,13 @@ Test cases use competitive programming terminology with color highlighting: < ============================================================================== -ANSI COLORS AND HIGHLIGHTING *cp-ansi* +ANSI COLORS AND HIGHLIGHTING *cp-ansi* cp.nvim provides comprehensive ANSI color support and highlighting for compiler output, program stderr, and diff visualization. ============================================================================== -HIGHLIGHT GROUPS *cp-highlights* +HIGHLIGHT GROUPS *cp-highlights* Test Status Groups ~ @@ -514,20 +503,19 @@ These groups are automatically used by the git diff backend for character-level difference visualization with optimal colorscheme integration. ============================================================================== -TERMINAL COLOR INTEGRATION *cp-terminal-colors* +TERMINAL COLOR INTEGRATION *cp-terminal-colors* ANSI colors automatically use your terminal's color palette through Neovim's vim.g.terminal_color_* variables. This ensures compiler colors match your colorscheme without manual configuration. -If your colorscheme doesn't set terminal colors, cp.nvim falls back to -sensible defaults. You can override terminal colors in your configuration: >vim - let g:terminal_color_1 = '#ff6b6b' " Custom red - let g:terminal_color_2 = '#51cf66' " Custom green -< +If your colorscheme doesn't set terminal colors, cp.nvim will warn you and +ANSI colors won't display properly - set them like so: >vim + let g:terminal_color_1 = '#ff6b6b' + ... ============================================================================== -HIGHLIGHT CUSTOMIZATION *cp-highlight-custom* +HIGHLIGHT CUSTOMIZATION *cp-highlight-custom* You can customize any highlight group by linking to existing groups or defining custom colors: >lua @@ -552,7 +540,7 @@ prevent them from being overridden: >lua < ============================================================================== -RUN PANEL KEYMAPS *cp-test-keys* +RUN PANEL KEYMAPS *cp-test-keys* Navigate to next test case (configurable via run_panel.next_test_key) Navigate to previous test case (configurable via @@ -591,97 +579,6 @@ cp.nvim creates the following file structure upon problem setup: > {problem_id}.n.cpout " nth program output {problem_id}.expected " Expected output < - -The plugin automatically manages this structure and navigation between problems -maintains proper file associations. - -============================================================================== -CACHING SYSTEM *cp-caching* - -cp.nvim maintains a persistent cache to improve performance and enable offline -functionality. The cache stores contest metadata, problem lists, test cases, -and file-to-context mappings. - -Cache Location ~ - *cp-cache-location* -Cache is stored at: > - vim.fn.stdpath('data') .. '/cp-nvim.json' -< -Cache Structure ~ - *cp-cache-structure* -The cache contains four main sections: - -contest_data Contest metadata and problem lists -• Indexed by: `platform:contest_id` -• Contains: Problem names, IDs, URLs, constraints -• Expiry: Platform-dependent (see |cp-cache-expiry|) - -test_cases Scraped test case input/output pairs -• Indexed by: `platform:contest_id:problem_id` -• Contains: Input data, expected output, test case count -• Expiry: Never (local test data persists) - -file_states File-to-context mapping -• Indexed by: Absolute file path -• Contains: Platform, contest_id, problem_id, language -• Purpose: Enables context restoration with `:CP` - -timestamps Last update times for cache validation -• Tracks: When each cache entry was last refreshed -• Used for: Expiry checking and incremental updates - -Cache Expiry Policy ~ - *cp-cache-expiry* -Different data types have different expiry policies: - -AtCoder/Codeforces contest data Never expires (contests are immutable) -CSES problem data 30 days (problems may be updated) -Test cases Never expires (local test data) -File states Never expires (tracks user workspace) - -Manual Cache Management ~ - *cp-cache-management* -While cache management is automatic, you can manually intervene: - -Clear specific contest cache: > - :lua require('cp.cache').clear_contest('atcoder', 'abc324') -< -Clear all cache data: > - :lua require('cp.cache').clear_all() -< -Force refresh contest metadata: > - :lua require('cp.cache').refresh_contest('codeforces', '1934') -< -View cache statistics: > - :lua print(vim.inspect(require('cp.cache').get_stats())) -< - Note: Manual cache operations require Lua - knowledge and are primarily for debugging. - -Offline Functionality ~ - *cp-cache-offline* -The cache enables limited offline functionality: - -✓ Restore context from cached file states -✓ Navigate between cached problems in a contest -✓ Access cached test cases for local development -✓ Use cached templates and configuration - -✗ Scrape new problems without internet connection -✗ Download new contest metadata -✗ Update problem constraints or test cases - -Performance Considerations ~ - *cp-cache-performance* -The cache provides several performance benefits: - -• Instant context restoration: No network requests needed -• Fast problem navigation: Problem lists loaded from cache -• Reduced scraping: Test cases cached after first download -• Batch operations: Multiple problems can be set up quickly - -Cache size typically remains under 1MB even with extensive usage. - ============================================================================== SNIPPETS *cp-snippets* @@ -690,7 +587,15 @@ snippets include basic C++ and Python templates for each contest type. Snippet trigger names must match the following format exactly: > - cp.nvim/{platform} + cp.nvim/{platform}.{language} +< +Where {platform} is the contest platform (atcoder, codeforces, cses) and +{language} is the programming language (cpp, python). + +Examples: > + cp.nvim/atcoder.cpp + cp.nvim/codeforces.python + cp.nvim/cses.cpp < Custom snippets can be added via the `snippets` configuration field. diff --git a/lua/cp/cache.lua b/lua/cp/cache.lua index 5d8f6b8..04d7dbd 100644 --- a/lua/cp/cache.lua +++ b/lua/cp/cache.lua @@ -7,6 +7,12 @@ ---@class CacheData ---@field [string] table ---@field file_states? table +---@field contest_lists? table + +---@class ContestListData +---@field contests table[] +---@field cached_at number +---@field expires_at number ---@class ContestData ---@field problems Problem[] @@ -33,6 +39,12 @@ local cache_file = vim.fn.stdpath('data') .. '/cp-nvim.json' local cache_data = {} local loaded = false +local CONTEST_LIST_TTL = { + cses = 7 * 24 * 60 * 60, -- 1 week + codeforces = 24 * 60 * 60, -- 1 day + atcoder = 24 * 60 * 60, -- 1 day +} + ---@param platform string ---@return number? local function get_expiry_date(platform) @@ -277,4 +289,57 @@ function M.set_file_state(file_path, platform, contest_id, problem_id, language) M.save() end +---@param platform string +---@return table[]? +function M.get_contest_list(platform) + vim.validate({ + platform = { platform, 'string' }, + }) + + if not cache_data.contest_lists or not cache_data.contest_lists[platform] then + return nil + end + + local contest_list_data = cache_data.contest_lists[platform] + if os.time() >= contest_list_data.expires_at then + return nil + end + + return contest_list_data.contests +end + +---@param platform string +---@param contests table[] +function M.set_contest_list(platform, contests) + vim.validate({ + platform = { platform, 'string' }, + contests = { contests, 'table' }, + }) + + if not cache_data.contest_lists then + cache_data.contest_lists = {} + end + + local ttl = CONTEST_LIST_TTL[platform] or (24 * 60 * 60) -- Default 1 day + cache_data.contest_lists[platform] = { + contests = contests, + cached_at = os.time(), + expires_at = os.time() + ttl, + } + + M.save() +end + +---@param platform string +function M.clear_contest_list(platform) + vim.validate({ + platform = { platform, 'string' }, + }) + + if cache_data.contest_lists and cache_data.contest_lists[platform] then + cache_data.contest_lists[platform] = nil + M.save() + end +end + return M diff --git a/lua/cp/config.lua b/lua/cp/config.lua index a1564fd..838107b 100644 --- a/lua/cp/config.lua +++ b/lua/cp/config.lua @@ -52,6 +52,7 @@ ---@field filename? fun(contest: string, contest_id: string, problem_id?: string, config: cp.Config, language?: string): string ---@field run_panel RunPanelConfig ---@field diff DiffConfig +---@field picker "telescope"|"fzf-lua"|nil ---@class cp.UserConfig ---@field contests? table @@ -62,6 +63,7 @@ ---@field filename? fun(contest: string, contest_id: string, problem_id?: string, config: cp.Config, language?: string): string ---@field run_panel? RunPanelConfig ---@field diff? DiffConfig +---@field picker? "telescope"|"fzf-lua"|nil local M = {} local constants = require('cp.constants') @@ -110,6 +112,7 @@ M.defaults = { args = { 'diff', '--no-index', '--word-diff=plain', '--word-diff-regex=.', '--no-prefix' }, }, }, + picker = nil, } ---@param user_config cp.UserConfig|nil @@ -129,6 +132,7 @@ function M.setup(user_config) filename = { user_config.filename, { 'function', 'nil' }, true }, run_panel = { user_config.run_panel, { 'table', 'nil' }, true }, diff = { user_config.diff, { 'table', 'nil' }, true }, + picker = { user_config.picker, { 'string', 'nil' }, true }, }) if user_config.contests then @@ -164,6 +168,12 @@ function M.setup(user_config) end end end + + if user_config.picker then + if not vim.tbl_contains({ 'telescope', 'fzf-lua' }, user_config.picker) then + error(("Invalid picker '%s'. Must be 'telescope' or 'fzf-lua'"):format(user_config.picker)) + end + end end local config = vim.tbl_deep_extend('force', M.defaults, user_config or {}) diff --git a/lua/cp/constants.lua b/lua/cp/constants.lua index 13cb27d..c14569b 100644 --- a/lua/cp/constants.lua +++ b/lua/cp/constants.lua @@ -1,7 +1,13 @@ local M = {} M.PLATFORMS = { 'atcoder', 'codeforces', 'cses' } -M.ACTIONS = { 'run', 'next', 'prev' } +M.ACTIONS = { 'run', 'next', 'prev', 'pick' } + +M.PLATFORM_DISPLAY_NAMES = { + atcoder = 'AtCoder', + codeforces = 'CodeForces', + cses = 'CSES', +} M.CPP = 'cpp' M.PYTHON = 'python' diff --git a/lua/cp/health.lua b/lua/cp/health.lua index af89cc8..563c065 100644 --- a/lua/cp/health.lua +++ b/lua/cp/health.lua @@ -1,5 +1,7 @@ local M = {} +local utils = require('cp.utils') + local function check_nvim_version() if vim.fn.has('nvim-0.10.0') == 1 then vim.health.ok('Neovim 0.10.0+ detected') @@ -22,8 +24,7 @@ local function check_uv() end local function check_python_env() - local plugin_path = debug.getinfo(1, 'S').source:sub(2) - plugin_path = vim.fn.fnamemodify(plugin_path, ':h:h:h') + local plugin_path = utils.get_plugin_path() local venv_dir = plugin_path .. '/.venv' if vim.fn.isdirectory(venv_dir) == 1 then @@ -34,8 +35,7 @@ local function check_python_env() end local function check_scrapers() - local plugin_path = debug.getinfo(1, 'S').source:sub(2) - plugin_path = vim.fn.fnamemodify(plugin_path, ':h:h:h') + local plugin_path = utils.get_plugin_path() local scrapers = { 'atcoder.py', 'codeforces.py', 'cses.py' } for _, scraper in ipairs(scrapers) do diff --git a/lua/cp/init.lua b/lua/cp/init.lua index 36c2faf..1c21880 100644 --- a/lua/cp/init.lua +++ b/lua/cp/init.lua @@ -83,6 +83,17 @@ local function setup_problem(contest_id, problem_id, language) state.test_cases = cached_test_cases logger.log(('using cached test cases (%d)'):format(#cached_test_cases)) elseif vim.tbl_contains(config.scrapers, state.platform) then + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[state.platform] or state.platform + logger.log( + ('Scraping %s %s %s for test cases, this may take a few seconds...'):format( + platform_display_name, + contest_id, + problem_id + ), + vim.log.levels.INFO, + true + ) + local scrape_result = scrape.scrape_problem(ctx) if not scrape_result.success then @@ -562,9 +573,9 @@ local function toggle_run_panel(is_debug) config.hooks.before_debug(ctx) end - local execute_module = require('cp.execute') + local execute = require('cp.execute') local contest_config = config.contests[state.platform] - local compile_result = execute_module.compile_problem(ctx, contest_config, is_debug) + local compile_result = execute.compile_problem(ctx, contest_config, is_debug) if compile_result.success then run.run_all_test_cases(ctx, contest_config, config) else @@ -698,6 +709,48 @@ local function navigate_problem(delta, language) setup_problem(state.contest_id, new_problem.id, language) end +local function handle_pick_action() + if not config.picker then + logger.log( + 'No picker configured. Set picker = "telescope" or picker = "fzf-lua" in config', + vim.log.levels.ERROR + ) + return + end + + if config.picker == 'telescope' then + local ok = pcall(require, 'telescope') + if not ok then + logger.log( + 'Telescope not available. Install telescope.nvim or change picker config', + vim.log.levels.ERROR + ) + return + end + local ok_cp, telescope_cp = pcall(require, 'cp.pickers.telescope') + if not ok_cp then + logger.log('Failed to load telescope integration', vim.log.levels.ERROR) + return + end + telescope_cp.platform_picker() + elseif config.picker == 'fzf-lua' then + local ok, _ = pcall(require, 'fzf-lua') + if not ok then + logger.log( + 'fzf-lua not available. Install fzf-lua or change picker config', + vim.log.levels.ERROR + ) + return + end + local ok_cp, fzf_cp = pcall(require, 'cp.pickers.fzf_lua') + if not ok_cp then + logger.log('Failed to load fzf-lua integration', vim.log.levels.ERROR) + return + end + fzf_cp.platform_picker() + end +end + local function restore_from_current_file() local current_file = vim.fn.expand('%:p') if current_file == '' then @@ -837,6 +890,8 @@ function M.handle_command(opts) navigate_problem(1, cmd.language) elseif cmd.action == 'prev' then navigate_problem(-1, cmd.language) + elseif cmd.action == 'pick' then + handle_pick_action() end return end diff --git a/lua/cp/log.lua b/lua/cp/log.lua index ca80de1..fd98c8a 100644 --- a/lua/cp/log.lua +++ b/lua/cp/log.lua @@ -6,9 +6,9 @@ function M.set_config(user_config) config = user_config end -function M.log(msg, level) +function M.log(msg, level, override) level = level or vim.log.levels.INFO - if not config or config.debug or level >= vim.log.levels.WARN then + if not config or config.debug or level >= vim.log.levels.WARN or override then vim.notify(('[cp.nvim]: %s'):format(msg), level) end end diff --git a/lua/cp/pickers/fzf_lua.lua b/lua/cp/pickers/fzf_lua.lua new file mode 100644 index 0000000..cf3a47e --- /dev/null +++ b/lua/cp/pickers/fzf_lua.lua @@ -0,0 +1,131 @@ +local picker_utils = require('cp.pickers') + +local function problem_picker(platform, contest_id) + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + local fzf = require('fzf-lua') + local problems = picker_utils.get_problems_for_contest(platform, contest_id) + + if #problems == 0 then + vim.notify( + ('No problems found for contest: %s %s'):format(platform_display_name, contest_id), + vim.log.levels.WARN + ) + return + end + + local entries = vim.tbl_map(function(problem) + return problem.display_name + end, problems) + + return fzf.fzf_exec(entries, { + prompt = ('Select Problem (%s %s)> '):format(platform_display_name, contest_id), + actions = { + ['default'] = function(selected) + if not selected or #selected == 0 then + return + end + + local selected_name = selected[1] + local problem = nil + for _, p in ipairs(problems) do + if p.display_name == selected_name then + problem = p + break + end + end + + if problem then + picker_utils.setup_problem(platform, contest_id, problem.id) + end + end, + }, + }) +end + +local function contest_picker(platform) + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + local fzf = require('fzf-lua') + local contests = picker_utils.get_contests_for_platform(platform) + + if #contests == 0 then + vim.notify( + ('No contests found for platform: %s'):format(platform_display_name), + vim.log.levels.WARN + ) + return + end + + local entries = vim.tbl_map(function(contest) + return contest.display_name + end, contests) + + return fzf.fzf_exec(entries, { + prompt = ('Select Contest (%s)> '):format(platform_display_name), + fzf_opts = { + ['--header'] = 'ctrl-r: refresh', + }, + actions = { + ['default'] = function(selected) + if not selected or #selected == 0 then + return + end + + local selected_name = selected[1] + local contest = nil + for _, c in ipairs(contests) do + if c.display_name == selected_name then + contest = c + break + end + end + + if contest then + problem_picker(platform, contest.id) + end + end, + ['ctrl-r'] = function() + local cache = require('cp.cache') + cache.clear_contest_list(platform) + contest_picker(platform) + end, + }, + }) +end + +local function platform_picker() + local fzf = require('fzf-lua') + local platforms = picker_utils.get_platforms() + local entries = vim.tbl_map(function(platform) + return platform.display_name + end, platforms) + + return fzf.fzf_exec(entries, { + prompt = 'Select Platform> ', + actions = { + ['default'] = function(selected) + if not selected or #selected == 0 then + return + end + + local selected_name = selected[1] + local platform = nil + for _, p in ipairs(platforms) do + if p.display_name == selected_name then + platform = p + break + end + end + + if platform then + contest_picker(platform.id) + end + end, + }, + }) +end + +return { + platform_picker = platform_picker, +} diff --git a/lua/cp/pickers/init.lua b/lua/cp/pickers/init.lua new file mode 100644 index 0000000..5a1b644 --- /dev/null +++ b/lua/cp/pickers/init.lua @@ -0,0 +1,171 @@ +local M = {} + +local cache = require('cp.cache') +local logger = require('cp.log') +local scrape = require('cp.scrape') +local utils = require('cp.utils') + +---@class cp.PlatformItem +---@field id string Platform identifier (e.g. "codeforces", "atcoder", "cses") +---@field display_name string Human-readable platform name (e.g. "Codeforces", "AtCoder", "CSES") + +---@class cp.ContestItem +---@field id string Contest identifier (e.g. "1951", "abc324", "sorting") +---@field name string Full contest name (e.g. "Educational Codeforces Round 168") +---@field display_name string Formatted display name for picker + +---@class cp.ProblemItem +---@field id string Problem identifier (e.g. "a", "b", "c") +---@field name string Problem name (e.g. "Two Permutations", "Painting Walls") +---@field display_name string Formatted display name for picker + +---Get list of available competitive programming platforms +---@return cp.PlatformItem[] +local function get_platforms() + local constants = require('cp.constants') + return vim.tbl_map(function(platform) + return { + id = platform, + display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform, + } + end, constants.PLATFORMS) +end + +---Get list of contests for a specific platform +---@param platform string Platform identifier (e.g. "codeforces", "atcoder") +---@return cp.ContestItem[] +local function get_contests_for_platform(platform) + local contests = {} + + cache.load() + local cached_contests = cache.get_contest_list(platform) + if cached_contests then + return cached_contests + end + + if not utils.setup_python_env() then + return contests + end + + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + logger.log( + ('Scraping %s for contests, this may take a few seconds...'):format(platform_display_name), + vim.log.levels.INFO, + true + ) + + local plugin_path = utils.get_plugin_path() + local cmd = { + 'uv', + 'run', + '--directory', + plugin_path, + '-m', + 'scrapers.' .. platform, + 'contests', + } + local result = vim + .system(cmd, { + cwd = plugin_path, + text = true, + timeout = 30000, + }) + :wait() + + if result.code ~= 0 then + logger.log( + ('Failed to get contests for %s: %s'):format(platform, result.stderr or 'unknown error'), + vim.log.levels.ERROR + ) + return contests + end + + local ok, data = pcall(vim.json.decode, result.stdout) + if not ok or not data.success then + logger.log(('Failed to parse contest data for %s'):format(platform), vim.log.levels.ERROR) + return contests + end + + for _, contest in ipairs(data.contests or {}) do + table.insert(contests, { + id = contest.id, + name = contest.name, + display_name = contest.display_name, + }) + end + + cache.set_contest_list(platform, contests) + return contests +end + +---Get list of problems for a specific contest +---@param platform string Platform identifier +---@param contest_id string Contest identifier +---@return cp.ProblemItem[] +local function get_problems_for_contest(platform, contest_id) + local problems = {} + + cache.load() + local contest_data = cache.get_contest_data(platform, contest_id) + if contest_data and contest_data.problems then + for _, problem in ipairs(contest_data.problems) do + table.insert(problems, { + id = problem.id, + name = problem.name, + display_name = problem.name, + }) + end + return problems + end + + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + logger.log( + ('Scraping %s %s for problems, this may take a few seconds...'):format( + platform_display_name, + contest_id + ), + vim.log.levels.INFO, + true + ) + + local metadata_result = scrape.scrape_contest_metadata(platform, contest_id) + if not metadata_result.success then + logger.log( + ('Failed to get problems for %s %s: %s'):format( + platform, + contest_id, + metadata_result.error or 'unknown error' + ), + vim.log.levels.ERROR + ) + return problems + end + + for _, problem in ipairs(metadata_result.problems or {}) do + table.insert(problems, { + id = problem.id, + name = problem.name, + display_name = problem.name, + }) + end + + return problems +end + +---Set up a specific problem by calling the main CP handler +---@param platform string Platform identifier +---@param contest_id string Contest identifier +---@param problem_id string Problem identifier +local function setup_problem(platform, contest_id, problem_id) + local cp = require('cp') + cp.handle_command({ fargs = { platform, contest_id, problem_id } }) +end + +M.get_platforms = get_platforms +M.get_contests_for_platform = get_contests_for_platform +M.get_problems_for_contest = get_problems_for_contest +M.setup_problem = setup_problem + +return M diff --git a/lua/cp/pickers/telescope.lua b/lua/cp/pickers/telescope.lua new file mode 100644 index 0000000..6f65c93 --- /dev/null +++ b/lua/cp/pickers/telescope.lua @@ -0,0 +1,138 @@ +local finders = require('telescope.finders') +local pickers = require('telescope.pickers') +local conf = require('telescope.config').values +local action_state = require('telescope.actions.state') +local actions = require('telescope.actions') + +local picker_utils = require('cp.pickers') + +local function problem_picker(opts, platform, contest_id) + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + local problems = picker_utils.get_problems_for_contest(platform, contest_id) + + if #problems == 0 then + vim.notify( + ('No problems found for contest: %s %s'):format(platform_display_name, contest_id), + vim.log.levels.WARN + ) + return + end + + pickers + .new(opts, { + prompt_title = ('Select Problem (%s %s)'):format(platform_display_name, contest_id), + finder = finders.new_table({ + results = problems, + entry_maker = function(entry) + return { + value = entry, + display = entry.display_name, + ordinal = entry.display_name, + } + end, + }), + sorter = conf.generic_sorter(opts), + attach_mappings = function(prompt_bufnr) + actions.select_default:replace(function() + local selection = action_state.get_selected_entry() + actions.close(prompt_bufnr) + + if selection then + picker_utils.setup_problem(platform, contest_id, selection.value.id) + end + end) + return true + end, + }) + :find() +end + +local function contest_picker(opts, platform) + local constants = require('cp.constants') + local platform_display_name = constants.PLATFORM_DISPLAY_NAMES[platform] or platform + local contests = picker_utils.get_contests_for_platform(platform) + + if #contests == 0 then + vim.notify( + ('No contests found for platform: %s'):format(platform_display_name), + vim.log.levels.WARN + ) + return + end + + pickers + .new(opts, { + prompt_title = ('Select Contest (%s)'):format(platform_display_name), + results_title = ' refresh', + finder = finders.new_table({ + results = contests, + entry_maker = function(entry) + return { + value = entry, + display = entry.display_name, + ordinal = entry.display_name, + } + end, + }), + sorter = conf.generic_sorter(opts), + attach_mappings = function(prompt_bufnr, map) + actions.select_default:replace(function() + local selection = action_state.get_selected_entry() + actions.close(prompt_bufnr) + + if selection then + problem_picker(opts, platform, selection.value.id) + end + end) + + map('i', '', function() + local cache = require('cp.cache') + cache.clear_contest_list(platform) + actions.close(prompt_bufnr) + contest_picker(opts, platform) + end) + + return true + end, + }) + :find() +end + +local function platform_picker(opts) + opts = opts or {} + + local platforms = picker_utils.get_platforms() + + pickers + .new(opts, { + prompt_title = 'Select Platform', + finder = finders.new_table({ + results = platforms, + entry_maker = function(entry) + return { + value = entry, + display = entry.display_name, + ordinal = entry.display_name, + } + end, + }), + sorter = conf.generic_sorter(opts), + attach_mappings = function(prompt_bufnr) + actions.select_default:replace(function() + local selection = action_state.get_selected_entry() + actions.close(prompt_bufnr) + + if selection then + contest_picker(opts, selection.value.id) + end + end) + return true + end, + }) + :find() +end + +return { + platform_picker = platform_picker, +} diff --git a/lua/cp/scrape.lua b/lua/cp/scrape.lua index f8a5e31..5ecaf43 100644 --- a/lua/cp/scrape.lua +++ b/lua/cp/scrape.lua @@ -13,44 +13,14 @@ local M = {} local cache = require('cp.cache') -local logger = require('cp.log') local problem = require('cp.problem') - -local function get_plugin_path() - local plugin_path = debug.getinfo(1, 'S').source:sub(2) - return vim.fn.fnamemodify(plugin_path, ':h:h:h') -end +local utils = require('cp.utils') local function check_internet_connectivity() local result = vim.system({ 'ping', '-c', '1', '-W', '3', '8.8.8.8' }, { text = true }):wait() return result.code == 0 end -local function setup_python_env() - local plugin_path = get_plugin_path() - local venv_dir = plugin_path .. '/.venv' - - if vim.fn.executable('uv') == 0 then - logger.log( - 'uv is not installed. Install it to enable problem scraping: https://docs.astral.sh/uv/', - vim.log.levels.WARN - ) - return false - end - - if vim.fn.isdirectory(venv_dir) == 0 then - logger.log('setting up Python environment for scrapers...') - local result = vim.system({ 'uv', 'sync' }, { cwd = plugin_path, text = true }):wait() - if result.code ~= 0 then - logger.log('failed to setup Python environment: ' .. result.stderr, vim.log.levels.ERROR) - return false - end - logger.log('python environment setup complete') - end - - return true -end - ---@param platform string ---@param contest_id string ---@return {success: boolean, problems?: table[], error?: string} @@ -77,14 +47,14 @@ function M.scrape_contest_metadata(platform, contest_id) } end - if not setup_python_env() then + if not utils.setup_python_env() then return { success = false, error = 'Python environment setup failed', } end - local plugin_path = get_plugin_path() + local plugin_path = utils.get_plugin_path() local args = { 'uv', @@ -182,7 +152,7 @@ function M.scrape_problem(ctx) } end - if not setup_python_env() then + if not utils.setup_python_env() then return { success = false, problem_id = ctx.problem_name, @@ -190,33 +160,19 @@ function M.scrape_problem(ctx) } end - local plugin_path = get_plugin_path() + local plugin_path = utils.get_plugin_path() - local args - if ctx.contest == 'cses' then - args = { - 'uv', - 'run', - '--directory', - plugin_path, - '-m', - 'scrapers.' .. ctx.contest, - 'tests', - ctx.problem_id, - } - else - args = { - 'uv', - 'run', - '--directory', - plugin_path, - '-m', - 'scrapers.' .. ctx.contest, - 'tests', - ctx.contest_id, - ctx.problem_id, - } - end + local args = { + 'uv', + 'run', + '--directory', + plugin_path, + '-m', + 'scrapers.' .. ctx.contest, + 'tests', + ctx.contest_id, + ctx.problem_id, + } local result = vim .system(args, { @@ -308,11 +264,11 @@ function M.scrape_problems_parallel(platform, contest_id, problems, config) return {} end - if not setup_python_env() then + if not utils.setup_python_env() then return {} end - local plugin_path = get_plugin_path() + local plugin_path = utils.get_plugin_path() local jobs = {} for _, prob in ipairs(problems) do diff --git a/lua/cp/utils.lua b/lua/cp/utils.lua new file mode 100644 index 0000000..ff97a11 --- /dev/null +++ b/lua/cp/utils.lua @@ -0,0 +1,44 @@ +local M = {} + +local logger = require('cp.log') + +---@return string +function M.get_plugin_path() + local plugin_path = debug.getinfo(1, 'S').source:sub(2) + return vim.fn.fnamemodify(plugin_path, ':h:h:h') +end + +local python_env_setup = false + +---@return boolean success +function M.setup_python_env() + if python_env_setup then + return true + end + + local plugin_path = M.get_plugin_path() + local venv_dir = plugin_path .. '/.venv' + + if vim.fn.executable('uv') == 0 then + logger.log( + 'uv is not installed. Install it to enable problem scraping: https://docs.astral.sh/uv/', + vim.log.levels.WARN + ) + return false + end + + if vim.fn.isdirectory(venv_dir) == 0 then + logger.log('setting up Python environment for scrapers...') + local result = vim.system({ 'uv', 'sync' }, { cwd = plugin_path, text = true }):wait() + if result.code ~= 0 then + logger.log('failed to setup Python environment: ' .. result.stderr, vim.log.levels.ERROR) + return false + end + logger.log('Python environment setup complete') + end + + python_env_setup = true + return true +end + +return M diff --git a/lua/cp/version.lua b/lua/cp/version.lua index b454a07..308fe01 100644 --- a/lua/cp/version.lua +++ b/lua/cp/version.lua @@ -1,8 +1,9 @@ local M = {} +local utils = require('cp.utils') + local function get_git_version() - local plugin_path = debug.getinfo(1, 'S').source:sub(2) - local plugin_root = vim.fn.fnamemodify(plugin_path, ':h:h:h') + local plugin_root = utils.get_plugin_path() local result = vim .system({ 'git', 'describe', '--tags', '--always', '--dirty' }, { diff --git a/pyproject.toml b/pyproject.toml index 5c731b5..92c1cbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.11" dependencies = [ + "backoff>=2.2.1", "beautifulsoup4>=4.13.5", "cloudscraper>=1.2.71", "requests>=2.32.5", diff --git a/scrapers/atcoder.py b/scrapers/atcoder.py index 3dc1d16..83e1cc1 100644 --- a/scrapers/atcoder.py +++ b/scrapers/atcoder.py @@ -3,9 +3,9 @@ import json import re import sys -import time from dataclasses import asdict +import backoff import requests from bs4 import BeautifulSoup, Tag @@ -99,7 +99,6 @@ def scrape_contest_problems(contest_id: str) -> list[ProblemSummary]: if problem: problems.append(problem) - problems.sort(key=lambda x: x.id) return problems except Exception as e: @@ -169,7 +168,6 @@ def scrape(url: str) -> list[TestCase]: def scrape_contests() -> list[ContestSummary]: import concurrent.futures - import random def get_max_pages() -> int: try: @@ -197,168 +195,161 @@ def scrape_contests() -> list[ContestSummary]: except Exception: return 15 - def scrape_page_with_retry(page: int, max_retries: int = 3) -> list[ContestSummary]: - for attempt in range(max_retries): + def scrape_page(page: int) -> list[ContestSummary]: + @backoff.on_exception( + backoff.expo, + (requests.exceptions.RequestException, requests.exceptions.HTTPError), + max_tries=4, + jitter=backoff.random_jitter, + on_backoff=lambda details: print( + f"Request failed on page {page} (attempt {details['tries']}), retrying in {details['wait']:.1f}s: {details['exception']}", + file=sys.stderr, + ), + ) + @backoff.on_predicate( + backoff.expo, + lambda response: response.status_code == 429, + max_tries=4, + jitter=backoff.random_jitter, + on_backoff=lambda details: print( + f"Rate limited on page {page}, retrying in {details['wait']:.1f}s", + file=sys.stderr, + ), + ) + def make_request() -> requests.Response: + headers = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + } + url = f"https://atcoder.jp/contests/archive?page={page}" + response = requests.get(url, headers=headers, timeout=10) + response.raise_for_status() + return response + + try: + response = make_request() + except Exception: + return [] + + soup = BeautifulSoup(response.text, "html.parser") + table = soup.find("table", class_="table") + if not table: + return [] + + tbody = table.find("tbody") + if not tbody or not isinstance(tbody, Tag): + return [] + + rows = tbody.find_all("tr") + if not rows: + return [] + + contests = [] + for row in rows: + cells = row.find_all("td") + if len(cells) < 2: + continue + + contest_cell = cells[1] + link = contest_cell.find("a") + if not link or not link.get("href"): + continue + + href = link.get("href") + contest_id = href.split("/")[-1] + name = link.get_text().strip() + try: - headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - } - url = f"https://atcoder.jp/contests/archive?page={page}" - response = requests.get(url, headers=headers, timeout=10) + name = name.encode().decode("unicode_escape") + except (UnicodeDecodeError, UnicodeEncodeError): + pass - if response.status_code == 429: - backoff_time = (2**attempt) + random.uniform(0, 1) - print( - f"Rate limited on page {page}, retrying in {backoff_time:.1f}s", - file=sys.stderr, - ) - time.sleep(backoff_time) - continue + name = ( + name.replace("\uff08", "(") + .replace("\uff09", ")") + .replace("\u3000", " ") + ) + name = re.sub( + r"[\uff01-\uff5e]", lambda m: chr(ord(m.group()) - 0xFEE0), name + ) - response.raise_for_status() + def generate_display_name_from_id(contest_id: str) -> str: + parts = contest_id.replace("-", " ").replace("_", " ") - soup = BeautifulSoup(response.text, "html.parser") - table = soup.find("table", class_="table") - if not table: - return [] - - tbody = table.find("tbody") - if not tbody or not isinstance(tbody, Tag): - return [] - - rows = tbody.find_all("tr") - if not rows: - return [] - - contests = [] - for row in rows: - cells = row.find_all("td") - if len(cells) < 2: - continue - - contest_cell = cells[1] - link = contest_cell.find("a") - if not link or not link.get("href"): - continue - - href = link.get("href") - contest_id = href.split("/")[-1] - name = link.get_text().strip() - - try: - name = name.encode().decode("unicode_escape") - except (UnicodeDecodeError, UnicodeEncodeError): - pass - - name = ( - name.replace("\uff08", "(") - .replace("\uff09", ")") - .replace("\u3000", " ") - ) - name = re.sub( - r"[\uff01-\uff5e]", lambda m: chr(ord(m.group()) - 0xFEE0), name - ) - - def generate_display_name_from_id(contest_id: str) -> str: - parts = contest_id.replace("-", " ").replace("_", " ") - - parts = re.sub( - r"\b(jsc|JSC)\b", - "Japanese Student Championship", - parts, - flags=re.IGNORECASE, - ) - parts = re.sub( - r"\b(wtf|WTF)\b", - "World Tour Finals", - parts, - flags=re.IGNORECASE, - ) - parts = re.sub( - r"\b(ahc)(\d+)\b", - r"Heuristic Contest \2 (AHC)", - parts, - flags=re.IGNORECASE, - ) - parts = re.sub( - r"\b(arc)(\d+)\b", - r"Regular Contest \2 (ARC)", - parts, - flags=re.IGNORECASE, - ) - parts = re.sub( - r"\b(abc)(\d+)\b", - r"Beginner Contest \2 (ABC)", - parts, - flags=re.IGNORECASE, - ) - parts = re.sub( - r"\b(agc)(\d+)\b", - r"Grand Contest \2 (AGC)", - parts, - flags=re.IGNORECASE, - ) - - return parts.title() - - english_chars = sum(1 for c in name if c.isascii() and c.isalpha()) - total_chars = len(re.sub(r"\s+", "", name)) - - if total_chars > 0 and english_chars / total_chars < 0.3: - display_name = generate_display_name_from_id(contest_id) - else: - display_name = name - if "AtCoder Beginner Contest" in name: - match = re.search(r"AtCoder Beginner Contest (\d+)", name) - if match: - display_name = ( - f"Beginner Contest {match.group(1)} (ABC)" - ) - elif "AtCoder Regular Contest" in name: - match = re.search(r"AtCoder Regular Contest (\d+)", name) - if match: - display_name = f"Regular Contest {match.group(1)} (ARC)" - elif "AtCoder Grand Contest" in name: - match = re.search(r"AtCoder Grand Contest (\d+)", name) - if match: - display_name = f"Grand Contest {match.group(1)} (AGC)" - elif "AtCoder Heuristic Contest" in name: - match = re.search(r"AtCoder Heuristic Contest (\d+)", name) - if match: - display_name = ( - f"Heuristic Contest {match.group(1)} (AHC)" - ) - - contests.append( - ContestSummary( - id=contest_id, name=name, display_name=display_name - ) - ) - - return contests - - except requests.exceptions.RequestException as e: - if response.status_code == 429: - continue - print( - f"Failed to scrape page {page} (attempt {attempt + 1}): {e}", - file=sys.stderr, + parts = re.sub( + r"\b(jsc|JSC)\b", + "Japanese Student Championship", + parts, + flags=re.IGNORECASE, + ) + parts = re.sub( + r"\b(wtf|WTF)\b", + "World Tour Finals", + parts, + flags=re.IGNORECASE, + ) + parts = re.sub( + r"\b(ahc)(\d+)\b", + r"Heuristic Contest \2 (AHC)", + parts, + flags=re.IGNORECASE, + ) + parts = re.sub( + r"\b(arc)(\d+)\b", + r"Regular Contest \2 (ARC)", + parts, + flags=re.IGNORECASE, + ) + parts = re.sub( + r"\b(abc)(\d+)\b", + r"Beginner Contest \2 (ABC)", + parts, + flags=re.IGNORECASE, + ) + parts = re.sub( + r"\b(agc)(\d+)\b", + r"Grand Contest \2 (AGC)", + parts, + flags=re.IGNORECASE, ) - if attempt == max_retries - 1: - return [] - except Exception as e: - print(f"Unexpected error on page {page}: {e}", file=sys.stderr) - return [] - return [] + return parts.title() + + english_chars = sum(1 for c in name if c.isascii() and c.isalpha()) + total_chars = len(re.sub(r"\s+", "", name)) + + if total_chars > 0 and english_chars / total_chars < 0.3: + display_name = generate_display_name_from_id(contest_id) + else: + display_name = name + if "AtCoder Beginner Contest" in name: + match = re.search(r"AtCoder Beginner Contest (\d+)", name) + if match: + display_name = f"Beginner Contest {match.group(1)} (ABC)" + elif "AtCoder Regular Contest" in name: + match = re.search(r"AtCoder Regular Contest (\d+)", name) + if match: + display_name = f"Regular Contest {match.group(1)} (ARC)" + elif "AtCoder Grand Contest" in name: + match = re.search(r"AtCoder Grand Contest (\d+)", name) + if match: + display_name = f"Grand Contest {match.group(1)} (AGC)" + elif "AtCoder Heuristic Contest" in name: + match = re.search(r"AtCoder Heuristic Contest (\d+)", name) + if match: + display_name = f"Heuristic Contest {match.group(1)} (AHC)" + + contests.append( + ContestSummary(id=contest_id, name=name, display_name=display_name) + ) + + return contests max_pages = get_max_pages() page_results = {} with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_page = { - executor.submit(scrape_page_with_retry, page): page - for page in range(1, max_pages + 1) + executor.submit(scrape_page, page): page for page in range(1, max_pages + 1) } for future in concurrent.futures.as_completed(future_to_page): diff --git a/scrapers/codeforces.py b/scrapers/codeforces.py index 0aa7d07..1402827 100644 --- a/scrapers/codeforces.py +++ b/scrapers/codeforces.py @@ -203,8 +203,6 @@ def scrape_contest_problems(contest_id: str) -> list[ProblemSummary]: ProblemSummary(id=problem_letter, name=problem_name) ) - problems.sort(key=lambda x: x.id) - seen: set[str] = set() unique_problems: list[ProblemSummary] = [] for p in problems: diff --git a/scrapers/cses.py b/scrapers/cses.py index b2f1733..3c5db7a 100755 --- a/scrapers/cses.py +++ b/scrapers/cses.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 import json -import random import re import sys -import time from dataclasses import asdict +import backoff import requests from bs4 import BeautifulSoup, Tag @@ -41,36 +40,29 @@ def denormalize_category_name(category_id: str) -> str: return category_map.get(category_id, category_id.replace("_", " ").title()) -def request_with_retry( - url: str, headers: dict, max_retries: int = 3 -) -> requests.Response: - for attempt in range(max_retries): - try: - delay = 0.5 + random.uniform(0, 0.3) - time.sleep(delay) - - response = requests.get(url, headers=headers, timeout=10) - - if response.status_code == 429: - backoff = (2**attempt) + random.uniform(0, 1) - print(f"Rate limited, retrying in {backoff:.1f}s", file=sys.stderr) - time.sleep(backoff) - continue - - response.raise_for_status() - return response - - except requests.exceptions.RequestException as e: - if attempt == max_retries - 1: - raise - backoff = 2**attempt - print( - f"Request failed (attempt {attempt + 1}), retrying in {backoff}s: {e}", - file=sys.stderr, - ) - time.sleep(backoff) - - raise Exception("All retry attempts failed") +@backoff.on_exception( + backoff.expo, + (requests.exceptions.RequestException, requests.exceptions.HTTPError), + max_tries=4, + jitter=backoff.random_jitter, + on_backoff=lambda details: print( + f"Request failed (attempt {details['tries']}), retrying in {details['wait']:.1f}s: {details['exception']}", + file=sys.stderr, + ), +) +@backoff.on_predicate( + backoff.expo, + lambda response: response.status_code == 429, + max_tries=4, + jitter=backoff.random_jitter, + on_backoff=lambda details: print( + f"Rate limited, retrying in {details['wait']:.1f}s", file=sys.stderr + ), +) +def make_request(url: str, headers: dict) -> requests.Response: + response = requests.get(url, headers=headers, timeout=10) + response.raise_for_status() + return response def scrape_category_problems(category_id: str) -> list[ProblemSummary]: @@ -82,7 +74,7 @@ def scrape_category_problems(category_id: str) -> list[ProblemSummary]: "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } - response = request_with_retry(problemset_url, headers) + response = make_request(problemset_url, headers) soup = BeautifulSoup(response.text, "html.parser") @@ -122,7 +114,6 @@ def scrape_category_problems(category_id: str) -> list[ProblemSummary]: problems.append(ProblemSummary(id=problem_id, name=problem_name)) - problems.sort(key=lambda x: int(x.id)) return problems except Exception as e: @@ -176,7 +167,7 @@ def scrape_categories() -> list[ContestSummary]: headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } - response = request_with_retry("https://cses.fi/problemset/", headers) + response = make_request("https://cses.fi/problemset/", headers) soup = BeautifulSoup(response.text, "html.parser") categories = [] @@ -188,12 +179,7 @@ def scrape_categories() -> list[ContestSummary]: category_id = normalize_category_name(category_name) - ul = h2.find_next_sibling("ul", class_="task-list") - problem_count = 0 - if ul: - problem_count = len(ul.find_all("li", class_="task")) - - display_name = f"{category_name} ({problem_count} problems)" + display_name = category_name categories.append( ContestSummary( @@ -276,9 +262,6 @@ def scrape_all_problems() -> dict[str, list[ProblemSummary]]: problem = ProblemSummary(id=problem_id, name=problem_name) all_categories[current_category].append(problem) - for category in all_categories: - all_categories[category].sort(key=lambda x: int(x.id)) - print( f"Found {len(all_categories)} categories with {sum(len(probs) for probs in all_categories.values())} problems", file=sys.stderr, @@ -323,7 +306,7 @@ def scrape(url: str) -> list[TestCase]: "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" } - response = request_with_retry(url, headers) + response = make_request(url, headers) soup = BeautifulSoup(response.text, "html.parser") diff --git a/spec/config_spec.lua b/spec/config_spec.lua index 8c4c79c..f3f3738 100644 --- a/spec/config_spec.lua +++ b/spec/config_spec.lua @@ -231,6 +231,39 @@ describe('cp.config', function() end) end) end) + + describe('picker validation', function() + it('validates picker is valid value', function() + local invalid_config = { + picker = 'invalid_picker', + } + + assert.has_error(function() + config.setup(invalid_config) + end, "Invalid picker 'invalid_picker'. Must be 'telescope' or 'fzf-lua'") + end) + + it('allows nil picker', function() + assert.has_no.errors(function() + local result = config.setup({ picker = nil }) + assert.is_nil(result.picker) + end) + end) + + it('allows telescope picker without checking availability', function() + assert.has_no.errors(function() + local result = config.setup({ picker = 'telescope' }) + assert.equals('telescope', result.picker) + end) + end) + + it('allows fzf-lua picker without checking availability', function() + assert.has_no.errors(function() + local result = config.setup({ picker = 'fzf-lua' }) + assert.equals('fzf-lua', result.picker) + end) + end) + end) end) describe('default_filename', function() diff --git a/spec/fzf_lua_spec.lua b/spec/fzf_lua_spec.lua new file mode 100644 index 0000000..eb6f3e8 --- /dev/null +++ b/spec/fzf_lua_spec.lua @@ -0,0 +1,31 @@ +describe('cp.fzf_lua', function() + local spec_helper = require('spec.spec_helper') + + before_each(function() + spec_helper.setup() + + package.preload['fzf-lua'] = function() + return { + fzf_exec = function(_, _) end, + } + end + end) + + after_each(function() + spec_helper.teardown() + end) + + describe('module loading', function() + it('loads fzf-lua integration without error', function() + assert.has_no.errors(function() + require('cp.pickers.fzf_lua') + end) + end) + + it('returns module with platform_picker function', function() + local fzf_lua_cp = require('cp.pickers.fzf_lua') + assert.is_table(fzf_lua_cp) + assert.is_function(fzf_lua_cp.platform_picker) + end) + end) +end) diff --git a/spec/picker_spec.lua b/spec/picker_spec.lua new file mode 100644 index 0000000..ab4d36e --- /dev/null +++ b/spec/picker_spec.lua @@ -0,0 +1,204 @@ +describe('cp.picker', function() + local picker + local spec_helper = require('spec.spec_helper') + + before_each(function() + spec_helper.setup() + picker = require('cp.pickers') + end) + + after_each(function() + spec_helper.teardown() + end) + + describe('get_platforms', function() + it('returns platform list with display names', function() + local platforms = picker.get_platforms() + + assert.is_table(platforms) + assert.is_true(#platforms > 0) + + for _, platform in ipairs(platforms) do + assert.is_string(platform.id) + assert.is_string(platform.display_name) + assert.is_not_nil(platform.display_name:match('^%u')) + end + end) + + it('includes expected platforms with correct display names', function() + local platforms = picker.get_platforms() + local platform_map = {} + for _, p in ipairs(platforms) do + platform_map[p.id] = p.display_name + end + + assert.equals('CodeForces', platform_map['codeforces']) + assert.equals('AtCoder', platform_map['atcoder']) + assert.equals('CSES', platform_map['cses']) + end) + end) + + describe('get_contests_for_platform', function() + it('returns empty list when scraper fails', function() + vim.system = function(_, _) + return { + wait = function() + return { code = 1, stderr = 'test error' } + end, + } + end + + local contests = picker.get_contests_for_platform('test_platform') + assert.is_table(contests) + assert.equals(0, #contests) + end) + + it('returns empty list when JSON is invalid', function() + vim.system = function(_, _) + return { + wait = function() + return { code = 0, stdout = 'invalid json' } + end, + } + end + + local contests = picker.get_contests_for_platform('test_platform') + assert.is_table(contests) + assert.equals(0, #contests) + end) + + it('returns contest list when scraper succeeds', function() + local cache = require('cp.cache') + local utils = require('cp.utils') + + cache.load = function() end + cache.get_contest_list = function() + return nil + end + cache.set_contest_list = function() end + + utils.setup_python_env = function() + return true + end + utils.get_plugin_path = function() + return '/test/path' + end + + vim.system = function(_, _) + return { + wait = function() + return { + code = 0, + stdout = vim.json.encode({ + success = true, + contests = { + { + id = 'abc123', + name = 'AtCoder Beginner Contest 123', + display_name = 'Beginner Contest 123 (ABC)', + }, + { + id = '1951', + name = 'Educational Round 168', + display_name = 'Educational Round 168', + }, + }, + }), + } + end, + } + end + + local contests = picker.get_contests_for_platform('test_platform') + assert.is_table(contests) + assert.equals(2, #contests) + assert.equals('abc123', contests[1].id) + assert.equals('AtCoder Beginner Contest 123', contests[1].name) + assert.equals('Beginner Contest 123 (ABC)', contests[1].display_name) + end) + end) + + describe('get_problems_for_contest', function() + it('returns problems from cache when available', function() + local cache = require('cp.cache') + cache.load = function() end + cache.get_contest_data = function(_, _) + return { + problems = { + { id = 'a', name = 'Problem A' }, + { id = 'b', name = 'Problem B' }, + }, + } + end + + local problems = picker.get_problems_for_contest('test_platform', 'test_contest') + assert.is_table(problems) + assert.equals(2, #problems) + assert.equals('a', problems[1].id) + assert.equals('Problem A', problems[1].name) + assert.equals('Problem A', problems[1].display_name) + end) + + it('falls back to scraping when cache miss', function() + local cache = require('cp.cache') + local scrape = require('cp.scrape') + + cache.load = function() end + cache.get_contest_data = function(_, _) + return nil + end + scrape.scrape_contest_metadata = function(_, _) + return { + success = true, + problems = { + { id = 'x', name = 'Problem X' }, + }, + } + end + + local problems = picker.get_problems_for_contest('test_platform', 'test_contest') + assert.is_table(problems) + assert.equals(1, #problems) + assert.equals('x', problems[1].id) + end) + + it('returns empty list when scraping fails', function() + local cache = require('cp.cache') + local scrape = require('cp.scrape') + + cache.load = function() end + cache.get_contest_data = function(_, _) + return nil + end + scrape.scrape_contest_metadata = function(_, _) + return { + success = false, + error = 'test error', + } + end + + local problems = picker.get_problems_for_contest('test_platform', 'test_contest') + assert.is_table(problems) + assert.equals(0, #problems) + end) + end) + + describe('setup_problem', function() + it('calls cp.handle_command with correct arguments', function() + local cp = require('cp') + local called_with = nil + + cp.handle_command = function(opts) + called_with = opts + end + + picker.setup_problem('codeforces', '1951', 'a') + + assert.is_table(called_with) + assert.is_table(called_with.fargs) + assert.equals('codeforces', called_with.fargs[1]) + assert.equals('1951', called_with.fargs[2]) + assert.equals('a', called_with.fargs[3]) + end) + end) +end) diff --git a/spec/scraper_spec.lua b/spec/scraper_spec.lua index 67c1daa..cc02b6b 100644 --- a/spec/scraper_spec.lua +++ b/spec/scraper_spec.lua @@ -1,6 +1,7 @@ describe('cp.scrape', function() local scrape local mock_cache + local mock_utils local mock_system_calls local temp_files local spec_helper = require('spec.spec_helper') @@ -8,6 +9,7 @@ describe('cp.scrape', function() before_each(function() spec_helper.setup() temp_files = {} + mock_system_calls = {} mock_cache = { load = function() end, @@ -18,7 +20,14 @@ describe('cp.scrape', function() set_test_cases = function() end, } - mock_system_calls = {} + mock_utils = { + setup_python_env = function() + return true + end, + get_plugin_path = function() + return '/test/plugin/path' + end, + } vim.system = function(cmd, opts) table.insert(mock_system_calls, { cmd = cmd, opts = opts }) @@ -46,6 +55,8 @@ describe('cp.scrape', function() end package.loaded['cp.cache'] = mock_cache + package.loaded['cp.utils'] = mock_utils + package.loaded['cp.scrape'] = nil scrape = require('cp.scrape') local original_fn = vim.fn @@ -129,17 +140,22 @@ describe('cp.scrape', function() describe('system dependency checks', function() it('handles missing uv executable', function() + local cache = require('cp.cache') + local utils = require('cp.utils') + + cache.load = function() end + cache.get_contest_data = function() + return nil + end + vim.fn.executable = function(cmd) return cmd == 'uv' and 0 or 1 end - local result = scrape.scrape_contest_metadata('atcoder', 'abc123') + utils.setup_python_env = function() + return vim.fn.executable('uv') == 1 + end - assert.is_false(result.success) - assert.is_not_nil(result.error:match('Python environment setup failed')) - end) - - it('handles python environment setup failure', function() vim.system = function(cmd) if cmd[1] == 'ping' then return { @@ -147,12 +163,6 @@ describe('cp.scrape', function() return { code = 0 } end, } - elseif cmd[1] == 'uv' and cmd[2] == 'sync' then - return { - wait = function() - return { code = 1, stderr = 'setup failed' } - end, - } end return { wait = function() @@ -161,14 +171,43 @@ describe('cp.scrape', function() } end - vim.fn.isdirectory = function() - return 0 + local result = scrape.scrape_contest_metadata('atcoder', 'abc123') + + assert.is_false(result.success) + assert.is_not_nil(result.error) + end) + + it('handles python environment setup failure', function() + local cache = require('cp.cache') + + cache.load = function() end + cache.get_contest_data = function() + return nil + end + + mock_utils.setup_python_env = function() + return false + end + + vim.system = function(cmd) + if cmd[1] == 'ping' then + return { + wait = function() + return { code = 0 } + end, + } + end + return { + wait = function() + return { code = 0 } + end, + } end local result = scrape.scrape_contest_metadata('atcoder', 'abc123') assert.is_false(result.success) - assert.is_not_nil(result.error:match('Python environment setup failed')) + assert.equals('Python environment setup failed', result.error) end) it('handles network connectivity issues', function() @@ -396,7 +435,7 @@ describe('cp.scrape', function() assert.is_not_nil(tests_call) assert.is_true(vim.tbl_contains(tests_call.cmd, 'tests')) assert.is_true(vim.tbl_contains(tests_call.cmd, '1001')) - assert.is_false(vim.tbl_contains(tests_call.cmd, 'sorting_and_searching')) + assert.is_true(vim.tbl_contains(tests_call.cmd, 'sorting_and_searching')) end) end) diff --git a/spec/telescope_spec.lua b/spec/telescope_spec.lua new file mode 100644 index 0000000..794fecc --- /dev/null +++ b/spec/telescope_spec.lua @@ -0,0 +1,78 @@ +describe('cp.telescope', function() + local spec_helper = require('spec.spec_helper') + + before_each(function() + spec_helper.setup() + + package.preload['telescope'] = function() + return { + register_extension = function(ext_config) + return ext_config + end, + } + end + + package.preload['telescope.pickers'] = function() + return { + new = function(_, _) + return { + find = function() end, + } + end, + } + end + + package.preload['telescope.finders'] = function() + return { + new_table = function(opts) + return opts + end, + } + end + + package.preload['telescope.config'] = function() + return { + values = { + generic_sorter = function() + return {} + end, + }, + } + end + + package.preload['telescope.actions'] = function() + return { + select_default = { + replace = function() end, + }, + close = function() end, + } + end + + package.preload['telescope.actions.state'] = function() + return { + get_selected_entry = function() + return nil + end, + } + end + end) + + after_each(function() + spec_helper.teardown() + end) + + describe('module loading', function() + it('registers telescope extension without error', function() + assert.has_no.errors(function() + require('cp.pickers.telescope') + end) + end) + + it('returns module with platform_picker function', function() + local telescope_cp = require('cp.pickers.telescope') + assert.is_table(telescope_cp) + assert.is_function(telescope_cp.platform_picker) + end) + end) +end) diff --git a/tests/scrapers/test_atcoder.py b/tests/scrapers/test_atcoder.py index 5ff91d9..a2a88e5 100644 --- a/tests/scrapers/test_atcoder.py +++ b/tests/scrapers/test_atcoder.py @@ -94,7 +94,6 @@ def test_scrape_contests_success(mocker): return mock_response mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect) - mocker.patch("scrapers.atcoder.time.sleep") result = scrape_contests() @@ -116,7 +115,6 @@ def test_scrape_contests_no_table(mocker): mock_response.text = "No table found" mocker.patch("scrapers.atcoder.requests.get", return_value=mock_response) - mocker.patch("scrapers.atcoder.time.sleep") result = scrape_contests() @@ -127,7 +125,6 @@ def test_scrape_contests_network_error(mocker): mocker.patch( "scrapers.atcoder.requests.get", side_effect=Exception("Network error") ) - mocker.patch("scrapers.atcoder.time.sleep") result = scrape_contests() diff --git a/tests/scrapers/test_cses.py b/tests/scrapers/test_cses.py index a1e84a2..545176d 100644 --- a/tests/scrapers/test_cses.py +++ b/tests/scrapers/test_cses.py @@ -168,12 +168,12 @@ def test_scrape_categories_success(mocker): assert result[0] == ContestSummary( id="introductory_problems", name="Introductory Problems", - display_name="Introductory Problems (2 problems)", + display_name="Introductory Problems", ) assert result[1] == ContestSummary( id="sorting_and_searching", name="Sorting and Searching", - display_name="Sorting and Searching (3 problems)", + display_name="Sorting and Searching", ) diff --git a/uv.lock b/uv.lock index 744b4ae..aa9248d 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.11" +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.13.5" @@ -375,6 +384,7 @@ name = "scrapers" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "backoff" }, { name = "beautifulsoup4" }, { name = "cloudscraper" }, { name = "requests" }, @@ -392,6 +402,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "backoff", specifier = ">=2.2.1" }, { name = "beautifulsoup4", specifier = ">=4.13.5" }, { name = "cloudscraper", specifier = ">=1.2.71" }, { name = "requests", specifier = ">=2.32.5" },