feat: async scraper

spec/async_init_spec.lua

@@ -0,0 +1,50 @@
+describe('cp.async.init', function()
+  local async
+  local spec_helper = require('spec.spec_helper')
+
+  before_each(function()
+    spec_helper.setup()
+    async = spec_helper.fresh_require('cp.async.init')
+  end)
+
+  after_each(function()
+    spec_helper.teardown()
+  end)
+
+  describe('contest operation guard', function()
+    it('allows starting operation when none active', function()
+      assert.has_no_errors(function()
+        async.start_contest_operation('test_operation')
+      end)
+      assert.equals('test_operation', async.get_active_operation())
+    end)
+
+    it('throws error when starting operation while one is active', function()
+      async.start_contest_operation('first_operation')
+
+      assert.has_error(function()
+        async.start_contest_operation('second_operation')
+      end, "Contest operation 'first_operation' already active, cannot start 'second_operation'")
+    end)
+
+    it('allows starting operation after finishing previous one', function()
+      async.start_contest_operation('first_operation')
+      async.finish_contest_operation()
+
+      assert.has_no_errors(function()
+        async.start_contest_operation('second_operation')
+      end)
+      assert.equals('second_operation', async.get_active_operation())
+    end)
+
+    it('correctly reports active operation status', function()
+      assert.is_nil(async.get_active_operation())
+
+      async.start_contest_operation('test_operation')
+      assert.equals('test_operation', async.get_active_operation())
+
+      async.finish_contest_operation()
+      assert.is_nil(async.get_active_operation())
+    end)
+  end)
+end)

spec/async_integration_spec.lua

@@ -0,0 +1,288 @@
+describe('async integration', function()
+  local cp
+  local spec_helper = require('spec.spec_helper')
+  local logged_messages = {}
+
+  before_each(function()
+    logged_messages = {}
+    local mock_logger = {
+      log = function(msg, level)
+        table.insert(logged_messages, { msg = msg, level = level })
+      end,
+      set_config = function() end,
+    }
+    package.loaded['cp.log'] = mock_logger
+
+    spec_helper.mock_async_scraper_success()
+
+    local mock_async = {
+      start_contest_operation = function() end,
+      finish_contest_operation = function() end,
+      get_active_operation = function()
+        return nil
+      end,
+    }
+
+    local mock_state = {
+      get_platform = function()
+        return 'atcoder'
+      end,
+      get_contest_id = function()
+        return 'abc123'
+      end,
+      set_platform = function() end,
+      set_contest_id = function() end,
+      set_problem_id = function() end,
+      set_test_cases = function() end,
+      set_run_panel_active = function() end,
+    }
+
+    local mock_config = {
+      setup = function()
+        return {}
+      end,
+      get_config = function()
+        return {
+          scrapers = { 'atcoder', 'codeforces' },
+          hooks = nil,
+        }
+      end,
+    }
+
+    local mock_cache = {
+      load = function() end,
+      get_contest_data = function()
+        return nil
+      end,
+      get_test_cases = function()
+        return nil
+      end,
+      set_file_state = function() end,
+    }
+
+    local mock_problem = {
+      create_context = function()
+        return {
+          source_file = '/test/source.cpp',
+          problem_name = 'abc123a',
+        }
+      end,
+    }
+
+    local mock_setup = {
+      set_platform = function()
+        return true
+      end,
+    }
+
+    vim.cmd = { e = function() end, only = function() end }
+    vim.api.nvim_get_current_buf = function()
+      return 1
+    end
+    vim.api.nvim_buf_get_lines = function()
+      return { '' }
+    end
+    vim.fn.expand = function()
+      return '/test/file.cpp'
+    end
+
+    package.loaded['cp.async'] = mock_async
+    package.loaded['cp.state'] = mock_state
+    package.loaded['cp.config'] = mock_config
+    package.loaded['cp.cache'] = mock_cache
+    package.loaded['cp.problem'] = mock_problem
+    package.loaded['cp.setup'] = mock_setup
+
+    cp = spec_helper.fresh_require('cp')
+    cp.setup({})
+  end)
+
+  after_each(function()
+    spec_helper.teardown()
+    logged_messages = {}
+  end)
+
+  describe('command routing', function()
+    it('contest_setup command uses async setup', function()
+      local opts = { fargs = { 'atcoder', 'abc123' } }
+
+      assert.has_no_errors(function()
+        cp.handle_command(opts)
+      end)
+    end)
+
+    it('full_setup command uses async setup', function()
+      local opts = { fargs = { 'atcoder', 'abc123', 'a' } }
+
+      assert.has_no_errors(function()
+        cp.handle_command(opts)
+      end)
+    end)
+
+    it('problem_switch uses async setup', function()
+      local mock_state = require('cp.state')
+      mock_state.get_contest_id = function()
+        return 'abc123'
+      end
+
+      local opts = { fargs = { 'a' } }
+
+      assert.has_no_errors(function()
+        cp.handle_command(opts)
+      end)
+    end)
+  end)
+
+  describe('end-to-end workflow', function()
+    it('handles complete contest setup workflow', function()
+      local setup_completed = false
+      local mock_async_setup = {
+        setup_contest_async = function(contest_id, language)
+          assert.equals('abc123', contest_id)
+          assert.is_nil(language)
+          setup_completed = true
+        end,
+      }
+      package.loaded['cp.async.setup'] = mock_async_setup
+
+      local opts = { fargs = { 'atcoder', 'abc123' } }
+      cp.handle_command(opts)
+
+      assert.is_true(setup_completed)
+    end)
+
+    it('handles problem switching within contest', function()
+      local mock_state = require('cp.state')
+      mock_state.get_contest_id = function()
+        return 'abc123'
+      end
+
+      local problem_setup_called = false
+      local mock_async_setup = {
+        setup_problem_async = function(contest_id, problem_id, language)
+          assert.equals('abc123', contest_id)
+          assert.equals('b', problem_id)
+          problem_setup_called = true
+        end,
+      }
+      package.loaded['cp.async.setup'] = mock_async_setup
+
+      local opts = { fargs = { 'b' } }
+      cp.handle_command(opts)
+
+      assert.is_true(problem_setup_called)
+    end)
+
+    it('handles language flags correctly', function()
+      local language_passed = nil
+      local mock_async_setup = {
+        setup_contest_async = function(contest_id, language)
+          language_passed = language
+        end,
+      }
+      package.loaded['cp.async.setup'] = mock_async_setup
+
+      local opts = { fargs = { 'atcoder', 'abc123', '--lang=python' } }
+      cp.handle_command(opts)
+
+      assert.equals('python', language_passed)
+    end)
+
+    it('handles scraping failures gracefully', function()
+      spec_helper.mock_async_scraper_failure()
+
+      local opts = { fargs = { 'atcoder', 'abc123' } }
+
+      assert.has_no_errors(function()
+        cp.handle_command(opts)
+      end)
+    end)
+  end)
+
+  describe('error handling', function()
+    it('handles invalid platform gracefully', function()
+      local opts = { fargs = { 'invalid_platform', 'abc123' } }
+
+      cp.handle_command(opts)
+
+      local error_logged = false
+      for _, log_entry in ipairs(logged_messages) do
+        if log_entry.level == vim.log.levels.ERROR then
+          error_logged = true
+          break
+        end
+      end
+      assert.is_true(error_logged)
+    end)
+
+    it('handles platform setup failure', function()
+      local mock_setup = require('cp.setup')
+      mock_setup.set_platform = function()
+        return false
+      end
+
+      local opts = { fargs = { 'atcoder', 'abc123' } }
+
+      assert.has_no_errors(function()
+        cp.handle_command(opts)
+      end)
+    end)
+
+    it('handles empty contest context for problem switch', function()
+      local mock_state = require('cp.state')
+      mock_state.get_contest_id = function()
+        return nil
+      end
+
+      local opts = { fargs = { 'a' } }
+
+      cp.handle_command(opts)
+
+      local error_logged = false
+      for _, log_entry in ipairs(logged_messages) do
+        if log_entry.level == vim.log.levels.ERROR then
+          error_logged = true
+          break
+        end
+      end
+      assert.is_true(error_logged)
+    end)
+  end)
+
+  describe('callback behavior', function()
+    it('maintains execution context in callbacks', function()
+      local callback_executed = false
+
+      local mock_scraper = {
+        scrape_contest_metadata_async = function(platform, contest_id, callback)
+          vim.schedule(function()
+            callback({ success = true, problems = { { id = 'a' } } })
+            callback_executed = true
+          end)
+        end,
+      }
+      package.loaded['cp.async.scraper'] = mock_scraper
+
+      local opts = { fargs = { 'atcoder', 'abc123' } }
+      cp.handle_command(opts)
+
+      assert.is_true(callback_executed)
+    end)
+
+    it('handles multiple rapid commands', function()
+      local command_count = 0
+      local mock_async_setup = {
+        setup_contest_async = function()
+          command_count = command_count + 1
+        end,
+      }
+      package.loaded['cp.async.setup'] = mock_async_setup
+
+      cp.handle_command({ fargs = { 'atcoder', 'abc123' } })
+      cp.handle_command({ fargs = { 'atcoder', 'abc124' } })
+      cp.handle_command({ fargs = { 'atcoder', 'abc125' } })
+
+      assert.equals(3, command_count)
+    end)
+  end)
+end)

spec/async_jobs_spec.lua

@@ -0,0 +1,111 @@
+describe('cp.async.jobs', function()
+  local jobs
+  local spec_helper = require('spec.spec_helper')
+  local mock_jobs = {}
+
+  before_each(function()
+    spec_helper.setup()
+    mock_jobs = {}
+
+    vim.system = function(args, opts, callback)
+      local job = {
+        kill = function() end,
+        args = args,
+        opts = opts,
+        callback = callback,
+      }
+      mock_jobs[#mock_jobs + 1] = job
+      return job
+    end
+
+    jobs = spec_helper.fresh_require('cp.async.jobs')
+  end)
+
+  after_each(function()
+    spec_helper.teardown()
+    mock_jobs = {}
+  end)
+
+  describe('job management', function()
+    it('starts job with unique ID', function()
+      local callback = function() end
+      local args = { 'test', 'command' }
+      local opts = { cwd = '/test' }
+
+      local job = jobs.start_job('test_job', args, opts, callback)
+
+      assert.is_not_nil(job)
+      assert.equals(1, #mock_jobs)
+      assert.same(args, mock_jobs[1].args)
+      assert.same(opts, mock_jobs[1].opts)
+      assert.equals(callback, mock_jobs[1].callback)
+    end)
+
+    it('kills existing job when starting new job with same ID', function()
+      local killed = false
+      vim.system = function(args, opts, callback)
+        return {
+          kill = function()
+            killed = true
+          end,
+          args = args,
+          opts = opts,
+          callback = callback,
+        }
+      end
+
+      jobs.start_job('same_id', { 'first' }, {}, function() end)
+      jobs.start_job('same_id', { 'second' }, {}, function() end)
+
+      assert.is_true(killed)
+    end)
+
+    it('kills specific job by ID', function()
+      local killed = false
+      vim.system = function()
+        return {
+          kill = function()
+            killed = true
+          end,
+        }
+      end
+
+      jobs.start_job('target_job', { 'test' }, {}, function() end)
+      jobs.kill_job('target_job')
+
+      assert.is_true(killed)
+    end)
+
+    it('kills all active jobs', function()
+      local kill_count = 0
+      vim.system = function()
+        return {
+          kill = function()
+            kill_count = kill_count + 1
+          end,
+        }
+      end
+
+      jobs.start_job('job1', { 'test1' }, {}, function() end)
+      jobs.start_job('job2', { 'test2' }, {}, function() end)
+      jobs.kill_all_jobs()
+
+      assert.equals(2, kill_count)
+    end)
+
+    it('tracks active job IDs correctly', function()
+      jobs.start_job('job1', { 'test1' }, {}, function() end)
+      jobs.start_job('job2', { 'test2' }, {}, function() end)
+
+      local active_jobs = jobs.get_active_jobs()
+      assert.equals(2, #active_jobs)
+      assert.is_true(vim.tbl_contains(active_jobs, 'job1'))
+      assert.is_true(vim.tbl_contains(active_jobs, 'job2'))
+
+      jobs.kill_job('job1')
+      active_jobs = jobs.get_active_jobs()
+      assert.equals(1, #active_jobs)
+      assert.is_true(vim.tbl_contains(active_jobs, 'job2'))
+    end)
+  end)
+end)

spec/async_scraper_spec.lua

@@ -0,0 +1,185 @@
+describe('cp.async.scraper', function()
+  local scraper
+  local spec_helper = require('spec.spec_helper')
+  local mock_cache, mock_utils
+  local callback_results = {}
+
+  before_each(function()
+    spec_helper.setup()
+    callback_results = {}
+
+    mock_cache = {
+      load = function() end,
+      get_contest_data = function()
+        return nil
+      end,
+      set_contest_data = function() end,
+      set_test_cases = function() end,
+    }
+
+    mock_utils = {
+      setup_python_env = function()
+        return true
+      end,
+      get_plugin_path = function()
+        return '/test/plugin'
+      end,
+    }
+
+    vim.system = function(cmd, opts, callback)
+      local result = { code = 0, stdout = '{}', stderr = '' }
+      if cmd[1] == 'ping' then
+        result = { code = 0 }
+      elseif cmd[1] == 'uv' and vim.tbl_contains(cmd, 'metadata') then
+        result.stdout = '{"success": true, "problems": [{"id": "a", "name": "Test Problem"}]}'
+      elseif cmd[1] == 'uv' and vim.tbl_contains(cmd, 'tests') then
+        result.stdout =
+          '{"success": true, "tests": [{"input": "1 2", "expected": "3"}], "timeout_ms": 2000, "memory_mb": 256.0, "url": "https://example.com"}'
+      end
+      callback(result)
+    end
+
+    vim.fn.mkdir = function() end
+
+    package.loaded['cp.cache'] = mock_cache
+    package.loaded['cp.utils'] = mock_utils
+    scraper = spec_helper.fresh_require('cp.async.scraper')
+  end)
+
+  after_each(function()
+    spec_helper.teardown()
+  end)
+
+  describe('scrape_contest_metadata_async', function()
+    it('returns cached data immediately if available', function()
+      mock_cache.get_contest_data = function()
+        return { problems = { { id = 'cached', name = 'Cached Problem' } } }
+      end
+
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_true(callback_results[1].success)
+      assert.equals('Cached Problem', callback_results[1].problems[1].name)
+    end)
+
+    it('calls callback with success result after scraping', function()
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_true(callback_results[1].success)
+      assert.equals(1, #callback_results[1].problems)
+      assert.equals('Test Problem', callback_results[1].problems[1].name)
+    end)
+
+    it('calls callback with error on network failure', function()
+      vim.system = function(cmd, opts, callback)
+        if cmd[1] == 'ping' then
+          callback({ code = 1 })
+        end
+      end
+
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_false(callback_results[1].success)
+      assert.equals('No internet connection available', callback_results[1].error)
+    end)
+
+    it('calls callback with error on python env failure', function()
+      mock_utils.setup_python_env = function()
+        return false
+      end
+
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_false(callback_results[1].success)
+      assert.equals('Python environment setup failed', callback_results[1].error)
+    end)
+
+    it('calls callback with error on subprocess failure', function()
+      vim.system = function(cmd, opts, callback)
+        if cmd[1] == 'ping' then
+          callback({ code = 0 })
+        else
+          callback({ code = 1, stderr = 'execution failed' })
+        end
+      end
+
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_false(callback_results[1].success)
+      assert.is_not_nil(callback_results[1].error:match('Failed to run metadata scraper'))
+    end)
+
+    it('calls callback with error on invalid JSON', function()
+      vim.system = function(cmd, opts, callback)
+        if cmd[1] == 'ping' then
+          callback({ code = 0 })
+        else
+          callback({ code = 0, stdout = 'invalid json' })
+        end
+      end
+
+      scraper.scrape_contest_metadata_async('atcoder', 'abc123', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_false(callback_results[1].success)
+      assert.is_not_nil(callback_results[1].error:match('Failed to parse metadata scraper output'))
+    end)
+  end)
+
+  describe('scrape_problem_async', function()
+    it('calls callback with success after scraping tests', function()
+      scraper.scrape_problem_async('atcoder', 'abc123', 'a', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_true(callback_results[1].success)
+      assert.equals('a', callback_results[1].problem_id)
+      assert.equals(1, callback_results[1].test_count)
+    end)
+
+    it('handles network failure gracefully', function()
+      vim.system = function(cmd, opts, callback)
+        if cmd[1] == 'ping' then
+          callback({ code = 1 })
+        end
+      end
+
+      scraper.scrape_problem_async('atcoder', 'abc123', 'a', function(result)
+        callback_results[#callback_results + 1] = result
+      end)
+
+      assert.equals(1, #callback_results)
+      assert.is_false(callback_results[1].success)
+      assert.equals('a', callback_results[1].problem_id)
+      assert.equals('No internet connection available', callback_results[1].error)
+    end)
+
+    it('validates input parameters', function()
+      assert.has_error(function()
+        scraper.scrape_contest_metadata_async(nil, 'abc123', function() end)
+      end)
+
+      assert.has_error(function()
+        scraper.scrape_problem_async('atcoder', nil, 'a', function() end)
+      end)
+    end)
+  end)
+end)

spec/async_setup_spec.lua

@@ -0,0 +1,286 @@
+describe('cp.async.setup', function()
+  local setup
+  local spec_helper = require('spec.spec_helper')
+  local mock_async, mock_scraper, mock_state
+  local callback_calls = {}
+
+  before_each(function()
+    spec_helper.setup()
+    callback_calls = {}
+
+    mock_async = {
+      start_contest_operation = function() end,
+      finish_contest_operation = function() end,
+    }
+
+    mock_scraper = {
+      scrape_contest_metadata_async = function(platform, contest_id, callback)
+        callback({
+          success = true,
+          problems = {
+            { id = 'a', name = 'Problem A' },
+            { id = 'b', name = 'Problem B' },
+          },
+        })
+      end,
+      scrape_problem_async = function(platform, contest_id, problem_id, callback)
+        callback({
+          success = true,
+          problem_id = problem_id,
+          test_cases = { { input = '1', expected = '1' } },
+          test_count = 1,
+        })
+      end,
+    }
+
+    mock_state = {
+      get_platform = function()
+        return 'atcoder'
+      end,
+      get_contest_id = function()
+        return 'abc123'
+      end,
+      set_contest_id = function() end,
+      set_problem_id = function() end,
+      set_test_cases = function() end,
+      set_run_panel_active = function() end,
+    }
+
+    local mock_config = {
+      get_config = function()
+        return {
+          scrapers = { 'atcoder', 'codeforces' },
+          hooks = nil,
+        }
+      end,
+    }
+
+    local mock_cache = {
+      load = function() end,
+      get_test_cases = function()
+        return nil
+      end,
+      set_file_state = function() end,
+    }
+
+    local mock_problem = {
+      create_context = function()
+        return {
+          source_file = '/test/source.cpp',
+          problem_name = 'abc123a',
+        }
+      end,
+    }
+
+    vim.cmd = { e = function() end, only = function() end }
+    vim.api.nvim_get_current_buf = function()
+      return 1
+    end
+    vim.api.nvim_buf_get_lines = function()
+      return { '' }
+    end
+    vim.fn.expand = function()
+      return '/test/file.cpp'
+    end
+
+    package.loaded['cp.async'] = mock_async
+    package.loaded['cp.async.scraper'] = mock_scraper
+    package.loaded['cp.state'] = mock_state
+    package.loaded['cp.config'] = mock_config
+    package.loaded['cp.cache'] = mock_cache
+    package.loaded['cp.problem'] = mock_problem
+
+    setup = spec_helper.fresh_require('cp.async.setup')
+  end)
+
+  after_each(function()
+    spec_helper.teardown()
+  end)
+
+  describe('setup_contest_async', function()
+    it('guards against multiple simultaneous operations', function()
+      local started = false
+      mock_async.start_contest_operation = function()
+        started = true
+      end
+
+      setup.setup_contest_async('abc123', 'cpp')
+
+      assert.is_true(started)
+    end)
+
+    it('handles metadata scraping success', function()
+      local finished = false
+      mock_async.finish_contest_operation = function()
+        finished = true
+      end
+
+      setup.setup_contest_async('abc123', 'cpp')
+
+      assert.is_true(finished)
+    end)
+
+    it('handles metadata scraping failure gracefully', function()
+      mock_scraper.scrape_contest_metadata_async = function(platform, contest_id, callback)
+        callback({
+          success = false,
+          error = 'network error',
+        })
+      end
+
+      local finished = false
+      mock_async.finish_contest_operation = function()
+        finished = true
+      end
+
+      setup.setup_contest_async('abc123', 'cpp')
+
+      assert.is_true(finished)
+    end)
+
+    it('handles disabled scraping platform', function()
+      mock_state.get_platform = function()
+        return 'disabled_platform'
+      end
+
+      assert.has_no_errors(function()
+        setup.setup_contest_async('abc123', 'cpp')
+      end)
+    end)
+  end)
+
+  describe('setup_problem_async', function()
+    it('opens buffer immediately', function()
+      local buffer_opened = false
+      vim.cmd.e = function()
+        buffer_opened = true
+      end
+
+      setup.setup_problem_async('abc123', 'a', 'cpp')
+
+      assert.is_true(buffer_opened)
+    end)
+
+    it('uses cached test cases if available', function()
+      local cached_cases = { { input = 'cached', expected = 'result' } }
+      local mock_cache = require('cp.cache')
+      mock_cache.get_test_cases = function()
+        return cached_cases
+      end
+
+      local set_test_cases_called = false
+      mock_state.set_test_cases = function(cases)
+        assert.same(cached_cases, cases)
+        set_test_cases_called = true
+      end
+
+      setup.setup_problem_async('abc123', 'a', 'cpp')
+
+      assert.is_true(set_test_cases_called)
+    end)
+
+    it('starts background test scraping if not cached', function()
+      local scraping_started = false
+      mock_scraper.scrape_problem_async = function(platform, contest_id, problem_id, callback)
+        scraping_started = true
+        callback({ success = true, problem_id = problem_id, test_cases = {} })
+      end
+
+      setup.setup_problem_async('abc123', 'a', 'cpp')
+
+      assert.is_true(scraping_started)
+    end)
+
+    it('finishes contest operation on completion', function()
+      local finished = false
+      mock_async.finish_contest_operation = function()
+        finished = true
+      end
+
+      setup.setup_problem_async('abc123', 'a', 'cpp')
+
+      assert.is_true(finished)
+    end)
+  end)
+
+  describe('handle_full_setup_async', function()
+    it('validates problem exists in contest', function()
+      mock_scraper.scrape_contest_metadata_async = function(platform, contest_id, callback)
+        callback({
+          success = true,
+          problems = { { id = 'a' }, { id = 'b' } },
+        })
+      end
+
+      local cmd = { platform = 'atcoder', contest = 'abc123', problem = 'c' }
+
+      local finished = false
+      mock_async.finish_contest_operation = function()
+        finished = true
+      end
+
+      setup.handle_full_setup_async(cmd)
+
+      assert.is_true(finished)
+    end)
+
+    it('proceeds with valid problem', function()
+      mock_scraper.scrape_contest_metadata_async = function(platform, contest_id, callback)
+        callback({
+          success = true,
+          problems = { { id = 'a' }, { id = 'b' } },
+        })
+      end
+
+      local cmd = { platform = 'atcoder', contest = 'abc123', problem = 'a' }
+
+      assert.has_no_errors(function()
+        setup.handle_full_setup_async(cmd)
+      end)
+    end)
+  end)
+
+  describe('background problem scraping', function()
+    it('scrapes uncached problems in background', function()
+      local problems = { { id = 'a' }, { id = 'b' }, { id = 'c' } }
+      local scraping_calls = {}
+
+      mock_scraper.scrape_problem_async = function(platform, contest_id, problem_id, callback)
+        scraping_calls[#scraping_calls + 1] = problem_id
+        callback({ success = true, problem_id = problem_id })
+      end
+
+      local mock_cache = require('cp.cache')
+      mock_cache.get_test_cases = function()
+        return nil
+      end
+
+      setup.start_background_problem_scraping('abc123', problems, { scrapers = { 'atcoder' } })
+
+      assert.equals(3, #scraping_calls)
+      assert.is_true(vim.tbl_contains(scraping_calls, 'a'))
+      assert.is_true(vim.tbl_contains(scraping_calls, 'b'))
+      assert.is_true(vim.tbl_contains(scraping_calls, 'c'))
+    end)
+
+    it('skips already cached problems', function()
+      local problems = { { id = 'a' }, { id = 'b' } }
+      local scraping_calls = {}
+
+      mock_scraper.scrape_problem_async = function(platform, contest_id, problem_id, callback)
+        scraping_calls[#scraping_calls + 1] = problem_id
+        callback({ success = true, problem_id = problem_id })
+      end
+
+      local mock_cache = require('cp.cache')
+      mock_cache.get_test_cases = function(platform, contest_id, problem_id)
+        return problem_id == 'a' and { { input = '1', expected = '1' } } or nil
+      end
+
+      setup.start_background_problem_scraping('abc123', problems, { scrapers = { 'atcoder' } })
+
+      assert.equals(1, #scraping_calls)
+      assert.equals('b', scraping_calls[1])
+    end)
+  end)
+end)

spec/scraper_spec.lua

@@ -1,470 +0,0 @@
-describe('cp.scrape', function()
-  local scrape
-  local mock_cache
-  local mock_utils
-  local mock_system_calls
-  local temp_files
-  local spec_helper = require('spec.spec_helper')
-
-  before_each(function()
-    spec_helper.setup()
-    temp_files = {}
-    mock_system_calls = {}
-
-    mock_cache = {
-      load = function() end,
-      get_contest_data = function()
-        return nil
-      end,
-      set_contest_data = function() end,
-      set_test_cases = function() end,
-    }
-
-    mock_utils = {
-      setup_python_env = function()
-        return true
-      end,
-      get_plugin_path = function()
-        return '/test/plugin/path'
-      end,
-    }
-
-    vim.system = function(cmd, opts)
-      table.insert(mock_system_calls, { cmd = cmd, opts = opts })
-
-      local result = { code = 0, stdout = '{}', stderr = '' }
-
-      if cmd[1] == 'ping' then
-        result = { code = 0 }
-      elseif cmd[1] == 'uv' and cmd[2] == 'sync' then
-        result = { code = 0, stdout = '', stderr = '' }
-      elseif cmd[1] == 'uv' and cmd[2] == 'run' then
-        if vim.tbl_contains(cmd, 'metadata') then
-          result.stdout = '{"success": true, "problems": [{"id": "a", "name": "Test Problem"}]}'
-        elseif vim.tbl_contains(cmd, 'tests') then
-          result.stdout =
-            '{"success": true, "tests": [{"input": "1 2", "expected": "3"}], "url": "https://example.com", "timeout_ms": 2000, "memory_mb": 256.0}'
-        end
-      end
-
-      return {
-        wait = function()
-          return result
-        end,
-      }
-    end
-
-    package.loaded['cp.cache'] = mock_cache
-    package.loaded['cp.utils'] = mock_utils
-    scrape = spec_helper.fresh_require('cp.scrape')
-
-    local original_fn = vim.fn
-    vim.fn = vim.tbl_extend('force', vim.fn, {
-      executable = function(cmd)
-        if cmd == 'uv' then
-          return 1
-        end
-        return original_fn.executable(cmd)
-      end,
-      isdirectory = function(path)
-        if path:match('%.venv$') then
-          return 1
-        end
-        return original_fn.isdirectory(path)
-      end,
-      filereadable = function(path)
-        if temp_files[path] then
-          return 1
-        end
-        return 0
-      end,
-      readfile = function(path)
-        return temp_files[path] or {}
-      end,
-      writefile = function(lines, path)
-        temp_files[path] = lines
-      end,
-      mkdir = function() end,
-      fnamemodify = function(path, modifier)
-        if modifier == ':r' then
-          return path:gsub('%..*$', '')
-        end
-        return original_fn.fnamemodify(path, modifier)
-      end,
-    })
-  end)
-
-  after_each(function()
-    package.loaded['cp.cache'] = nil
-    vim.system = vim.system_original or vim.system
-    spec_helper.teardown()
-    temp_files = {}
-  end)
-
-  describe('cache integration', function()
-    it('returns cached data when available', function()
-      mock_cache.get_contest_data = function(platform, contest_id)
-        if platform == 'atcoder' and contest_id == 'abc123' then
-          return { problems = { { id = 'a', name = 'Cached Problem' } } }
-        end
-        return nil
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_true(result.success)
-      assert.equals(1, #result.problems)
-      assert.equals('Cached Problem', result.problems[1].name)
-      assert.equals(0, #mock_system_calls)
-    end)
-
-    it('stores scraped data in cache after successful scrape', function()
-      local stored_data = nil
-      mock_cache.set_contest_data = function(platform, contest_id, problems)
-        stored_data = { platform = platform, contest_id = contest_id, problems = problems }
-      end
-
-      scrape = spec_helper.fresh_require('cp.scrape')
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_true(result.success)
-      assert.is_not_nil(stored_data)
-      assert.equals('atcoder', stored_data.platform)
-      assert.equals('abc123', stored_data.contest_id)
-      assert.equals(1, #stored_data.problems)
-    end)
-  end)
-
-  describe('system dependency checks', function()
-    it('handles missing uv executable', function()
-      local cache = require('cp.cache')
-      local utils = require('cp.utils')
-
-      cache.load = function() end
-      cache.get_contest_data = function()
-        return nil
-      end
-
-      vim.fn.executable = function(cmd)
-        return cmd == 'uv' and 0 or 1
-      end
-
-      utils.setup_python_env = function()
-        return vim.fn.executable('uv') == 1
-      end
-
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 0 }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.is_not_nil(result.error)
-    end)
-
-    it('handles python environment setup failure', function()
-      local cache = require('cp.cache')
-
-      cache.load = function() end
-      cache.get_contest_data = function()
-        return nil
-      end
-
-      mock_utils.setup_python_env = function()
-        return false
-      end
-
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 0 }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.equals('Python environment setup failed', result.error)
-    end)
-
-    it('handles network connectivity issues', function()
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 1 }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.equals('No internet connection available', result.error)
-    end)
-  end)
-
-  describe('subprocess execution', function()
-    it('constructs correct command for atcoder metadata', function()
-      scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      local metadata_call = nil
-      for _, call in ipairs(mock_system_calls) do
-        if vim.tbl_contains(call.cmd, 'metadata') then
-          metadata_call = call
-          break
-        end
-      end
-
-      assert.is_not_nil(metadata_call)
-      assert.equals('uv', metadata_call.cmd[1])
-      assert.equals('run', metadata_call.cmd[2])
-      assert.is_true(vim.tbl_contains(metadata_call.cmd, 'metadata'))
-      assert.is_true(vim.tbl_contains(metadata_call.cmd, 'abc123'))
-    end)
-
-    it('constructs correct command for cses metadata', function()
-      scrape.scrape_contest_metadata('cses', 'sorting_and_searching')
-
-      local metadata_call = nil
-      for _, call in ipairs(mock_system_calls) do
-        if vim.tbl_contains(call.cmd, 'metadata') then
-          metadata_call = call
-          break
-        end
-      end
-
-      assert.is_not_nil(metadata_call)
-      assert.equals('uv', metadata_call.cmd[1])
-      assert.is_true(vim.tbl_contains(metadata_call.cmd, 'metadata'))
-      assert.is_true(vim.tbl_contains(metadata_call.cmd, 'sorting_and_searching'))
-    end)
-
-    it('handles subprocess execution failure', function()
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 0 }
-            end,
-          }
-        elseif cmd[1] == 'uv' and vim.tbl_contains(cmd, 'metadata') then
-          return {
-            wait = function()
-              return { code = 1, stderr = 'execution failed' }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.is_not_nil(result.error:match('Failed to run metadata scraper'))
-      assert.is_not_nil(result.error:match('execution failed'))
-    end)
-  end)
-
-  describe('json parsing', function()
-    it('handles invalid json output', function()
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 0 }
-            end,
-          }
-        elseif cmd[1] == 'uv' and vim.tbl_contains(cmd, 'metadata') then
-          return {
-            wait = function()
-              return { code = 0, stdout = 'invalid json' }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.is_not_nil(result.error:match('Failed to parse metadata scraper output'))
-    end)
-
-    it('handles scraper-reported failures', function()
-      vim.system = function(cmd)
-        if cmd[1] == 'ping' then
-          return {
-            wait = function()
-              return { code = 0 }
-            end,
-          }
-        elseif cmd[1] == 'uv' and vim.tbl_contains(cmd, 'metadata') then
-          return {
-            wait = function()
-              return {
-                code = 0,
-                stdout = '{"success": false, "error": "contest not found"}',
-              }
-            end,
-          }
-        end
-        return {
-          wait = function()
-            return { code = 0 }
-          end,
-        }
-      end
-
-      local result = scrape.scrape_contest_metadata('atcoder', 'abc123')
-
-      assert.is_false(result.success)
-      assert.equals('contest not found', result.error)
-    end)
-  end)
-
-  describe('problem scraping', function()
-    local test_context
-
-    before_each(function()
-      test_context = {
-        contest = 'atcoder',
-        contest_id = 'abc123',
-        problem_id = 'a',
-        problem_name = 'abc123a',
-        input_file = 'io/abc123a.cpin',
-        expected_file = 'io/abc123a.expected',
-      }
-    end)
-
-    it('uses existing files when available', function()
-      temp_files['io/abc123a.cpin'] = { '1 2' }
-      temp_files['io/abc123a.expected'] = { '3' }
-      temp_files['io/abc123a.1.cpin'] = { '4 5' }
-      temp_files['io/abc123a.1.cpout'] = { '9' }
-
-      local result = scrape.scrape_problem(test_context)
-
-      assert.is_true(result.success)
-      assert.equals('abc123a', result.problem_id)
-      assert.equals(1, result.test_count)
-      assert.equals(0, #mock_system_calls)
-    end)
-
-    it('scrapes and writes test case files', function()
-      local result = scrape.scrape_problem(test_context)
-
-      assert.is_true(result.success)
-      assert.equals('abc123a', result.problem_id)
-      assert.equals(1, result.test_count)
-      assert.is_not_nil(temp_files['io/abc123a.1.cpin'])
-      assert.is_not_nil(temp_files['io/abc123a.1.cpout'])
-      assert.equals('1 2', table.concat(temp_files['io/abc123a.1.cpin'], '\n'))
-      assert.equals('3', table.concat(temp_files['io/abc123a.1.cpout'], '\n'))
-    end)
-
-    it('constructs correct command for atcoder problem tests', function()
-      scrape.scrape_problem(test_context)
-
-      local tests_call = nil
-      for _, call in ipairs(mock_system_calls) do
-        if vim.tbl_contains(call.cmd, 'tests') then
-          tests_call = call
-          break
-        end
-      end
-
-      assert.is_not_nil(tests_call)
-      assert.is_true(vim.tbl_contains(tests_call.cmd, 'tests'))
-      assert.is_true(vim.tbl_contains(tests_call.cmd, 'abc123'))
-      assert.is_true(vim.tbl_contains(tests_call.cmd, 'a'))
-    end)
-
-    it('constructs correct command for cses problem tests', function()
-      test_context.contest = 'cses'
-      test_context.contest_id = 'sorting_and_searching'
-      test_context.problem_id = '1001'
-
-      scrape.scrape_problem(test_context)
-
-      local tests_call = nil
-      for _, call in ipairs(mock_system_calls) do
-        if vim.tbl_contains(call.cmd, 'tests') then
-          tests_call = call
-          break
-        end
-      end
-
-      assert.is_not_nil(tests_call)
-      assert.is_true(vim.tbl_contains(tests_call.cmd, 'tests'))
-      assert.is_true(vim.tbl_contains(tests_call.cmd, '1001'))
-      assert.is_true(vim.tbl_contains(tests_call.cmd, 'sorting_and_searching'))
-    end)
-  end)
-
-  describe('error scenarios', function()
-    it('validates input parameters', function()
-      assert.has_error(function()
-        scrape.scrape_contest_metadata(nil, 'abc123')
-      end)
-
-      assert.has_error(function()
-        scrape.scrape_contest_metadata('atcoder', nil)
-      end)
-    end)
-
-    it('handles file system errors gracefully', function()
-      vim.fn.mkdir = function()
-        error('permission denied')
-      end
-
-      local ctx = {
-        contest = 'atcoder',
-        contest_id = 'abc123',
-        problem_id = 'a',
-        problem_name = 'abc123a',
-        input_file = 'io/abc123a.cpin',
-        expected_file = 'io/abc123a.expected',
-      }
-
-      assert.has_error(function()
-        scrape.scrape_problem(ctx)
-      end)
-    end)
-  end)
-end)

spec/spec_helper.lua

@@ -103,6 +103,61 @@ function M.mock_scraper_success()
   }
 end
 
+function M.mock_async_scraper_success()
+  package.loaded['cp.async.scraper'] = {
+    scrape_contest_metadata_async = function(platform, contest_id, callback)
+      vim.schedule(function()
+        callback({
+          success = true,
+          problems = {
+            { id = 'a' },
+            { id = 'b' },
+            { id = 'c' },
+          },
+        })
+      end)
+    end,
+    scrape_problem_async = function(platform, contest_id, problem_id, callback)
+      vim.schedule(function()
+        callback({
+          success = true,
+          problem_id = problem_id,
+          test_cases = {
+            { input = '1 2', expected = '3' },
+            { input = '3 4', expected = '7' },
+          },
+          test_count = 2,
+          timeout_ms = 2000,
+          memory_mb = 256.0,
+          url = 'https://example.com',
+        })
+      end)
+    end,
+  }
+end
+
+function M.mock_async_scraper_failure()
+  package.loaded['cp.async.scraper'] = {
+    scrape_contest_metadata_async = function(platform, contest_id, callback)
+      vim.schedule(function()
+        callback({
+          success = false,
+          error = 'mock network error',
+        })
+      end)
+    end,
+    scrape_problem_async = function(platform, contest_id, problem_id, callback)
+      vim.schedule(function()
+        callback({
+          success = false,
+          problem_id = problem_id,
+          error = 'mock scraping failed',
+        })
+      end)
+    end,
+  }
+end
+
 function M.has_error_logged()
   for _, log_entry in ipairs(M.logged_messages) do
     if log_entry.level == vim.log.levels.ERROR then
@@ -135,6 +190,10 @@ end
 function M.teardown()
   package.loaded['cp.log'] = nil
   package.loaded['cp.scrape'] = nil
+  package.loaded['cp.async.scraper'] = nil
+  package.loaded['cp.async.jobs'] = nil
+  package.loaded['cp.async.setup'] = nil
+  package.loaded['cp.async'] = nil
   M.logged_messages = {}
 end