fix(scrape): proper vars

This commit is contained in:
Barrett Ruth 2025-09-21 00:31:10 -04:00
parent df1b4c2009
commit 1f38dba57f
3 changed files with 22 additions and 36 deletions

View file

@ -9,7 +9,6 @@
local M = {} local M = {}
---Vim's built-in diff backend using diffthis
---@type DiffBackend ---@type DiffBackend
local vim_backend = { local vim_backend = {
name = 'vim', name = 'vim',
@ -18,17 +17,15 @@ local vim_backend = {
return { return {
content = actual_lines, content = actual_lines,
highlights = nil, -- diffthis handles highlighting highlights = nil,
} }
end, end,
} }
---Git word-diff backend for character-level precision
---@type DiffBackend ---@type DiffBackend
local git_backend = { local git_backend = {
name = 'git', name = 'git',
render = function(expected, actual) render = function(expected, actual)
-- Create temporary files for git diff
local tmp_expected = vim.fn.tempname() local tmp_expected = vim.fn.tempname()
local tmp_actual = vim.fn.tempname() local tmp_actual = vim.fn.tempname()
@ -48,7 +45,6 @@ local git_backend = {
local result = vim.system(cmd, { text = true }):wait() local result = vim.system(cmd, { text = true }):wait()
-- Clean up temp files
vim.fn.delete(tmp_expected) vim.fn.delete(tmp_expected)
vim.fn.delete(tmp_actual) vim.fn.delete(tmp_actual)
@ -58,25 +54,21 @@ local git_backend = {
highlights = {}, highlights = {},
} }
else else
-- Parse git diff output to extract content and highlights
local diff_content = result.stdout or '' local diff_content = result.stdout or ''
local lines = {} local lines = {}
local highlights = {} local highlights = {}
local line_num = 0 local line_num = 0
-- Extract content lines that start with space, +, or -
for line in diff_content:gmatch('[^\n]*') do for line in diff_content:gmatch('[^\n]*') do
if if
line:match('^[%s%+%-]') line:match('^[%s%+%-]')
or (not line:match('^[@%-+]') and not line:match('^index') and not line:match('^diff')) or (not line:match('^[@%-+]') and not line:match('^index') and not line:match('^diff'))
then then
-- This is content, not metadata
local clean_line = line local clean_line = line
if line:match('^[%+%-]') then if line:match('^[%+%-]') then
clean_line = line:sub(2) -- Remove +/- prefix clean_line = line:sub(2)
end end
-- Parse diff markers in the line
local col_pos = 0 local col_pos = 0
local processed_line = '' local processed_line = ''
local i = 1 local i = 1
@ -97,28 +89,26 @@ local git_backend = {
end end
if next_marker_start then if next_marker_start then
-- Add text before marker
if next_marker_start > i then if next_marker_start > i then
local before_text = clean_line:sub(i, next_marker_start - 1) local before_text = clean_line:sub(i, next_marker_start - 1)
processed_line = processed_line .. before_text processed_line = processed_line .. before_text
col_pos = col_pos + #before_text col_pos = col_pos + #before_text
end end
-- Extract and add marker content with highlighting
local marker_end = (marker_type == 'removed') and removed_end or added_end local marker_end = (marker_type == 'removed') and removed_end or added_end
local marker_text = clean_line:sub(next_marker_start, marker_end) local marker_text = clean_line:sub(next_marker_start, marker_end)
local content_text local content_text
if marker_type == 'removed' then if marker_type == 'removed' then
content_text = marker_text:sub(3, -3) -- Remove [- and -] content_text = marker_text:sub(3, -3)
table.insert(highlights, { table.insert(highlights, {
line = line_num, line = line_num,
col_start = col_pos, col_start = col_pos,
col_end = col_pos + #content_text, col_end = col_pos + #content_text,
highlight_group = 'DiffDelete', highlight_group = 'DiffDelete',
}) })
else -- added else
content_text = marker_text:sub(3, -3) -- Remove {+ and +} content_text = marker_text:sub(3, -3)
table.insert(highlights, { table.insert(highlights, {
line = line_num, line = line_num,
col_start = col_pos, col_start = col_pos,
@ -131,7 +121,6 @@ local git_backend = {
col_pos = col_pos + #content_text col_pos = col_pos + #content_text
i = marker_end + 1 i = marker_end + 1
else else
-- No more markers, add rest of line
local rest = clean_line:sub(i) local rest = clean_line:sub(i)
processed_line = processed_line .. rest processed_line = processed_line .. rest
break break
@ -152,34 +141,29 @@ local git_backend = {
end, end,
} }
---Available diff backends
---@type table<string, DiffBackend> ---@type table<string, DiffBackend>
local backends = { local backends = {
vim = vim_backend, vim = vim_backend,
git = git_backend, git = git_backend,
} }
---Get available backend names
---@return string[] ---@return string[]
function M.get_available_backends() function M.get_available_backends()
return vim.tbl_keys(backends) return vim.tbl_keys(backends)
end end
---Get a diff backend by name
---@param name string ---@param name string
---@return DiffBackend? ---@return DiffBackend?
function M.get_backend(name) function M.get_backend(name)
return backends[name] return backends[name]
end end
---Check if git backend is available
---@return boolean ---@return boolean
function M.is_git_available() function M.is_git_available()
local result = vim.system({ 'git', '--version' }, { text = true }):wait() local result = vim.system({ 'git', '--version' }, { text = true }):wait()
return result.code == 0 return result.code == 0
end end
---Get the best available backend based on config and system availability
---@param preferred_backend? string ---@param preferred_backend? string
---@return DiffBackend ---@return DiffBackend
function M.get_best_backend(preferred_backend) function M.get_best_backend(preferred_backend)
@ -193,7 +177,6 @@ function M.get_best_backend(preferred_backend)
return backends.vim return backends.vim
end end
---Render diff using specified backend
---@param expected string ---@param expected string
---@param actual string ---@param actual string
---@param backend_name? string ---@param backend_name? string

View file

@ -269,7 +269,6 @@ def scrape_contests() -> list[ContestSummary]:
if single_div_match: if single_div_match:
display_name = f"Round {single_div_match.group(1)} (Div. 1)" display_name = f"Round {single_div_match.group(1)} (Div. 1)"
else: else:
# Fallback: extract just the round number
round_match = re.search(r"Codeforces Round (\d+)", name) round_match = re.search(r"Codeforces Round (\d+)", name)
if round_match: if round_match:
display_name = f"Round {round_match.group(1)}" display_name = f"Round {round_match.group(1)}"
@ -278,7 +277,7 @@ def scrape_contests() -> list[ContestSummary]:
ContestSummary(id=contest_id, name=name, display_name=display_name) ContestSummary(id=contest_id, name=name, display_name=display_name)
) )
return contests[:100] # Limit to recent 100 contests return contests[:100]
except Exception as e: except Exception as e:
print(f"Failed to fetch contests: {e}", file=sys.stderr) print(f"Failed to fetch contests: {e}", file=sys.stderr)

View file

@ -54,18 +54,22 @@ def test_scrape_network_error(mocker):
def test_scrape_contests_success(mocker): def test_scrape_contests_success(mocker):
def mock_get_side_effect(url, **kwargs): def mock_get_side_effect(url, **kwargs):
if "page=1" in url: if url == "https://atcoder.jp/contests/archive":
mock_response = Mock() mock_response = Mock()
mock_response.raise_for_status.return_value = None
mock_response.text = """ mock_response.text = """
<table class="table table-default table-striped table-hover table-condensed table-bordered small"> <html>
<thead> <ul class="pagination">
<tr> <li>1</li>
<th>Start Time</th> </ul>
<th>Contest Name</th> </html>
<th>Duration</th> """
<th>Rated Range</th> return mock_response
</tr> elif "page=1" in url:
</thead> mock_response = Mock()
mock_response.raise_for_status.return_value = None
mock_response.text = """
<table class="table">
<tbody> <tbody>
<tr> <tr>
<td>2025-01-15 21:00:00+0900</td> <td>2025-01-15 21:00:00+0900</td>
@ -84,9 +88,9 @@ def test_scrape_contests_success(mocker):
""" """
return mock_response return mock_response
else: else:
# Return empty page for all other pages
mock_response = Mock() mock_response = Mock()
mock_response.text = "<html><body>No table found</body></html>" mock_response.raise_for_status.return_value = None
mock_response.text = "<html></html>"
return mock_response return mock_response
mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect) mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect)