fix(scrape): proper vars
This commit is contained in:
parent
df1b4c2009
commit
1f38dba57f
3 changed files with 22 additions and 36 deletions
|
|
@ -9,7 +9,6 @@
|
||||||
|
|
||||||
local M = {}
|
local M = {}
|
||||||
|
|
||||||
---Vim's built-in diff backend using diffthis
|
|
||||||
---@type DiffBackend
|
---@type DiffBackend
|
||||||
local vim_backend = {
|
local vim_backend = {
|
||||||
name = 'vim',
|
name = 'vim',
|
||||||
|
|
@ -18,17 +17,15 @@ local vim_backend = {
|
||||||
|
|
||||||
return {
|
return {
|
||||||
content = actual_lines,
|
content = actual_lines,
|
||||||
highlights = nil, -- diffthis handles highlighting
|
highlights = nil,
|
||||||
}
|
}
|
||||||
end,
|
end,
|
||||||
}
|
}
|
||||||
|
|
||||||
---Git word-diff backend for character-level precision
|
|
||||||
---@type DiffBackend
|
---@type DiffBackend
|
||||||
local git_backend = {
|
local git_backend = {
|
||||||
name = 'git',
|
name = 'git',
|
||||||
render = function(expected, actual)
|
render = function(expected, actual)
|
||||||
-- Create temporary files for git diff
|
|
||||||
local tmp_expected = vim.fn.tempname()
|
local tmp_expected = vim.fn.tempname()
|
||||||
local tmp_actual = vim.fn.tempname()
|
local tmp_actual = vim.fn.tempname()
|
||||||
|
|
||||||
|
|
@ -48,7 +45,6 @@ local git_backend = {
|
||||||
|
|
||||||
local result = vim.system(cmd, { text = true }):wait()
|
local result = vim.system(cmd, { text = true }):wait()
|
||||||
|
|
||||||
-- Clean up temp files
|
|
||||||
vim.fn.delete(tmp_expected)
|
vim.fn.delete(tmp_expected)
|
||||||
vim.fn.delete(tmp_actual)
|
vim.fn.delete(tmp_actual)
|
||||||
|
|
||||||
|
|
@ -58,25 +54,21 @@ local git_backend = {
|
||||||
highlights = {},
|
highlights = {},
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
-- Parse git diff output to extract content and highlights
|
|
||||||
local diff_content = result.stdout or ''
|
local diff_content = result.stdout or ''
|
||||||
local lines = {}
|
local lines = {}
|
||||||
local highlights = {}
|
local highlights = {}
|
||||||
local line_num = 0
|
local line_num = 0
|
||||||
|
|
||||||
-- Extract content lines that start with space, +, or -
|
|
||||||
for line in diff_content:gmatch('[^\n]*') do
|
for line in diff_content:gmatch('[^\n]*') do
|
||||||
if
|
if
|
||||||
line:match('^[%s%+%-]')
|
line:match('^[%s%+%-]')
|
||||||
or (not line:match('^[@%-+]') and not line:match('^index') and not line:match('^diff'))
|
or (not line:match('^[@%-+]') and not line:match('^index') and not line:match('^diff'))
|
||||||
then
|
then
|
||||||
-- This is content, not metadata
|
|
||||||
local clean_line = line
|
local clean_line = line
|
||||||
if line:match('^[%+%-]') then
|
if line:match('^[%+%-]') then
|
||||||
clean_line = line:sub(2) -- Remove +/- prefix
|
clean_line = line:sub(2)
|
||||||
end
|
end
|
||||||
|
|
||||||
-- Parse diff markers in the line
|
|
||||||
local col_pos = 0
|
local col_pos = 0
|
||||||
local processed_line = ''
|
local processed_line = ''
|
||||||
local i = 1
|
local i = 1
|
||||||
|
|
@ -97,28 +89,26 @@ local git_backend = {
|
||||||
end
|
end
|
||||||
|
|
||||||
if next_marker_start then
|
if next_marker_start then
|
||||||
-- Add text before marker
|
|
||||||
if next_marker_start > i then
|
if next_marker_start > i then
|
||||||
local before_text = clean_line:sub(i, next_marker_start - 1)
|
local before_text = clean_line:sub(i, next_marker_start - 1)
|
||||||
processed_line = processed_line .. before_text
|
processed_line = processed_line .. before_text
|
||||||
col_pos = col_pos + #before_text
|
col_pos = col_pos + #before_text
|
||||||
end
|
end
|
||||||
|
|
||||||
-- Extract and add marker content with highlighting
|
|
||||||
local marker_end = (marker_type == 'removed') and removed_end or added_end
|
local marker_end = (marker_type == 'removed') and removed_end or added_end
|
||||||
local marker_text = clean_line:sub(next_marker_start, marker_end)
|
local marker_text = clean_line:sub(next_marker_start, marker_end)
|
||||||
local content_text
|
local content_text
|
||||||
|
|
||||||
if marker_type == 'removed' then
|
if marker_type == 'removed' then
|
||||||
content_text = marker_text:sub(3, -3) -- Remove [- and -]
|
content_text = marker_text:sub(3, -3)
|
||||||
table.insert(highlights, {
|
table.insert(highlights, {
|
||||||
line = line_num,
|
line = line_num,
|
||||||
col_start = col_pos,
|
col_start = col_pos,
|
||||||
col_end = col_pos + #content_text,
|
col_end = col_pos + #content_text,
|
||||||
highlight_group = 'DiffDelete',
|
highlight_group = 'DiffDelete',
|
||||||
})
|
})
|
||||||
else -- added
|
else
|
||||||
content_text = marker_text:sub(3, -3) -- Remove {+ and +}
|
content_text = marker_text:sub(3, -3)
|
||||||
table.insert(highlights, {
|
table.insert(highlights, {
|
||||||
line = line_num,
|
line = line_num,
|
||||||
col_start = col_pos,
|
col_start = col_pos,
|
||||||
|
|
@ -131,7 +121,6 @@ local git_backend = {
|
||||||
col_pos = col_pos + #content_text
|
col_pos = col_pos + #content_text
|
||||||
i = marker_end + 1
|
i = marker_end + 1
|
||||||
else
|
else
|
||||||
-- No more markers, add rest of line
|
|
||||||
local rest = clean_line:sub(i)
|
local rest = clean_line:sub(i)
|
||||||
processed_line = processed_line .. rest
|
processed_line = processed_line .. rest
|
||||||
break
|
break
|
||||||
|
|
@ -152,34 +141,29 @@ local git_backend = {
|
||||||
end,
|
end,
|
||||||
}
|
}
|
||||||
|
|
||||||
---Available diff backends
|
|
||||||
---@type table<string, DiffBackend>
|
---@type table<string, DiffBackend>
|
||||||
local backends = {
|
local backends = {
|
||||||
vim = vim_backend,
|
vim = vim_backend,
|
||||||
git = git_backend,
|
git = git_backend,
|
||||||
}
|
}
|
||||||
|
|
||||||
---Get available backend names
|
|
||||||
---@return string[]
|
---@return string[]
|
||||||
function M.get_available_backends()
|
function M.get_available_backends()
|
||||||
return vim.tbl_keys(backends)
|
return vim.tbl_keys(backends)
|
||||||
end
|
end
|
||||||
|
|
||||||
---Get a diff backend by name
|
|
||||||
---@param name string
|
---@param name string
|
||||||
---@return DiffBackend?
|
---@return DiffBackend?
|
||||||
function M.get_backend(name)
|
function M.get_backend(name)
|
||||||
return backends[name]
|
return backends[name]
|
||||||
end
|
end
|
||||||
|
|
||||||
---Check if git backend is available
|
|
||||||
---@return boolean
|
---@return boolean
|
||||||
function M.is_git_available()
|
function M.is_git_available()
|
||||||
local result = vim.system({ 'git', '--version' }, { text = true }):wait()
|
local result = vim.system({ 'git', '--version' }, { text = true }):wait()
|
||||||
return result.code == 0
|
return result.code == 0
|
||||||
end
|
end
|
||||||
|
|
||||||
---Get the best available backend based on config and system availability
|
|
||||||
---@param preferred_backend? string
|
---@param preferred_backend? string
|
||||||
---@return DiffBackend
|
---@return DiffBackend
|
||||||
function M.get_best_backend(preferred_backend)
|
function M.get_best_backend(preferred_backend)
|
||||||
|
|
@ -193,7 +177,6 @@ function M.get_best_backend(preferred_backend)
|
||||||
return backends.vim
|
return backends.vim
|
||||||
end
|
end
|
||||||
|
|
||||||
---Render diff using specified backend
|
|
||||||
---@param expected string
|
---@param expected string
|
||||||
---@param actual string
|
---@param actual string
|
||||||
---@param backend_name? string
|
---@param backend_name? string
|
||||||
|
|
|
||||||
|
|
@ -269,7 +269,6 @@ def scrape_contests() -> list[ContestSummary]:
|
||||||
if single_div_match:
|
if single_div_match:
|
||||||
display_name = f"Round {single_div_match.group(1)} (Div. 1)"
|
display_name = f"Round {single_div_match.group(1)} (Div. 1)"
|
||||||
else:
|
else:
|
||||||
# Fallback: extract just the round number
|
|
||||||
round_match = re.search(r"Codeforces Round (\d+)", name)
|
round_match = re.search(r"Codeforces Round (\d+)", name)
|
||||||
if round_match:
|
if round_match:
|
||||||
display_name = f"Round {round_match.group(1)}"
|
display_name = f"Round {round_match.group(1)}"
|
||||||
|
|
@ -278,7 +277,7 @@ def scrape_contests() -> list[ContestSummary]:
|
||||||
ContestSummary(id=contest_id, name=name, display_name=display_name)
|
ContestSummary(id=contest_id, name=name, display_name=display_name)
|
||||||
)
|
)
|
||||||
|
|
||||||
return contests[:100] # Limit to recent 100 contests
|
return contests[:100]
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Failed to fetch contests: {e}", file=sys.stderr)
|
print(f"Failed to fetch contests: {e}", file=sys.stderr)
|
||||||
|
|
|
||||||
|
|
@ -54,18 +54,22 @@ def test_scrape_network_error(mocker):
|
||||||
|
|
||||||
def test_scrape_contests_success(mocker):
|
def test_scrape_contests_success(mocker):
|
||||||
def mock_get_side_effect(url, **kwargs):
|
def mock_get_side_effect(url, **kwargs):
|
||||||
if "page=1" in url:
|
if url == "https://atcoder.jp/contests/archive":
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
|
mock_response.raise_for_status.return_value = None
|
||||||
mock_response.text = """
|
mock_response.text = """
|
||||||
<table class="table table-default table-striped table-hover table-condensed table-bordered small">
|
<html>
|
||||||
<thead>
|
<ul class="pagination">
|
||||||
<tr>
|
<li>1</li>
|
||||||
<th>Start Time</th>
|
</ul>
|
||||||
<th>Contest Name</th>
|
</html>
|
||||||
<th>Duration</th>
|
"""
|
||||||
<th>Rated Range</th>
|
return mock_response
|
||||||
</tr>
|
elif "page=1" in url:
|
||||||
</thead>
|
mock_response = Mock()
|
||||||
|
mock_response.raise_for_status.return_value = None
|
||||||
|
mock_response.text = """
|
||||||
|
<table class="table">
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td>2025-01-15 21:00:00+0900</td>
|
<td>2025-01-15 21:00:00+0900</td>
|
||||||
|
|
@ -84,9 +88,9 @@ def test_scrape_contests_success(mocker):
|
||||||
"""
|
"""
|
||||||
return mock_response
|
return mock_response
|
||||||
else:
|
else:
|
||||||
# Return empty page for all other pages
|
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.text = "<html><body>No table found</body></html>"
|
mock_response.raise_for_status.return_value = None
|
||||||
|
mock_response.text = "<html></html>"
|
||||||
return mock_response
|
return mock_response
|
||||||
|
|
||||||
mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect)
|
mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue