fix async
This commit is contained in:
parent
540364926d
commit
7ac91a3c4d
8 changed files with 155 additions and 21 deletions
|
|
@ -290,6 +290,10 @@ Usage examples: >
|
||||||
for multi-test case problems commonly found
|
for multi-test case problems commonly found
|
||||||
in contests.
|
in contests.
|
||||||
|
|
||||||
|
AtCoder Heuristic Contests (AHC) are excluded
|
||||||
|
from the contest list as they don't have
|
||||||
|
standard sample test cases.
|
||||||
|
|
||||||
Codeforces ~
|
Codeforces ~
|
||||||
*cp-codeforces*
|
*cp-codeforces*
|
||||||
URL format: https://codeforces.com/contest/1234/problem/A
|
URL format: https://codeforces.com/contest/1234/problem/A
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,13 @@ local function problem_picker(platform, contest_id)
|
||||||
|
|
||||||
if #problems == 0 then
|
if #problems == 0 then
|
||||||
vim.notify(
|
vim.notify(
|
||||||
('No problems found for contest: %s %s'):format(platform_display_name, contest_id),
|
("Contest %s %s hasn't started yet or has no available problems"):format(
|
||||||
|
platform_display_name,
|
||||||
|
contest_id
|
||||||
|
),
|
||||||
vim.log.levels.WARN
|
vim.log.levels.WARN
|
||||||
)
|
)
|
||||||
|
contest_picker(platform)
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,8 @@ local function get_contests_for_platform(platform)
|
||||||
'contests',
|
'contests',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.progress(('running: %s'):format(table.concat(cmd, ' ')))
|
||||||
|
|
||||||
local result = vim
|
local result = vim
|
||||||
.system(cmd, {
|
.system(cmd, {
|
||||||
cwd = plugin_path,
|
cwd = plugin_path,
|
||||||
|
|
@ -67,6 +69,11 @@ local function get_contests_for_platform(platform)
|
||||||
})
|
})
|
||||||
:wait()
|
:wait()
|
||||||
|
|
||||||
|
logger.progress(('exit code: %d, stdout length: %d'):format(result.code, #(result.stdout or '')))
|
||||||
|
if result.stderr and #result.stderr > 0 then
|
||||||
|
logger.progress(('stderr: %s'):format(result.stderr:sub(1, 200)))
|
||||||
|
end
|
||||||
|
|
||||||
if result.code ~= 0 then
|
if result.code ~= 0 then
|
||||||
logger.log(
|
logger.log(
|
||||||
('Failed to load contests: %s'):format(result.stderr or 'unknown error'),
|
('Failed to load contests: %s'):format(result.stderr or 'unknown error'),
|
||||||
|
|
@ -75,9 +82,18 @@ local function get_contests_for_platform(platform)
|
||||||
return {}
|
return {}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
logger.progress(('stdout preview: %s'):format(result.stdout:sub(1, 100)))
|
||||||
|
|
||||||
local ok, data = pcall(vim.json.decode, result.stdout)
|
local ok, data = pcall(vim.json.decode, result.stdout)
|
||||||
if not ok or not data.success then
|
if not ok then
|
||||||
logger.log('Failed to parse contest data', vim.log.levels.ERROR)
|
logger.log(('JSON parse error: %s'):format(tostring(data)), vim.log.levels.ERROR)
|
||||||
|
return {}
|
||||||
|
end
|
||||||
|
if not data.success then
|
||||||
|
logger.log(
|
||||||
|
('Scraper returned success=false: %s'):format(data.error or 'no error message'),
|
||||||
|
vim.log.levels.ERROR
|
||||||
|
)
|
||||||
return {}
|
return {}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
@ -151,10 +167,14 @@ local function get_problems_for_contest(platform, contest_id)
|
||||||
end
|
end
|
||||||
|
|
||||||
local ok, data = pcall(vim.json.decode, result.stdout)
|
local ok, data = pcall(vim.json.decode, result.stdout)
|
||||||
if not ok or not data.success then
|
if not ok then
|
||||||
logger.log('Failed to parse contest data', vim.log.levels.ERROR)
|
logger.log('Failed to parse contest data', vim.log.levels.ERROR)
|
||||||
return problems
|
return problems
|
||||||
end
|
end
|
||||||
|
if not data.success then
|
||||||
|
logger.log(data.error or 'Contest scraping failed', vim.log.levels.ERROR)
|
||||||
|
return problems
|
||||||
|
end
|
||||||
|
|
||||||
if not data.problems or #data.problems == 0 then
|
if not data.problems or #data.problems == 0 then
|
||||||
logger.log('Contest has no problems available', vim.log.levels.WARN)
|
logger.log('Contest has no problems available', vim.log.levels.WARN)
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,13 @@ local function problem_picker(opts, platform, contest_id)
|
||||||
|
|
||||||
if #problems == 0 then
|
if #problems == 0 then
|
||||||
vim.notify(
|
vim.notify(
|
||||||
('No problems found for contest: %s %s'):format(platform_display_name, contest_id),
|
("Contest %s %s hasn't started yet or has no available problems"):format(
|
||||||
|
platform_display_name,
|
||||||
|
contest_id
|
||||||
|
),
|
||||||
vim.log.levels.WARN
|
vim.log.levels.WARN
|
||||||
)
|
)
|
||||||
|
contest_picker(opts, platform)
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,45 @@
|
||||||
from .atcoder import AtCoderScraper
|
# Lazy imports to avoid module loading conflicts when running scrapers with -m
|
||||||
from .base import BaseScraper, ScraperConfig
|
def __getattr__(name):
|
||||||
from .codeforces import CodeforcesScraper
|
if name == "AtCoderScraper":
|
||||||
from .cses import CSESScraper
|
from .atcoder import AtCoderScraper
|
||||||
from .models import (
|
|
||||||
ContestListResult,
|
return AtCoderScraper
|
||||||
ContestSummary,
|
elif name == "BaseScraper":
|
||||||
MetadataResult,
|
from .base import BaseScraper
|
||||||
ProblemSummary,
|
|
||||||
TestCase,
|
return BaseScraper
|
||||||
TestsResult,
|
elif name == "ScraperConfig":
|
||||||
)
|
from .base import ScraperConfig
|
||||||
|
|
||||||
|
return ScraperConfig
|
||||||
|
elif name == "CodeforcesScraper":
|
||||||
|
from .codeforces import CodeforcesScraper
|
||||||
|
|
||||||
|
return CodeforcesScraper
|
||||||
|
elif name == "CSESScraper":
|
||||||
|
from .cses import CSESScraper
|
||||||
|
|
||||||
|
return CSESScraper
|
||||||
|
elif name in [
|
||||||
|
"ContestListResult",
|
||||||
|
"ContestSummary",
|
||||||
|
"MetadataResult",
|
||||||
|
"ProblemSummary",
|
||||||
|
"TestCase",
|
||||||
|
"TestsResult",
|
||||||
|
]:
|
||||||
|
from .models import (
|
||||||
|
ContestListResult,
|
||||||
|
ContestSummary,
|
||||||
|
MetadataResult,
|
||||||
|
ProblemSummary,
|
||||||
|
TestCase,
|
||||||
|
TestsResult,
|
||||||
|
)
|
||||||
|
|
||||||
|
return locals()[name]
|
||||||
|
raise AttributeError(f"module 'scrapers' has no attribute '{name}'")
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AtCoderScraper",
|
"AtCoderScraper",
|
||||||
|
|
|
||||||
|
|
@ -272,7 +272,11 @@ def scrape_contests() -> list[ContestSummary]:
|
||||||
r"[\uff01-\uff5e]", lambda m: chr(ord(m.group()) - 0xFEE0), name
|
r"[\uff01-\uff5e]", lambda m: chr(ord(m.group()) - 0xFEE0), name
|
||||||
)
|
)
|
||||||
|
|
||||||
contests.append(ContestSummary(id=contest_id, name=name, display_name=name))
|
# Skip AtCoder Heuristic Contests (AHC) as they don't have standard sample tests
|
||||||
|
if not contest_id.startswith("ahc"):
|
||||||
|
contests.append(
|
||||||
|
ContestSummary(id=contest_id, name=name, display_name=name)
|
||||||
|
)
|
||||||
|
|
||||||
return contests
|
return contests
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -129,3 +129,71 @@ def test_scrape_contests_network_error(mocker):
|
||||||
result = scrape_contests()
|
result = scrape_contests()
|
||||||
|
|
||||||
assert result == []
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_scrape_contests_filters_ahc(mocker):
|
||||||
|
def mock_get_side_effect(url, **kwargs):
|
||||||
|
if url == "https://atcoder.jp/contests/archive":
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.raise_for_status.return_value = None
|
||||||
|
mock_response.text = """
|
||||||
|
<html>
|
||||||
|
<ul class="pagination">
|
||||||
|
<li>1</li>
|
||||||
|
</ul>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
return mock_response
|
||||||
|
elif "page=1" in url:
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.raise_for_status.return_value = None
|
||||||
|
mock_response.text = """
|
||||||
|
<table class="table">
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>2025-01-15 21:00:00+0900</td>
|
||||||
|
<td><a href="/contests/abc350">AtCoder Beginner Contest 350</a></td>
|
||||||
|
<td>01:40</td>
|
||||||
|
<td> - 1999</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2025-01-14 21:00:00+0900</td>
|
||||||
|
<td><a href="/contests/ahc044">AtCoder Heuristic Contest 044</a></td>
|
||||||
|
<td>05:00</td>
|
||||||
|
<td>-</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2025-01-13 21:00:00+0900</td>
|
||||||
|
<td><a href="/contests/arc170">AtCoder Regular Contest 170</a></td>
|
||||||
|
<td>02:00</td>
|
||||||
|
<td>1000 - 2799</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
"""
|
||||||
|
return mock_response
|
||||||
|
else:
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.raise_for_status.return_value = None
|
||||||
|
mock_response.text = "<html></html>"
|
||||||
|
return mock_response
|
||||||
|
|
||||||
|
mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect)
|
||||||
|
|
||||||
|
result = scrape_contests()
|
||||||
|
|
||||||
|
assert len(result) == 2
|
||||||
|
assert result[0] == ContestSummary(
|
||||||
|
id="abc350",
|
||||||
|
name="AtCoder Beginner Contest 350",
|
||||||
|
display_name="AtCoder Beginner Contest 350",
|
||||||
|
)
|
||||||
|
assert result[1] == ContestSummary(
|
||||||
|
id="arc170",
|
||||||
|
name="AtCoder Regular Contest 170",
|
||||||
|
display_name="AtCoder Regular Contest 170",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ensure ahc044 is filtered out
|
||||||
|
contest_ids = [contest.id for contest in result]
|
||||||
|
assert "ahc044" not in contest_ids
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,9 @@ from scrapers.base import BaseScraper
|
||||||
from scrapers.models import ContestListResult, MetadataResult, TestsResult
|
from scrapers.models import ContestListResult, MetadataResult, TestsResult
|
||||||
|
|
||||||
SCRAPERS = [
|
SCRAPERS = [
|
||||||
cls
|
scrapers.AtCoderScraper,
|
||||||
for name, cls in inspect.getmembers(scrapers, inspect.isclass)
|
scrapers.CodeforcesScraper,
|
||||||
if issubclass(cls, BaseScraper) and cls != BaseScraper
|
scrapers.CSESScraper,
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue