fix async
This commit is contained in:
parent
540364926d
commit
7ac91a3c4d
8 changed files with 155 additions and 21 deletions
|
|
@ -290,6 +290,10 @@ Usage examples: >
|
|||
for multi-test case problems commonly found
|
||||
in contests.
|
||||
|
||||
AtCoder Heuristic Contests (AHC) are excluded
|
||||
from the contest list as they don't have
|
||||
standard sample test cases.
|
||||
|
||||
Codeforces ~
|
||||
*cp-codeforces*
|
||||
URL format: https://codeforces.com/contest/1234/problem/A
|
||||
|
|
|
|||
|
|
@ -8,9 +8,13 @@ local function problem_picker(platform, contest_id)
|
|||
|
||||
if #problems == 0 then
|
||||
vim.notify(
|
||||
('No problems found for contest: %s %s'):format(platform_display_name, contest_id),
|
||||
("Contest %s %s hasn't started yet or has no available problems"):format(
|
||||
platform_display_name,
|
||||
contest_id
|
||||
),
|
||||
vim.log.levels.WARN
|
||||
)
|
||||
contest_picker(platform)
|
||||
return
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,8 @@ local function get_contests_for_platform(platform)
|
|||
'contests',
|
||||
}
|
||||
|
||||
logger.progress(('running: %s'):format(table.concat(cmd, ' ')))
|
||||
|
||||
local result = vim
|
||||
.system(cmd, {
|
||||
cwd = plugin_path,
|
||||
|
|
@ -67,6 +69,11 @@ local function get_contests_for_platform(platform)
|
|||
})
|
||||
:wait()
|
||||
|
||||
logger.progress(('exit code: %d, stdout length: %d'):format(result.code, #(result.stdout or '')))
|
||||
if result.stderr and #result.stderr > 0 then
|
||||
logger.progress(('stderr: %s'):format(result.stderr:sub(1, 200)))
|
||||
end
|
||||
|
||||
if result.code ~= 0 then
|
||||
logger.log(
|
||||
('Failed to load contests: %s'):format(result.stderr or 'unknown error'),
|
||||
|
|
@ -75,9 +82,18 @@ local function get_contests_for_platform(platform)
|
|||
return {}
|
||||
end
|
||||
|
||||
logger.progress(('stdout preview: %s'):format(result.stdout:sub(1, 100)))
|
||||
|
||||
local ok, data = pcall(vim.json.decode, result.stdout)
|
||||
if not ok or not data.success then
|
||||
logger.log('Failed to parse contest data', vim.log.levels.ERROR)
|
||||
if not ok then
|
||||
logger.log(('JSON parse error: %s'):format(tostring(data)), vim.log.levels.ERROR)
|
||||
return {}
|
||||
end
|
||||
if not data.success then
|
||||
logger.log(
|
||||
('Scraper returned success=false: %s'):format(data.error or 'no error message'),
|
||||
vim.log.levels.ERROR
|
||||
)
|
||||
return {}
|
||||
end
|
||||
|
||||
|
|
@ -151,10 +167,14 @@ local function get_problems_for_contest(platform, contest_id)
|
|||
end
|
||||
|
||||
local ok, data = pcall(vim.json.decode, result.stdout)
|
||||
if not ok or not data.success then
|
||||
if not ok then
|
||||
logger.log('Failed to parse contest data', vim.log.levels.ERROR)
|
||||
return problems
|
||||
end
|
||||
if not data.success then
|
||||
logger.log(data.error or 'Contest scraping failed', vim.log.levels.ERROR)
|
||||
return problems
|
||||
end
|
||||
|
||||
if not data.problems or #data.problems == 0 then
|
||||
logger.log('Contest has no problems available', vim.log.levels.WARN)
|
||||
|
|
|
|||
|
|
@ -13,9 +13,13 @@ local function problem_picker(opts, platform, contest_id)
|
|||
|
||||
if #problems == 0 then
|
||||
vim.notify(
|
||||
('No problems found for contest: %s %s'):format(platform_display_name, contest_id),
|
||||
("Contest %s %s hasn't started yet or has no available problems"):format(
|
||||
platform_display_name,
|
||||
contest_id
|
||||
),
|
||||
vim.log.levels.WARN
|
||||
)
|
||||
contest_picker(opts, platform)
|
||||
return
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,45 @@
|
|||
from .atcoder import AtCoderScraper
|
||||
from .base import BaseScraper, ScraperConfig
|
||||
from .codeforces import CodeforcesScraper
|
||||
from .cses import CSESScraper
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
MetadataResult,
|
||||
ProblemSummary,
|
||||
TestCase,
|
||||
TestsResult,
|
||||
)
|
||||
# Lazy imports to avoid module loading conflicts when running scrapers with -m
|
||||
def __getattr__(name):
|
||||
if name == "AtCoderScraper":
|
||||
from .atcoder import AtCoderScraper
|
||||
|
||||
return AtCoderScraper
|
||||
elif name == "BaseScraper":
|
||||
from .base import BaseScraper
|
||||
|
||||
return BaseScraper
|
||||
elif name == "ScraperConfig":
|
||||
from .base import ScraperConfig
|
||||
|
||||
return ScraperConfig
|
||||
elif name == "CodeforcesScraper":
|
||||
from .codeforces import CodeforcesScraper
|
||||
|
||||
return CodeforcesScraper
|
||||
elif name == "CSESScraper":
|
||||
from .cses import CSESScraper
|
||||
|
||||
return CSESScraper
|
||||
elif name in [
|
||||
"ContestListResult",
|
||||
"ContestSummary",
|
||||
"MetadataResult",
|
||||
"ProblemSummary",
|
||||
"TestCase",
|
||||
"TestsResult",
|
||||
]:
|
||||
from .models import (
|
||||
ContestListResult,
|
||||
ContestSummary,
|
||||
MetadataResult,
|
||||
ProblemSummary,
|
||||
TestCase,
|
||||
TestsResult,
|
||||
)
|
||||
|
||||
return locals()[name]
|
||||
raise AttributeError(f"module 'scrapers' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AtCoderScraper",
|
||||
|
|
|
|||
|
|
@ -272,7 +272,11 @@ def scrape_contests() -> list[ContestSummary]:
|
|||
r"[\uff01-\uff5e]", lambda m: chr(ord(m.group()) - 0xFEE0), name
|
||||
)
|
||||
|
||||
contests.append(ContestSummary(id=contest_id, name=name, display_name=name))
|
||||
# Skip AtCoder Heuristic Contests (AHC) as they don't have standard sample tests
|
||||
if not contest_id.startswith("ahc"):
|
||||
contests.append(
|
||||
ContestSummary(id=contest_id, name=name, display_name=name)
|
||||
)
|
||||
|
||||
return contests
|
||||
|
||||
|
|
|
|||
|
|
@ -129,3 +129,71 @@ def test_scrape_contests_network_error(mocker):
|
|||
result = scrape_contests()
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_scrape_contests_filters_ahc(mocker):
|
||||
def mock_get_side_effect(url, **kwargs):
|
||||
if url == "https://atcoder.jp/contests/archive":
|
||||
mock_response = Mock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_response.text = """
|
||||
<html>
|
||||
<ul class="pagination">
|
||||
<li>1</li>
|
||||
</ul>
|
||||
</html>
|
||||
"""
|
||||
return mock_response
|
||||
elif "page=1" in url:
|
||||
mock_response = Mock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_response.text = """
|
||||
<table class="table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>2025-01-15 21:00:00+0900</td>
|
||||
<td><a href="/contests/abc350">AtCoder Beginner Contest 350</a></td>
|
||||
<td>01:40</td>
|
||||
<td> - 1999</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>2025-01-14 21:00:00+0900</td>
|
||||
<td><a href="/contests/ahc044">AtCoder Heuristic Contest 044</a></td>
|
||||
<td>05:00</td>
|
||||
<td>-</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>2025-01-13 21:00:00+0900</td>
|
||||
<td><a href="/contests/arc170">AtCoder Regular Contest 170</a></td>
|
||||
<td>02:00</td>
|
||||
<td>1000 - 2799</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
"""
|
||||
return mock_response
|
||||
else:
|
||||
mock_response = Mock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
mock_response.text = "<html></html>"
|
||||
return mock_response
|
||||
|
||||
mocker.patch("scrapers.atcoder.requests.get", side_effect=mock_get_side_effect)
|
||||
|
||||
result = scrape_contests()
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0] == ContestSummary(
|
||||
id="abc350",
|
||||
name="AtCoder Beginner Contest 350",
|
||||
display_name="AtCoder Beginner Contest 350",
|
||||
)
|
||||
assert result[1] == ContestSummary(
|
||||
id="arc170",
|
||||
name="AtCoder Regular Contest 170",
|
||||
display_name="AtCoder Regular Contest 170",
|
||||
)
|
||||
|
||||
# Ensure ahc044 is filtered out
|
||||
contest_ids = [contest.id for contest in result]
|
||||
assert "ahc044" not in contest_ids
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ from scrapers.base import BaseScraper
|
|||
from scrapers.models import ContestListResult, MetadataResult, TestsResult
|
||||
|
||||
SCRAPERS = [
|
||||
cls
|
||||
for name, cls in inspect.getmembers(scrapers, inspect.isclass)
|
||||
if issubclass(cls, BaseScraper) and cls != BaseScraper
|
||||
scrapers.AtCoderScraper,
|
||||
scrapers.CodeforcesScraper,
|
||||
scrapers.CSESScraper,
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue