feat(tests): basic tests
This commit is contained in:
parent
a7eb731730
commit
c509102b37
22 changed files with 17879 additions and 93 deletions
|
|
@ -5,7 +5,6 @@ import json
|
|||
import re
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
import backoff
|
||||
|
|
@ -231,16 +230,12 @@ def _scrape_problem_page_sync(contest_id: str, slug: str) -> dict[str, Any]:
|
|||
|
||||
def _to_problem_summaries(rows: list[dict[str, str]]) -> list[ProblemSummary]:
|
||||
out: list[ProblemSummary] = []
|
||||
seen: set[str] = set()
|
||||
for r in rows:
|
||||
letter = (r.get("letter") or "").strip().upper()
|
||||
title = r.get("title") or ""
|
||||
if not letter:
|
||||
continue
|
||||
pid = letter.lower()
|
||||
if pid in seen:
|
||||
continue
|
||||
seen.add(pid)
|
||||
out.append(ProblemSummary(id=pid, name=title))
|
||||
return out
|
||||
|
||||
|
|
@ -341,7 +336,7 @@ async def main_async() -> int:
|
|||
success=False,
|
||||
error="Usage: atcoder.py metadata <contest_id> OR atcoder.py tests <contest_id> OR atcoder.py contests",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
mode: str = sys.argv[1]
|
||||
|
|
@ -352,11 +347,11 @@ async def main_async() -> int:
|
|||
result = MetadataResult(
|
||||
success=False, error="Usage: atcoder.py metadata <contest_id>"
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
contest_id = sys.argv[2]
|
||||
result = await scraper.scrape_contest_metadata(contest_id)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 0 if result.success else 1
|
||||
|
||||
if mode == "tests":
|
||||
|
|
@ -370,7 +365,7 @@ async def main_async() -> int:
|
|||
timeout_ms=0,
|
||||
memory_mb=0,
|
||||
)
|
||||
print(json.dumps(asdict(tests_result)))
|
||||
print(tests_result.model_dump_json())
|
||||
return 1
|
||||
contest_id = sys.argv[2]
|
||||
await scraper.stream_tests_for_category_async(contest_id)
|
||||
|
|
@ -381,17 +376,17 @@ async def main_async() -> int:
|
|||
contest_result = ContestListResult(
|
||||
success=False, error="Usage: atcoder.py contests"
|
||||
)
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 1
|
||||
contest_result = await scraper.scrape_contest_list()
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 0 if contest_result.success else 1
|
||||
|
||||
result = MetadataResult(
|
||||
success=False,
|
||||
error="Unknown mode. Use 'metadata <contest_id>', 'tests <contest_id>', or 'contests'",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, ParamSpec, cast
|
||||
|
||||
from .models import ContestListResult, MetadataResult, TestsResult
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScraperConfig:
|
||||
timeout_seconds: int = 30
|
||||
max_retries: int = 3
|
||||
backoff_base: float = 2.0
|
||||
rate_limit_delay: float = 1.0
|
||||
from .models import ContestListResult, MetadataResult, TestsResult
|
||||
|
||||
|
||||
class BaseScraper(ABC):
|
||||
|
|
@ -38,6 +27,7 @@ class BaseScraper(ABC):
|
|||
success=False,
|
||||
error=f"{self.platform_name}: {error_msg}",
|
||||
contest_id=contest_id,
|
||||
problems=[],
|
||||
)
|
||||
|
||||
def _create_tests_error(
|
||||
|
|
@ -51,11 +41,14 @@ class BaseScraper(ABC):
|
|||
tests=[],
|
||||
timeout_ms=0,
|
||||
memory_mb=0,
|
||||
interactive=False,
|
||||
)
|
||||
|
||||
def _create_contests_error(self, error_msg: str) -> ContestListResult:
|
||||
return ContestListResult(
|
||||
success=False, error=f"{self.platform_name}: {error_msg}"
|
||||
success=False,
|
||||
error=f"{self.platform_name}: {error_msg}",
|
||||
contests=[],
|
||||
)
|
||||
|
||||
async def _safe_execute(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import json
|
|||
import logging
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
|
@ -63,8 +62,6 @@ def _extract_limits(block: Tag) -> tuple[int, float]:
|
|||
|
||||
def _group_lines_by_id(pre: Tag) -> dict[int, list[str]]:
|
||||
groups: dict[int, list[str]] = {}
|
||||
if not isinstance(pre, Tag):
|
||||
return groups
|
||||
for div in pre.find_all("div", class_="test-example-line"):
|
||||
cls = " ".join(div.get("class", []))
|
||||
m = re.search(r"\btest-example-line-(\d+)\b", cls)
|
||||
|
|
@ -182,12 +179,8 @@ def _scrape_contest_problems_sync(contest_id: str) -> list[ProblemSummary]:
|
|||
html = _fetch_problems_html(contest_id)
|
||||
blocks = _parse_all_blocks(html)
|
||||
problems: list[ProblemSummary] = []
|
||||
seen: set[str] = set()
|
||||
for b in blocks:
|
||||
pid = b["letter"].upper()
|
||||
if pid in seen:
|
||||
continue
|
||||
seen.add(pid)
|
||||
problems.append(ProblemSummary(id=pid.lower(), name=b["name"]))
|
||||
return problems
|
||||
|
||||
|
|
@ -267,7 +260,7 @@ async def main_async() -> int:
|
|||
success=False,
|
||||
error="Usage: codeforces.py metadata <contest_id> OR codeforces.py tests <contest_id> OR codeforces.py contests",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
mode: str = sys.argv[1]
|
||||
|
|
@ -278,11 +271,11 @@ async def main_async() -> int:
|
|||
result = MetadataResult(
|
||||
success=False, error="Usage: codeforces.py metadata <contest_id>"
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
contest_id = sys.argv[2]
|
||||
result = await scraper.scrape_contest_metadata(contest_id)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 0 if result.success else 1
|
||||
|
||||
if mode == "tests":
|
||||
|
|
@ -296,7 +289,7 @@ async def main_async() -> int:
|
|||
timeout_ms=0,
|
||||
memory_mb=0,
|
||||
)
|
||||
print(json.dumps(asdict(tests_result)))
|
||||
print(tests_result.model_dump_json())
|
||||
return 1
|
||||
contest_id = sys.argv[2]
|
||||
await scraper.stream_tests_for_category_async(contest_id)
|
||||
|
|
@ -307,17 +300,17 @@ async def main_async() -> int:
|
|||
contest_result = ContestListResult(
|
||||
success=False, error="Usage: codeforces.py contests"
|
||||
)
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 1
|
||||
contest_result = await scraper.scrape_contest_list()
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 0 if contest_result.success else 1
|
||||
|
||||
result = MetadataResult(
|
||||
success=False,
|
||||
error="Unknown mode. Use 'metadata <contest_id>', 'tests <contest_id>', or 'contests'",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import asyncio
|
|||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
|
@ -251,7 +250,7 @@ async def main_async() -> int:
|
|||
success=False,
|
||||
error="Usage: cses.py metadata <category_id> OR cses.py tests <category> OR cses.py contests",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
mode: str = sys.argv[1]
|
||||
|
|
@ -262,11 +261,11 @@ async def main_async() -> int:
|
|||
result = MetadataResult(
|
||||
success=False, error="Usage: cses.py metadata <category_id>"
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
category_id = sys.argv[2]
|
||||
result = await scraper.scrape_contest_metadata(category_id)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 0 if result.success else 1
|
||||
|
||||
if mode == "tests":
|
||||
|
|
@ -280,7 +279,7 @@ async def main_async() -> int:
|
|||
timeout_ms=0,
|
||||
memory_mb=0,
|
||||
)
|
||||
print(json.dumps(asdict(tests_result)))
|
||||
print(tests_result.model_dump_json())
|
||||
return 1
|
||||
category = sys.argv[2]
|
||||
await scraper.stream_tests_for_category_async(category)
|
||||
|
|
@ -291,17 +290,17 @@ async def main_async() -> int:
|
|||
contest_result = ContestListResult(
|
||||
success=False, error="Usage: cses.py contests"
|
||||
)
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 1
|
||||
contest_result = await scraper.scrape_contest_list()
|
||||
print(json.dumps(asdict(contest_result)))
|
||||
print(contest_result.model_dump_json())
|
||||
return 0 if contest_result.success else 1
|
||||
|
||||
result = MetadataResult(
|
||||
success=False,
|
||||
error=f"Unknown mode: {mode}. Use 'metadata <category>', 'tests <category>', or 'contests'",
|
||||
)
|
||||
print(json.dumps(asdict(result)))
|
||||
print(result.model_dump_json())
|
||||
return 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,47 +1,71 @@
|
|||
from dataclasses import dataclass, field
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestCase:
|
||||
class TestCase(BaseModel):
|
||||
input: str
|
||||
expected: str
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
@dataclass
|
||||
class ProblemSummary:
|
||||
|
||||
class ProblemSummary(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
@dataclass
|
||||
class ContestSummary:
|
||||
|
||||
class ContestSummary(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
display_name: str
|
||||
display_name: str | None = None
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScrapingResult:
|
||||
class ScrapingResult(BaseModel):
|
||||
success: bool
|
||||
error: str
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetadataResult(ScrapingResult):
|
||||
contest_id: str = ""
|
||||
problems: list[ProblemSummary] = field(default_factory=list)
|
||||
problems: list[ProblemSummary] = Field(default_factory=list)
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContestListResult(ScrapingResult):
|
||||
contests: list[ContestSummary] = field(default_factory=list)
|
||||
contests: list[ContestSummary] = Field(default_factory=list)
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestsResult(ScrapingResult):
|
||||
problem_id: str
|
||||
url: str
|
||||
tests: list[TestCase]
|
||||
tests: list[TestCase] = Field(default_factory=list)
|
||||
timeout_ms: int
|
||||
memory_mb: float
|
||||
interactive: bool = False
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
class ScraperConfig(BaseModel):
|
||||
timeout_seconds: int = 30
|
||||
max_retries: int = 3
|
||||
backoff_base: float = 2.0
|
||||
rate_limit_delay: float = 1.0
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue