barrett/cp.nvim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(scrapers): total refactor

Browse source
This commit is contained in:
Barrett Ruth 2025-09-22 22:00:20 -04:00
parent eb3f7762de
commit db391da52c
9 changed files with 559 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

23
scrapers/base.py
View file

@ -1,8 +1,5 @@
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Protocol
import requests
from .models import ContestListResult, MetadataResult, TestsResult
@ -15,23 +12,14 @@ class ScraperConfig:
rate_limit_delay: float = 1.0
class HttpClient(Protocol):
def get(self, url: str, **kwargs) -> requests.Response: ...
def close(self) -> None: ...
class BaseScraper(ABC):
def __init__(self, config: ScraperConfig | None = None):
self.config = config or ScraperConfig()
self._client: HttpClient | None = None
@property
@abstractmethod
def platform_name(self) -> str: ...
@abstractmethod
def _create_client(self) -> HttpClient: ...
@abstractmethod
def scrape_contest_metadata(self, contest_id: str) -> MetadataResult: ...
@ -41,17 +29,6 @@ class BaseScraper(ABC):
@abstractmethod
def scrape_contest_list(self) -> ContestListResult: ...
@property
def client(self) -> HttpClient:
if self._client is None:
self._client = self._create_client()
return self._client
def close(self) -> None:
if self._client is not None:
self._client.close()
self._client = None
def _create_metadata_error(
self, error_msg: str, contest_id: str = ""
) -> MetadataResult: