feat(scraper): update scraper i/o for single test panel

This commit is contained in:
Barrett Ruth 2025-09-18 09:56:54 -04:00
parent 24624f9826
commit b2e1ea2c58
3 changed files with 108 additions and 39 deletions

View file

@ -169,21 +169,16 @@ def main() -> None:
print(json.dumps(result)) print(json.dumps(result))
sys.exit(1) sys.exit(1)
individual_test_cases: list[dict[str, str]] = [] test_list: list[dict[str, str]] = []
for index, (input_data, output_data) in enumerate(tests, 1): for input_data, output_data in tests:
individual_test_cases.append( normalized_input = "1\n" + input_data
{"index": index, "input": input_data, "output": output_data} test_list.append({"input": normalized_input, "expected": output_data})
)
combined_input = "\n".join(tc["input"] for tc in individual_test_cases)
combined_output = "\n".join(tc["output"] for tc in individual_test_cases)
result = { result = {
"success": True, "success": True,
"problem_id": problem_id, "problem_id": problem_id,
"url": url, "url": url,
"test_cases": individual_test_cases, "tests": test_list,
"combined": {"input": combined_input, "output": combined_output},
} }
print(json.dumps(result)) print(json.dumps(result))

View file

@ -14,42 +14,116 @@ def scrape(url: str) -> list[tuple[str, str]]:
response.raise_for_status() response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser") soup = BeautifulSoup(response.text, "html.parser")
tests: list[tuple[str, str]] = []
input_sections = soup.find_all("div", class_="input") input_sections = soup.find_all("div", class_="input")
output_sections = soup.find_all("div", class_="output") output_sections = soup.find_all("div", class_="output")
individual_inputs = {}
individual_outputs = {}
for inp_section in input_sections:
inp_pre = inp_section.find("pre")
if not inp_pre:
continue
test_line_divs = inp_pre.find_all(
"div", class_=lambda x: x and "test-example-line-" in x
)
if not test_line_divs:
continue
for div in test_line_divs:
class_name = next(
(
cls
for cls in div.get("class", [])
if "test-example-line-" in cls
),
None,
)
if not class_name:
continue
test_num = class_name.replace("test-example-line-", "")
if test_num not in individual_inputs:
individual_inputs[test_num] = []
individual_inputs[test_num].append(div.get_text().strip())
for out_section in output_sections:
out_pre = out_section.find("pre")
if not out_pre:
continue
test_line_divs = out_pre.find_all(
"div", class_=lambda x: x and "test-example-line-" in x
)
if not test_line_divs:
continue
for div in test_line_divs:
class_name = next(
(
cls
for cls in div.get("class", [])
if "test-example-line-" in cls
),
None,
)
if not class_name:
continue
test_num = class_name.replace("test-example-line-", "")
if test_num not in individual_outputs:
individual_outputs[test_num] = []
individual_outputs[test_num].append(div.get_text().strip())
if individual_inputs and individual_outputs:
common_tests = set(individual_inputs.keys()) & set(
individual_outputs.keys()
)
if common_tests:
tests = []
for test_num in sorted(common_tests):
input_text = "\n".join(individual_inputs[test_num])
output_text = "\n".join(individual_outputs[test_num])
prefixed_input = "1\n" + input_text
tests.append((prefixed_input, output_text))
return tests
all_inputs = [] all_inputs = []
all_outputs = [] all_outputs = []
for inp_section in input_sections: for inp_section in input_sections:
inp_pre = inp_section.find("pre") inp_pre = inp_section.find("pre")
if inp_pre: if not inp_pre:
divs = inp_pre.find_all("div") continue
if divs:
lines = [div.get_text().strip() for div in divs] divs = inp_pre.find_all("div")
text = "\n".join(lines) if divs:
else: lines = [div.get_text().strip() for div in divs]
text = inp_pre.get_text().replace("\r", "").strip() text = "\n".join(lines)
all_inputs.append(text) else:
text = inp_pre.get_text().replace("\r", "").strip()
all_inputs.append(text)
for out_section in output_sections: for out_section in output_sections:
out_pre = out_section.find("pre") out_pre = out_section.find("pre")
if out_pre: if not out_pre:
divs = out_pre.find_all("div") continue
if divs:
lines = [div.get_text().strip() for div in divs]
text = "\n".join(lines)
else:
text = out_pre.get_text().replace("\r", "").strip()
all_outputs.append(text)
if all_inputs and all_outputs: divs = out_pre.find_all("div")
combined_input = "\n".join(all_inputs) if divs:
combined_output = "\n".join(all_outputs) lines = [div.get_text().strip() for div in divs]
tests.append((combined_input, combined_output)) text = "\n".join(lines)
else:
text = out_pre.get_text().replace("\r", "").strip()
all_outputs.append(text)
return tests if not all_inputs or not all_outputs:
return []
combined_input = "\n".join(all_inputs)
combined_output = "\n".join(all_outputs)
return [(combined_input, combined_output)]
except Exception as e: except Exception as e:
print(f"CloudScraper failed: {e}", file=sys.stderr) print(f"CloudScraper failed: {e}", file=sys.stderr)
@ -170,15 +244,15 @@ def main() -> None:
print(json.dumps(result)) print(json.dumps(result))
sys.exit(1) sys.exit(1)
test_cases: list[dict[str, str]] = [] test_list: list[dict[str, str]] = []
for input_data, output_data in tests: for input_data, output_data in tests:
test_cases.append({"input": input_data, "output": output_data}) test_list.append({"input": input_data, "expected": output_data})
result: dict[str, str | bool | list] = { result: dict[str, str | bool | list] = {
"success": True, "success": True,
"problem_id": problem_id, "problem_id": problem_id,
"url": url, "url": url,
"test_cases": test_cases, "tests": test_list,
} }
print(json.dumps(result)) print(json.dumps(result))

View file

@ -174,15 +174,15 @@ def main() -> None:
print(json.dumps(result)) print(json.dumps(result))
sys.exit(1) sys.exit(1)
test_cases: list[dict[str, str]] = [] test_list: list[dict[str, str]] = []
for input_data, output_data in tests: for input_data, output_data in tests:
test_cases.append({"input": input_data, "output": output_data}) test_list.append({"input": input_data, "expected": output_data})
result = { result = {
"success": True, "success": True,
"problem_id": problem_id, "problem_id": problem_id,
"url": url, "url": url,
"test_cases": test_cases, "tests": test_list,
} }
print(json.dumps(result)) print(json.dumps(result))