feat(scraper): update scraper i/o for single test panel
This commit is contained in:
parent
24624f9826
commit
b2e1ea2c58
3 changed files with 108 additions and 39 deletions
|
|
@ -169,21 +169,16 @@ def main() -> None:
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
individual_test_cases: list[dict[str, str]] = []
|
test_list: list[dict[str, str]] = []
|
||||||
for index, (input_data, output_data) in enumerate(tests, 1):
|
for input_data, output_data in tests:
|
||||||
individual_test_cases.append(
|
normalized_input = "1\n" + input_data
|
||||||
{"index": index, "input": input_data, "output": output_data}
|
test_list.append({"input": normalized_input, "expected": output_data})
|
||||||
)
|
|
||||||
|
|
||||||
combined_input = "\n".join(tc["input"] for tc in individual_test_cases)
|
|
||||||
combined_output = "\n".join(tc["output"] for tc in individual_test_cases)
|
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"success": True,
|
"success": True,
|
||||||
"problem_id": problem_id,
|
"problem_id": problem_id,
|
||||||
"url": url,
|
"url": url,
|
||||||
"test_cases": individual_test_cases,
|
"tests": test_list,
|
||||||
"combined": {"input": combined_input, "output": combined_output},
|
|
||||||
}
|
}
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,42 +14,116 @@ def scrape(url: str) -> list[tuple[str, str]]:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
soup = BeautifulSoup(response.text, "html.parser")
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
tests: list[tuple[str, str]] = []
|
|
||||||
|
|
||||||
input_sections = soup.find_all("div", class_="input")
|
input_sections = soup.find_all("div", class_="input")
|
||||||
output_sections = soup.find_all("div", class_="output")
|
output_sections = soup.find_all("div", class_="output")
|
||||||
|
|
||||||
|
individual_inputs = {}
|
||||||
|
individual_outputs = {}
|
||||||
|
|
||||||
|
for inp_section in input_sections:
|
||||||
|
inp_pre = inp_section.find("pre")
|
||||||
|
if not inp_pre:
|
||||||
|
continue
|
||||||
|
|
||||||
|
test_line_divs = inp_pre.find_all(
|
||||||
|
"div", class_=lambda x: x and "test-example-line-" in x
|
||||||
|
)
|
||||||
|
if not test_line_divs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for div in test_line_divs:
|
||||||
|
class_name = next(
|
||||||
|
(
|
||||||
|
cls
|
||||||
|
for cls in div.get("class", [])
|
||||||
|
if "test-example-line-" in cls
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not class_name:
|
||||||
|
continue
|
||||||
|
|
||||||
|
test_num = class_name.replace("test-example-line-", "")
|
||||||
|
if test_num not in individual_inputs:
|
||||||
|
individual_inputs[test_num] = []
|
||||||
|
individual_inputs[test_num].append(div.get_text().strip())
|
||||||
|
|
||||||
|
for out_section in output_sections:
|
||||||
|
out_pre = out_section.find("pre")
|
||||||
|
if not out_pre:
|
||||||
|
continue
|
||||||
|
|
||||||
|
test_line_divs = out_pre.find_all(
|
||||||
|
"div", class_=lambda x: x and "test-example-line-" in x
|
||||||
|
)
|
||||||
|
if not test_line_divs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for div in test_line_divs:
|
||||||
|
class_name = next(
|
||||||
|
(
|
||||||
|
cls
|
||||||
|
for cls in div.get("class", [])
|
||||||
|
if "test-example-line-" in cls
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not class_name:
|
||||||
|
continue
|
||||||
|
|
||||||
|
test_num = class_name.replace("test-example-line-", "")
|
||||||
|
if test_num not in individual_outputs:
|
||||||
|
individual_outputs[test_num] = []
|
||||||
|
individual_outputs[test_num].append(div.get_text().strip())
|
||||||
|
|
||||||
|
if individual_inputs and individual_outputs:
|
||||||
|
common_tests = set(individual_inputs.keys()) & set(
|
||||||
|
individual_outputs.keys()
|
||||||
|
)
|
||||||
|
if common_tests:
|
||||||
|
tests = []
|
||||||
|
for test_num in sorted(common_tests):
|
||||||
|
input_text = "\n".join(individual_inputs[test_num])
|
||||||
|
output_text = "\n".join(individual_outputs[test_num])
|
||||||
|
prefixed_input = "1\n" + input_text
|
||||||
|
tests.append((prefixed_input, output_text))
|
||||||
|
return tests
|
||||||
|
|
||||||
all_inputs = []
|
all_inputs = []
|
||||||
all_outputs = []
|
all_outputs = []
|
||||||
|
|
||||||
for inp_section in input_sections:
|
for inp_section in input_sections:
|
||||||
inp_pre = inp_section.find("pre")
|
inp_pre = inp_section.find("pre")
|
||||||
if inp_pre:
|
if not inp_pre:
|
||||||
divs = inp_pre.find_all("div")
|
continue
|
||||||
if divs:
|
|
||||||
lines = [div.get_text().strip() for div in divs]
|
divs = inp_pre.find_all("div")
|
||||||
text = "\n".join(lines)
|
if divs:
|
||||||
else:
|
lines = [div.get_text().strip() for div in divs]
|
||||||
text = inp_pre.get_text().replace("\r", "").strip()
|
text = "\n".join(lines)
|
||||||
all_inputs.append(text)
|
else:
|
||||||
|
text = inp_pre.get_text().replace("\r", "").strip()
|
||||||
|
all_inputs.append(text)
|
||||||
|
|
||||||
for out_section in output_sections:
|
for out_section in output_sections:
|
||||||
out_pre = out_section.find("pre")
|
out_pre = out_section.find("pre")
|
||||||
if out_pre:
|
if not out_pre:
|
||||||
divs = out_pre.find_all("div")
|
continue
|
||||||
if divs:
|
|
||||||
lines = [div.get_text().strip() for div in divs]
|
|
||||||
text = "\n".join(lines)
|
|
||||||
else:
|
|
||||||
text = out_pre.get_text().replace("\r", "").strip()
|
|
||||||
all_outputs.append(text)
|
|
||||||
|
|
||||||
if all_inputs and all_outputs:
|
divs = out_pre.find_all("div")
|
||||||
combined_input = "\n".join(all_inputs)
|
if divs:
|
||||||
combined_output = "\n".join(all_outputs)
|
lines = [div.get_text().strip() for div in divs]
|
||||||
tests.append((combined_input, combined_output))
|
text = "\n".join(lines)
|
||||||
|
else:
|
||||||
|
text = out_pre.get_text().replace("\r", "").strip()
|
||||||
|
all_outputs.append(text)
|
||||||
|
|
||||||
return tests
|
if not all_inputs or not all_outputs:
|
||||||
|
return []
|
||||||
|
|
||||||
|
combined_input = "\n".join(all_inputs)
|
||||||
|
combined_output = "\n".join(all_outputs)
|
||||||
|
return [(combined_input, combined_output)]
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"CloudScraper failed: {e}", file=sys.stderr)
|
print(f"CloudScraper failed: {e}", file=sys.stderr)
|
||||||
|
|
@ -170,15 +244,15 @@ def main() -> None:
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
test_cases: list[dict[str, str]] = []
|
test_list: list[dict[str, str]] = []
|
||||||
for input_data, output_data in tests:
|
for input_data, output_data in tests:
|
||||||
test_cases.append({"input": input_data, "output": output_data})
|
test_list.append({"input": input_data, "expected": output_data})
|
||||||
|
|
||||||
result: dict[str, str | bool | list] = {
|
result: dict[str, str | bool | list] = {
|
||||||
"success": True,
|
"success": True,
|
||||||
"problem_id": problem_id,
|
"problem_id": problem_id,
|
||||||
"url": url,
|
"url": url,
|
||||||
"test_cases": test_cases,
|
"tests": test_list,
|
||||||
}
|
}
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -174,15 +174,15 @@ def main() -> None:
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
test_cases: list[dict[str, str]] = []
|
test_list: list[dict[str, str]] = []
|
||||||
for input_data, output_data in tests:
|
for input_data, output_data in tests:
|
||||||
test_cases.append({"input": input_data, "output": output_data})
|
test_list.append({"input": input_data, "expected": output_data})
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"success": True,
|
"success": True,
|
||||||
"problem_id": problem_id,
|
"problem_id": problem_id,
|
||||||
"url": url,
|
"url": url,
|
||||||
"test_cases": test_cases,
|
"tests": test_list,
|
||||||
}
|
}
|
||||||
print(json.dumps(result))
|
print(json.dumps(result))
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue