feat: add epsilon tolerance for floating-point output comparison

Problem: output comparison used exact string equality after whitespace
normalisation, causing correct solutions to fail on problems where
floating-point answers are accepted within a tolerance (e.g. 1e-6).

Solution: add an optional ui.panel.epsilon config value. When set,
actual and expected output are compared token-by-token: numeric tokens
are compared with math.abs(a - b) <= epsilon, non-numeric tokens fall
back to exact string equality. Per-problem epsilon can also be stored
in the cache and takes precedence over the global default.
This commit is contained in:
Barrett Ruth 2026-02-26 22:55:05 -05:00 committed by Barrett Ruth
parent 84d12758c2
commit e685a8089f
3 changed files with 87 additions and 3 deletions

View file

@ -19,6 +19,7 @@
---@class ProblemConstraints
---@field timeout_ms number
---@field memory_mb number
---@field epsilon number?
---@class PanelState
---@field test_cases RanTestCase[]
@ -56,7 +57,8 @@ local function load_constraints_from_cache(platform, contest_id, problem_id)
cache.load()
local timeout_ms, memory_mb = cache.get_constraints(platform, contest_id, problem_id)
if timeout_ms and memory_mb then
return { timeout_ms = timeout_ms, memory_mb = memory_mb }
local epsilon = cache.get_epsilon(platform, contest_id, problem_id)
return { timeout_ms = timeout_ms, memory_mb = memory_mb, epsilon = epsilon }
end
return nil
end
@ -99,6 +101,49 @@ local function build_command(cmd, substitutions)
return execute.build_command(cmd, substitutions)
end
local function compare_outputs(actual, expected, epsilon)
local norm_actual = normalize_lines(actual)
local norm_expected = normalize_lines(expected)
if epsilon == nil or epsilon == 0 then
return norm_actual == norm_expected
end
local actual_lines = vim.split(norm_actual, '\n', { plain = true })
local expected_lines = vim.split(norm_expected, '\n', { plain = true })
if #actual_lines ~= #expected_lines then
return false
end
for i = 1, #actual_lines do
local a_tokens = vim.split(actual_lines[i], '%s+', { plain = false, trimempty = true })
local e_tokens = vim.split(expected_lines[i], '%s+', { plain = false, trimempty = true })
if #a_tokens ~= #e_tokens then
return false
end
for j = 1, #a_tokens do
local a_tok, e_tok = a_tokens[j], e_tokens[j]
local a_num = tonumber(a_tok)
local e_num = tonumber(e_tok)
if a_num ~= nil and e_num ~= nil then
if math.abs(a_num - e_num) > epsilon then
return false
end
else
if a_tok ~= e_tok then
return false
end
end
end
end
return true
end
---@param test_case RanTestCase
---@param debug boolean?
---@param on_complete fun(result: { status: "pass"|"fail"|"tle"|"mle", actual: string, actual_highlights: Highlight[], error: string, stderr: string, time_ms: number, code: integer, ok: boolean, signal: string?, tled: boolean, mled: boolean, rss_mb: number })
@ -143,7 +188,9 @@ local function run_single_test_case(test_case, debug, on_complete)
end
local expected = test_case.expected or ''
local ok = normalize_lines(out) == normalize_lines(expected)
local epsilon = (panel_state.constraints and panel_state.constraints.epsilon)
or config.ui.panel.epsilon
local ok = compare_outputs(out, expected, epsilon)
local signal = r.signal
if not signal and r.code and r.code >= 128 then