From 67e05548b171702cf3082d808ebb44a2d3b52ae5 Mon Sep 17 00:00:00 2001 From: jyviko <2501648+jyviko@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:05:36 -0400 Subject: [PATCH] oracle: an ill-conditioned quantity gets a wider fixture band; Scordelis-Lo re-blessed Main's oracle job failed at 8cba3a7: scordelis-lo-roof-solid u_z moved from -0.3014029226397275 (blessed before #50) beyond its 1e-9 band. The case is a thin shell (R/t = 100) of one layer of hex20R solids, ill-conditioned, so rounding alone moves u_z by about 1e-9 relative: on macOS #50's multithreaded supernodal factorisation moved it by 8.2e-10 (to -0.3014029223929), #52's interpolation order by a further 1.3e-10 (to -0.3014029223541), and Linux CI by slightly more, past the band. #52 kept the lattice geometry the same. The result itself is unchanged at the 1e-9 level (0.33% from the reference). A quantity may now set `fixture_relative` (with a required `fixture_reason`) for a wider fixture band; Scordelis-Lo uses 1e-7 and is re-blessed. Moved-fixture messages print full precision. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm --- crates/testkit/src/oracle.rs | 30 +++++++++++++++---- docs/verification.md | 3 +- .../scordelis-lo-roof-solid/expected.json | 4 ++- .../scordelis-lo-roof-solid/fixture.json | 4 +-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/crates/testkit/src/oracle.rs b/crates/testkit/src/oracle.rs index 6fd0d5e..f3f3830 100644 --- a/crates/testkit/src/oracle.rs +++ b/crates/testkit/src/oracle.rs @@ -91,6 +91,9 @@ pub struct Quantity { pub tolerance: Tolerance, pub reason: String, pub known_failure: Option, + /// The fixture band's relative part, when the quantity is more sensitive to rounding than + /// [`FIXTURE_RELATIVE`] allows (an ill-conditioned problem); `fixture_reason` says why. + pub fixture_relative: f64, } /// One case's `expected.json`. @@ -150,7 +153,16 @@ impl Expected { if quantities.iter().any(|x: &Quantity| x.id == id) { return Err(format!("quantity `{id}` is listed twice")); } + let fixture_relative = match q.get("fixture_relative") { + None => FIXTURE_RELATIVE, + Some(r) => { + let r = r.as_f64().filter(|r| *r > 0.0 && *r <= 1e-3).ok_or_else(|| context("`fixture_relative` must be in (0, 1e-3]".into()))?; + text(q, "fixture_reason").map_err(|_| context("`fixture_relative` needs a `fixture_reason`".into()))?; + r + } + }; quantities.push(Quantity { + fixture_relative, description: text(q, "quantity").map_err(context)?, known_failure: q.get("known_failure").and_then(Value::as_str).map(str::to_string), id, @@ -181,11 +193,11 @@ pub struct Frozen { /// The band a fixture value gets when it is blessed. A quantity whose reference is zero (a /// rigid-body frequency, an equilibrium residual, an invalid-element count) is rounding noise /// inside its tolerance: its band is that tolerance, as freezing noise would only flag rounding. -pub fn fixture_band(value: f64, reference: f64, tolerance: Tolerance) -> f64 { +pub fn fixture_band(value: f64, reference: f64, tolerance: Tolerance, relative: f64) -> f64 { if reference == 0.0 { return tolerance.absolute; } - FIXTURE_RELATIVE * value.abs().max(reference.abs()) + 1e-3 * tolerance.absolute + relative * value.abs().max(reference.abs()) + 1e-3 * tolerance.absolute } fn read_json(path: &Path) -> Result { @@ -382,7 +394,10 @@ pub fn run(root: &Path, options: &Options) -> Result<(), String> { let mut values = serde_json::Map::new(); for r in case_rows.iter_mut().filter(|r| r.verdict == Verdict::Pass) { if let (Some(ours), Some(reference)) = (r.ours, r.quantity.reference) { - values.insert(r.quantity.id.clone(), json!({"value": ours, "band": fixture_band(ours, reference, r.quantity.tolerance)})); + values.insert( + r.quantity.id.clone(), + json!({"value": ours, "band": fixture_band(ours, reference, r.quantity.tolerance, r.quantity.fixture_relative)}), + ); r.fixture = FixtureCheck::Blessed; } } @@ -405,7 +420,9 @@ pub fn run(root: &Path, options: &Options) -> Result<(), String> { _ => {} } match &r.fixture { - FixtureCheck::Moved { frozen } => problems.push(format!("{id}: moved from the fixture {} to {}", number(*frozen), show(r.ours))), + FixtureCheck::Moved { frozen } => { + problems.push(format!("{id}: moved from the fixture {frozen:e} to {:e}", r.ours.unwrap_or(f64::NAN))) + } FixtureCheck::Missing => problems.push(format!("{id}: no fixture; bless the case (`cargo xtask oracle --bless {case}`)")), _ => {} } @@ -640,6 +657,7 @@ mod tests { tolerance, reason: "r".into(), known_failure: known.map(str::to_string), + fixture_relative: FIXTURE_RELATIVE, } } @@ -647,7 +665,7 @@ mod tests { fn verdicts_and_fixtures() { let t = Tolerance { relative: 0.01, absolute: 0.0 }; let q = quantity(2.0, t, None); - let frozen = Frozen { value: 2.01, band: fixture_band(2.01, 2.0, t) }; + let frozen = Frozen { value: 2.01, band: fixture_band(2.01, 2.0, t, FIXTURE_RELATIVE) }; assert_eq!(judge(&q, &Ok(2.01), Some(&frozen)), (Verdict::Pass, FixtureCheck::Matches)); assert_eq!(judge(&q, &Ok(2.011), Some(&frozen)), (Verdict::Pass, FixtureCheck::Moved { frozen: 2.01 })); assert_eq!(judge(&q, &Ok(2.01), None), (Verdict::Pass, FixtureCheck::Missing)); @@ -658,7 +676,7 @@ mod tests { // Zero references use the absolute tolerance, and the band keeps a floor from it. let a = Tolerance { relative: 0.0, absolute: 1e-9 }; assert!(a.allows(0.0, 5e-10) && !a.allows(0.0, 2e-9)); - assert_eq!(fixture_band(1e-16, 0.0, a), 1e-9, "a zero reference keeps its tolerance as the band"); + assert_eq!(fixture_band(1e-16, 0.0, a, FIXTURE_RELATIVE), 1e-9, "a zero reference keeps its tolerance as the band"); } #[test] diff --git a/docs/verification.md b/docs/verification.md index 1fcab8b..0e278fa 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -64,7 +64,8 @@ cargo xtask oracle --quick # skip cases marked "slow" case cites its reference values (or `null` until transcribed) and has no recipe yet. - `fixture.json` holds each passing quantity's accepted `value` and its `band` (10⁻⁹ of the larger of value and reference, plus 10⁻³ of an absolute tolerance; a quantity whose - reference is zero, which is rounding noise, keeps its absolute tolerance as its band): the result is + reference is zero, which is rounding noise, keeps its absolute tolerance as its band; an + ill-conditioned quantity may set a wider `fixture_relative` with a `fixture_reason`): the result is deterministic, the band only absorbs last-digit differences of the system maths library. The run fails when a quantity misses its reference (unless it is a `known_failure`, which is diff --git a/tests/oracle/scordelis-lo-roof-solid/expected.json b/tests/oracle/scordelis-lo-roof-solid/expected.json index 3cfc063..c54493d 100644 --- a/tests/oracle/scordelis-lo-roof-solid/expected.json +++ b/tests/oracle/scordelis-lo-roof-solid/expected.json @@ -24,7 +24,9 @@ "tolerance": { "relative": 0.02 }, - "reason": "MacNeal and Harder grade an element A within 2% of the theoretical value on this prescribed coarse mesh (p. 14); MacNeal and Harder normalise by 0.3024 (the deep-shell value quoted in their [5] is 0.3086), and a solid model of R/t = 100 differs from shell theory by O(t/R)" + "reason": "MacNeal and Harder grade an element A within 2% of the theoretical value on this prescribed coarse mesh (p. 14); MacNeal and Harder normalise by 0.3024 (the deep-shell value quoted in their [5] is 0.3086), and a solid model of R/t = 100 differs from shell theory by O(t/R)", + "fixture_relative": 1e-7, + "fixture_reason": "a thin shell (R/t = 100) of one layer of solids is ill-conditioned: rounding moves u_z by about 1e-9 relative between equivalent orderings of the arithmetic (8e-10 when the solver's factorisation changed in #50, 1e-10 from the interpolation order in #52, more between platforms); 1e-7 still catches any real change" } ] } diff --git a/tests/oracle/scordelis-lo-roof-solid/fixture.json b/tests/oracle/scordelis-lo-roof-solid/fixture.json index 3b1fbbf..beec7f1 100644 --- a/tests/oracle/scordelis-lo-roof-solid/fixture.json +++ b/tests/oracle/scordelis-lo-roof-solid/fixture.json @@ -2,8 +2,8 @@ "note": "ModelCraft's accepted results for this case, written by `cargo xtask oracle --bless`. A result outside its band fails CI.", "values": { "uz": { - "band": 3.024e-10, - "value": -0.3014029226397275 + "band": 3.024e-8, + "value": -0.30140292235414196 } } }