oracle: quantities of a generated mesh can be gated by their tolerance alone ("fixture": false)

The bracket's tet mesh is not bit-identical across platforms (#115): its fillet peak is 213.31 MPa
on macOS and 212.42 on Linux CI, which turned the oracle job red after #110. Those two quantities
are no longer frozen; the page names them mesh-dependent instead of printing one platform's value.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

crates/testkit/src/oracle.rs

@@ -94,6 +94,10 @@ pub struct Quantity {
     /// The fixture band's relative part, when the quantity is more sensitive to rounding than
     /// [`FIXTURE_RELATIVE`] allows (an ill-conditioned problem); `fixture_reason` says why.
     pub fixture_relative: f64,
+    /// Whether the result is frozen in `fixture.json`. A quantity that depends on a mesh the
+    /// mesher generates from geometry (`"fixture": false`, with a `fixture_reason`) is not: the
+    /// mesh is not bit-identical across platforms, so only its reference tolerance gates it.
+    pub frozen: bool,
 }
 
 /// One case's `expected.json`.
@@ -153,6 +157,17 @@ impl Expected {
             if quantities.iter().any(|x: &Quantity| x.id == id) {
                 return Err(format!("quantity `{id}` is listed twice"));
             }
+            let frozen = match q.get("fixture") {
+                None => true,
+                Some(Value::Bool(f)) => *f,
+                Some(_) => return Err(context("`fixture` must be true or false".into())),
+            };
+            if !frozen {
+                text(q, "fixture_reason").map_err(|_| context("`\"fixture\": false` needs a `fixture_reason`".into()))?;
+                if q.get("fixture_relative").is_some() {
+                    return Err(context("`fixture_relative` has no meaning with `\"fixture\": false`".into()));
+                }
+            }
             let fixture_relative = match q.get("fixture_relative") {
                 None => FIXTURE_RELATIVE,
                 Some(r) => {
@@ -163,6 +178,7 @@ impl Expected {
             };
             quantities.push(Quantity {
                 fixture_relative,
+                frozen,
                 description: text(q, "quantity").map_err(context)?,
                 known_failure: q.get("known_failure").and_then(Value::as_str).map(str::to_string),
                 id,
@@ -319,6 +335,7 @@ pub fn judge(q: &Quantity, ours: &Result<f64, String>, frozen: Option<&Frozen>)
         (false, Some(_)) => Verdict::Known,
     };
     let fixture = match frozen {
+        _ if !q.frozen => FixtureCheck::NotKept,
         Some(f) if (ours - f.value).abs() <= f.band => FixtureCheck::Matches,
         Some(f) => FixtureCheck::Moved { frozen: f.value },
         None if verdict == Verdict::Pass => FixtureCheck::Missing,
@@ -392,7 +409,7 @@ pub fn run(root: &Path, options: &Options) -> Result<(), String> {
                 problems.push(format!("{case}: not blessed, it does not pass against expected.json"));
             } else {
                 let mut values = serde_json::Map::new();
-                for r in case_rows.iter_mut().filter(|r| r.verdict == Verdict::Pass) {
+                for r in case_rows.iter_mut().filter(|r| r.verdict == Verdict::Pass && r.quantity.frozen) {
                     if let (Some(ours), Some(reference)) = (r.ours, r.quantity.reference) {
                         values.insert(
                             r.quantity.id.clone(),
@@ -513,6 +530,18 @@ fn ours_text(r: &Row) -> String {
     }
 }
 
+/// docs/oracle.md's result column. A quantity that is not frozen depends on a generated mesh
+/// that differs by platform, so the page names that instead of one platform's number (the run's
+/// console prints the value).
+fn page_ours(r: &Row) -> String {
+    if r.quantity.frozen { ours_text(r) } else { "mesh-dependent".into() }
+}
+
+/// docs/oracle.md's error column; see [`page_ours`].
+fn page_error(r: &Row) -> String {
+    if r.quantity.frozen { error_of(r) } else { format!("within {}", r.quantity.tolerance.describe()) }
+}
+
 /// The error against the reference: relative where the reference is not zero. Errors inside the
 /// fixture band are rounding and are shown as a bound.
 fn error_of(r: &Row) -> String {
@@ -615,8 +644,8 @@ benchmark, {} experiment; {} pending cases wait for capabilities still being bui
                     "| [{case}](../tests/oracle/{case}/) | {} | {} | {} | {} | {} | {} | {} |",
                     cell(&r.quantity.description),
                     show(r.quantity.reference),
-                    ours_text(r),
-                    error_of(r),
+                    page_ours(r),
+                    page_error(r),
                     r.quantity.tolerance.describe(),
                     cell(&result),
                     page_fixture_text(&r.fixture),
@@ -668,9 +697,21 @@ mod tests {
             reason: "r".into(),
             known_failure: known.map(str::to_string),
             fixture_relative: FIXTURE_RELATIVE,
+            frozen: true,
         }
     }
 
+    /// A quantity of a generated mesh is gated by its tolerance only: a stored fixture is ignored
+    /// and blessing skips it.
+    #[test]
+    fn unfrozen_quantities_skip_the_fixture() {
+        let t = Tolerance { relative: 0.03, absolute: 0.0 };
+        let q = Quantity { frozen: false, ..quantity(213.0, t, None) };
+        let old = Frozen { value: 213.3, band: 2e-7 };
+        assert_eq!(judge(&q, &Ok(212.4), Some(&old)), (Verdict::Pass, FixtureCheck::NotKept));
+        assert_eq!(judge(&q, &Ok(200.0), Some(&old)).0, Verdict::Fail, "the tolerance still gates it");
+    }
+
     #[test]
     fn verdicts_and_fixtures() {
         let t = Tolerance { relative: 0.01, absolute: 0.0 };

docs/oracle.md

@@ -28,9 +28,9 @@ Generated by `cargo xtask oracle` from [`tests/oracle/`](../tests/oracle/); the
 | [cdt-stepped-block](../tests/oracle/cdt-stepped-block/) | volume of the tetrahedra | 47000 | 47000 | < 1e-7% | 1e-12 rel. | pass | matches |
 | [cdt-stepped-block](../tests/oracle/cdt-stepped-block/) | elements that fail the certified Jacobian check | 0 | 0 | ≈ 0 | exact | pass | matches |
 | [cdt-stepped-block](../tests/oracle/cdt-stepped-block/) | elements the Jacobian check could not decide | 0 | 0 | ≈ 0 | exact | pass | matches |
-| [filleted-bracket-fillet-stress](../tests/oracle/filleted-bracket-fillet-stress/) | largest nodal von Mises stress on the fillet face (MPa) | 213 | 213.31 | 0.144% | 3% | pass | matches |
+| [filleted-bracket-fillet-stress](../tests/oracle/filleted-bracket-fillet-stress/) | largest nodal von Mises stress on the fillet face (MPa) | 213 | mesh-dependent | within 3% | 3% | pass | — |
 | [filleted-bracket-fillet-stress](../tests/oracle/filleted-bracket-fillet-stress/) | total reaction x (N) | -2000 | -2000 | < 1e-7% | 1e-9 rel. | pass | matches |
-| [filleted-bracket-fillet-stress-coarse](../tests/oracle/filleted-bracket-fillet-stress-coarse/) | largest nodal von Mises stress on the fillet face (MPa) | 213 | 213.31 | 0.144% | 3% | pass | matches |
+| [filleted-bracket-fillet-stress-coarse](../tests/oracle/filleted-bracket-fillet-stress-coarse/) | largest nodal von Mises stress on the fillet face (MPa) | 213 | mesh-dependent | within 3% | 3% | pass | — |
 | [filleted-bracket-fillet-stress-coarse](../tests/oracle/filleted-bracket-fillet-stress-coarse/) | total reaction x (N) | -2000 | -2000 | < 1e-7% | 1e-9 rel. | pass | matches |
 | [free-beam-frequencies](../tests/oracle/free-beam-frequencies/) | sixth frequency (the last rigid-body mode) | 0 | ≈ 0 | ≈ 0 | ± 8.1264e-5 | pass | matches |
 | [free-beam-frequencies](../tests/oracle/free-beam-frequencies/) | first free–free bending frequency (mode 7) | 0.081264 | 0.080618 | -0.795% | 1.5% | pass | matches |

docs/verification.md

@@ -65,7 +65,9 @@ cargo xtask oracle --quick              # skip cases marked "slow"
 - `fixture.json` holds each passing quantity's accepted `value` and its `band`
   (10⁻⁹ of the larger of value and reference, plus 10⁻³ of an absolute tolerance; a quantity whose
   reference is zero, which is rounding noise, keeps its absolute tolerance as its band; an
-  ill-conditioned quantity may set a wider `fixture_relative` with a `fixture_reason`): the result is
+  ill-conditioned quantity may set a wider `fixture_relative` with a `fixture_reason`; a quantity
+  that depends on a mesh generated from geometry, which is not bit-identical across platforms, sets
+  `"fixture": false` with a `fixture_reason` and is gated by its reference tolerance alone): the result is
   deterministic, the band only absorbs last-digit differences of the system maths library.
 
 The run fails when a quantity misses its reference (unless it is a `known_failure`, which is

tests/oracle/filleted-bracket-fillet-stress-coarse/expected.json

@@ -19,7 +19,9 @@
    "tolerance": {
     "relative": 0.03
    },
-   "reason": "the four-level study gives 213.3, 210.6, 214.3, 213.3 MPa (2k to 13.5k elements): the bending stress is converged across the fillet, and the peak nodal value moves only with which of the fillet's tessellation lines (fixed angles, through every level) carries the largest node, by under 2%"
+   "reason": "the four-level study gives 213.3, 210.6, 214.3, 213.3 MPa (2k to 13.5k elements): the bending stress is converged across the fillet, and the peak nodal value moves only with which of the fillet's tessellation lines (fixed angles, through every level) carries the largest node, by under 2%",
+   "fixture": false,
+   "fixture_reason": "the peak depends on the tet mesh generated from SolidCraft's tessellation, which is not bit-identical across platforms (212.4 MPa on Linux CI against 213.3 on macOS for the same recipe; issue #115): the 3% reference tolerance gates it until meshing is reproducible across platforms"
   },
   {
    "id": "reaction",

tests/oracle/filleted-bracket-fillet-stress-coarse/fixture.json

@@ -1,12 +1,8 @@
 {
   "note": "FieldCraft's accepted results for this case, written by `cargo xtask oracle --bless`. A result outside its band fails CI.",
   "values": {
-    "fillet_peak": {
-      "band": 2.1330697455903942e-7,
-      "value": 213.30697455903942
-    },
     "reaction": {
-      "band": 2.000000000000858e-6,
+      "band": 2.000000000000858e-06,
       "value": -2000.0000000008579
     }
   }

tests/oracle/filleted-bracket-fillet-stress/expected.json

@@ -19,7 +19,9 @@
    "tolerance": {
     "relative": 0.03
    },
-   "reason": "the four-level study gives 213.3, 210.6, 214.3, 213.3 MPa (2k to 13.5k elements): the bending stress is converged across the fillet, and the peak nodal value moves only with which of the fillet's tessellation lines (fixed angles, through every level) carries the largest node, by under 2%"
+   "reason": "the four-level study gives 213.3, 210.6, 214.3, 213.3 MPa (2k to 13.5k elements): the bending stress is converged across the fillet, and the peak nodal value moves only with which of the fillet's tessellation lines (fixed angles, through every level) carries the largest node, by under 2%",
+   "fixture": false,
+   "fixture_reason": "the peak depends on the tet mesh generated from SolidCraft's tessellation, which is not bit-identical across platforms (212.4 MPa on Linux CI against 213.3 on macOS for the same recipe; issue #115): the 3% reference tolerance gates it until meshing is reproducible across platforms"
   },
   {
    "id": "reaction",

tests/oracle/filleted-bracket-fillet-stress/fixture.json

@@ -1,12 +1,8 @@
 {
   "note": "FieldCraft's accepted results for this case, written by `cargo xtask oracle --bless`. A result outside its band fails CI.",
   "values": {
-    "fillet_peak": {
-      "band": 2.1330693756948646e-7,
-      "value": 213.30693756948645
-    },
     "reaction": {
-      "band": 2.0000000000001007e-6,
+      "band": 2.0000000000001007e-06,
       "value": -2000.0000000001007
     }
   }