forked from 1b-agent/fieldcraft-upstream
Verification oracle: tests/oracle cases, cargo xtask oracle, fixtures, docs/oracle.md, CI job
The oracle binary (modelcraft-testkit) runs each case's recipe through the command engine, measures its quantities by JSON pointers into step outputs or commands, compares them with the cited references and the frozen fixtures, and writes docs/oracle.md. 20 theory cases (patch tests of every solid element, Timoshenko cantilevers, Lame, Kirsch, beam frequencies, torsion and bending through couplings, explicit waves and impact, CDT invariants), 8 published benchmarks (MacNeal-Harder patch, straight and curved cantilevers, thick cylinder, Scordelis-Lo with solids, NAFEMS LE1, LE10, FV32) and 10 pending cases with citations. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm
This commit is contained in:
jyvikoandClaude Opus 5.5 authored and Irakli Kourtis committed
1 parent
a3cb6d749b
commit
6d9452ad84
144 files changed
+5162
-74
No files matched your search
+100
-73
@@ -145,7 +145,7 @@ fn block_mesh(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let merged: BTreeSet<u32> = existing.iter().flatten().copied().collect();
|
||||
let elements: Vec<Vec<u32>> = mesh.elements.iter().map(|e| e.iter().map(|&i| labels.get(i as usize).copied().unwrap_or(0)).collect()).collect();
|
||||
if !merged.is_empty() {
|
||||
check_conforming(&s.model, kind, &mesh, &elements, &merged)?;
|
||||
check_conforming(&s.model, kind, &mesh, &elements, &merged, merge)?;
|
||||
}
|
||||
|
||||
let body_tag = s.model.geometry_tag(&body)?;
|
||||
@@ -233,84 +233,95 @@ fn key(nodes: impl Iterator<Item = u32>) -> Vec<u32> {
|
||||
k
|
||||
}
|
||||
|
||||
/// Where the block joins existing nodes, its boundary faces and the existing element faces on
|
||||
/// those nodes must be the same faces: no hanging nodes, no mixed linear and quadratic sides.
|
||||
/// A block joins along whole block faces and edges: when every corner of a block face (both
|
||||
/// ends of a block edge) lands on an existing node, every node on it must.
|
||||
fn check_conforming(model: &Model, kind: ElementKind, mesh: &BlockMesh, elements: &[Vec<u32>], merged: &BTreeSet<u32>) -> Result<()> {
|
||||
// Labels of the block's corners: the corner point's label in any element that uses it.
|
||||
let mut point_label: BTreeMap<u32, u32> = BTreeMap::new();
|
||||
for (e, points) in mesh.elements.iter().enumerate() {
|
||||
for (k, &pt) in points.iter().enumerate() {
|
||||
if let Some(&l) = elements.get(e).and_then(|n| n.get(k)) {
|
||||
point_label.insert(pt, l);
|
||||
}
|
||||
}
|
||||
}
|
||||
let joined = |pt: u32| point_label.get(&pt).is_some_and(|l| merged.contains(l));
|
||||
let hex_faces = ElementKind::Hex8.faces();
|
||||
let pieces = |list: &[(u32, u8, u8)], which: usize, local: fn(ElementKind, usize) -> Option<&'static [usize]>| -> Vec<u32> {
|
||||
list.iter()
|
||||
.filter(|t| usize::from(t.2) == which)
|
||||
.flat_map(|&(e, k, _)| {
|
||||
let nodes = elements.get(e as usize);
|
||||
local(kind, usize::from(k)).unwrap_or(&[]).iter().filter_map(move |&i| nodes.and_then(|n| n.get(i)).copied())
|
||||
})
|
||||
.collect()
|
||||
/// Where the block joins existing nodes it must meet the mesh face to face: an element face
|
||||
/// (of the block, or of the mesh) whose corners all landed on shared nodes must be a face of
|
||||
/// the other side too, with the same nodes (no mixed linear and quadratic sides); and no node on
|
||||
/// the block's boundary may lie on an edge of the mesh between two of its nodes, nor the other
|
||||
/// way round (no hanging nodes where the divisions differ). Edges are tested as straight
|
||||
/// segments, so a mismatch along a curved shared edge is caught only through its faces.
|
||||
fn check_conforming(model: &Model, kind: ElementKind, mesh: &BlockMesh, elements: &[Vec<u32>], merged: &BTreeSet<u32>, merge: f64) -> Result<()> {
|
||||
let refuse = |what: String| {
|
||||
Err(EngineError::Params(format!("{what}: the block must meet the mesh face to face, with the same divisions and element kind")))
|
||||
};
|
||||
for (bf, corners) in hex_faces.iter().enumerate() {
|
||||
if corners.iter().all(|&c| mesh.corners.get(c).is_some_and(|&pt| joined(pt))) {
|
||||
let nodes = pieces(&mesh.faces, bf, |k, f| k.faces().get(f.checked_sub(1)?).copied());
|
||||
if nodes.iter().any(|n| !merged.contains(n)) {
|
||||
return Err(EngineError::Params(format!(
|
||||
"block face {} lies on existing nodes at its corners but not at every node: the block must meet the mesh face to face (same divisions and element kind)",
|
||||
bf + 1
|
||||
)));
|
||||
}
|
||||
}
|
||||
let corners_of = |k: ElementKind, nodes: &[u32], local: &[usize]| -> Vec<u32> {
|
||||
key(local.iter().filter(|&&i| i < k.corner_count()).filter_map(|&i| nodes.get(i).copied()))
|
||||
};
|
||||
// The block's boundary faces, by sorted corners, with their full node lists.
|
||||
let mut new_faces: BTreeMap<Vec<u32>, Vec<u32>> = BTreeMap::new();
|
||||
for &(e, f, _) in &mesh.faces {
|
||||
let (Some(nodes), Some(local)) = (elements.get(e as usize), usize::from(f).checked_sub(1).and_then(|i| kind.faces().get(i))) else { continue };
|
||||
new_faces.insert(corners_of(kind, nodes, local), key(local.iter().filter_map(|&i| nodes.get(i).copied())));
|
||||
}
|
||||
for (be, [a, b]) in modelcraft_mesh::block::EDGES.iter().enumerate() {
|
||||
if [a, b].iter().all(|&&c| mesh.corners.get(c).is_some_and(|&pt| joined(pt))) {
|
||||
let nodes = pieces(&mesh.edges, be, |k, f| k.edges().get(f.checked_sub(1)?).copied());
|
||||
if nodes.iter().any(|n| !merged.contains(n)) {
|
||||
return Err(EngineError::Params(format!(
|
||||
"the block edge from corner {} to {} ends on existing nodes but not every node on it does: the block must meet the mesh with the same divisions",
|
||||
a + 1,
|
||||
b + 1
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
let new_faces: BTreeSet<Vec<u32>> = mesh
|
||||
.faces
|
||||
.iter()
|
||||
.filter_map(|&(e, f, _)| {
|
||||
let nodes = elements.get(e as usize)?;
|
||||
let local = kind.faces().get(usize::from(f).checked_sub(1)?)?;
|
||||
Some(key(local.iter().filter_map(|&i| nodes.get(i).copied())))
|
||||
})
|
||||
.collect();
|
||||
let mut old_faces = BTreeSet::new();
|
||||
for (label, e) in model.elements() {
|
||||
if !e.nodes.iter().any(|n| merged.contains(n)) {
|
||||
continue;
|
||||
}
|
||||
let mut old_faces: BTreeMap<Vec<u32>, Vec<u32>> = BTreeMap::new();
|
||||
let mut old_edges: BTreeSet<[u32; 2]> = BTreeSet::new();
|
||||
for e in model.elements().values().filter(|e| e.nodes.iter().any(|n| merged.contains(n))) {
|
||||
for local in e.kind.faces() {
|
||||
let face = key(local.iter().filter_map(|&i| e.nodes.get(i).copied()));
|
||||
if face.iter().all(|n| merged.contains(n)) {
|
||||
if !new_faces.contains(&face) {
|
||||
return Err(EngineError::Params(format!(
|
||||
"element {label} has a face on the block that no block element shares: the block must meet the mesh face to face (same divisions and element kind)"
|
||||
)));
|
||||
}
|
||||
old_faces.insert(face);
|
||||
old_faces.insert(corners_of(e.kind, &e.nodes, local), key(local.iter().filter_map(|&i| e.nodes.get(i).copied())));
|
||||
}
|
||||
}
|
||||
for (corners, nodes) in &new_faces {
|
||||
if corners.iter().all(|n| merged.contains(n)) && old_faces.get(corners) != Some(nodes) {
|
||||
return refuse(format!("the block face on nodes {corners:?} lies on existing nodes but is no element face there"));
|
||||
}
|
||||
}
|
||||
for (corners, nodes) in &old_faces {
|
||||
if corners.iter().all(|n| merged.contains(n)) && new_faces.get(corners) != Some(nodes) {
|
||||
return refuse(format!("the element face on nodes {corners:?} lies on the block but no block element shares it"));
|
||||
}
|
||||
}
|
||||
// Hanging nodes: boundary nodes of either side on the other side's element edges.
|
||||
let xyz = |label: u32| model.node(label).map(|n| n.xyz);
|
||||
let block_points: BTreeMap<u32, [f64; 3]> =
|
||||
mesh.elements.iter().zip(elements).flat_map(|(pts, labels)| pts.iter().zip(labels).filter_map(|(&p, &l)| Some((l, *mesh.points.get(p as usize)?)))).collect();
|
||||
let mut new_edges: BTreeSet<[u32; 2]> = BTreeSet::new();
|
||||
for &(e, f, _) in &mesh.faces {
|
||||
let (Some(nodes), Some(local)) = (elements.get(e as usize), usize::from(f).checked_sub(1).and_then(|i| kind.faces().get(i))) else { continue };
|
||||
let cs: Vec<u32> = local.iter().filter(|&&i| i < kind.corner_count()).filter_map(|&i| nodes.get(i).copied()).collect();
|
||||
for (i, &a) in cs.iter().enumerate() {
|
||||
let b = cs.get((i + 1) % cs.len()).copied().unwrap_or(a);
|
||||
new_edges.insert(if a < b { [a, b] } else { [b, a] });
|
||||
}
|
||||
}
|
||||
for e in model.elements().values().filter(|e| e.nodes.iter().any(|n| merged.contains(n))) {
|
||||
for local in e.kind.edges() {
|
||||
if let (Some(&a), Some(&b)) = (local.first().and_then(|&i| e.nodes.get(i)), local.get(1).and_then(|&i| e.nodes.get(i))) {
|
||||
old_edges.insert(if a < b { [a, b] } else { [b, a] });
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(f) = new_faces.iter().find(|f| f.iter().all(|n| merged.contains(n)) && !old_faces.contains(*f)) {
|
||||
return Err(EngineError::Params(format!(
|
||||
"the block face on nodes {f:?} lies on existing nodes but matches no element face there: the block must meet the mesh face to face (same divisions and element kind)"
|
||||
)));
|
||||
let new_boundary: BTreeSet<u32> = new_faces.values().flatten().copied().filter(|n| !merged.contains(n)).collect();
|
||||
let on_segment = |p: [f64; 3], a: [f64; 3], b: [f64; 3]| {
|
||||
let ab = [b[0] - a[0], b[1] - a[1], b[2] - a[2]];
|
||||
let ap = [p[0] - a[0], p[1] - a[1], p[2] - a[2]];
|
||||
let len2 = ab[0] * ab[0] + ab[1] * ab[1] + ab[2] * ab[2];
|
||||
if !(len2 > 0.0) {
|
||||
return false;
|
||||
}
|
||||
let t = (ap[0] * ab[0] + ap[1] * ab[1] + ap[2] * ab[2]) / len2;
|
||||
let d = [ap[0] - t * ab[0], ap[1] - t * ab[1], ap[2] - t * ab[2]];
|
||||
let tol = merge / len2.sqrt();
|
||||
t > tol && t < 1.0 - tol && (d[0] * d[0] + d[1] * d[1] + d[2] * d[2]).sqrt() <= merge
|
||||
};
|
||||
for &n in &new_boundary {
|
||||
let Some(&p) = block_points.get(&n) else { continue };
|
||||
for [a, b] in &old_edges {
|
||||
if let (Some(xa), Some(xb)) = (xyz(*a), xyz(*b))
|
||||
&& on_segment(p, xa, xb)
|
||||
{
|
||||
return refuse(format!("a block node at {p:?} lies on the edge between nodes {a} and {b}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
let old_nodes: BTreeSet<u32> = old_faces.values().flatten().copied().filter(|n| !merged.contains(n)).collect();
|
||||
for n in old_nodes {
|
||||
let Some(p) = xyz(n) else { continue };
|
||||
for [a, b] in &new_edges {
|
||||
if let (Some(&xa), Some(&xb)) = (block_points.get(a), block_points.get(b))
|
||||
&& on_segment(p, xa, xb)
|
||||
{
|
||||
return refuse(format!("node {n} at {p:?} lies on a block edge"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -367,6 +378,22 @@ mod tests {
|
||||
let mut t = Session::default();
|
||||
t.execute("mesh.block", &json!({"body": "A", "kind": "hex8", "corners": cube(0.0), "divisions": [2, 2, 2]})).unwrap();
|
||||
assert!(t.execute("mesh.block", &json!({"body": "B", "kind": "hex8", "corners": cube(1.0), "divisions": [1, 1, 1]})).is_err());
|
||||
// Divisions that share no node inside the face: the hanging nodes on its edges are caught.
|
||||
assert!(t.execute("mesh.block", &json!({"body": "B", "kind": "hex8", "corners": cube(1.0), "divisions": [3, 3, 3]})).is_err());
|
||||
// A ring closed by its last block touches existing nodes at the corners of faces it does
|
||||
// not share (the ends): that is no conflict.
|
||||
let mut ring = Session::default();
|
||||
for k in 0..4 {
|
||||
let at = |r: f64, q: usize, z: f64| {
|
||||
let a = std::f64::consts::FRAC_PI_2 * q as f64;
|
||||
let c = |v: f64| if v.abs() < 1e-12 { 0.0 } else { v };
|
||||
json!([c(r * a.cos()), c(r * a.sin()), z])
|
||||
};
|
||||
let corners = json!([at(1.0, k, 0.0), at(2.0, k, 0.0), at(2.0, k + 1, 0.0), at(1.0, k + 1, 0.0), at(1.0, k, 1.0), at(2.0, k, 1.0), at(2.0, k + 1, 1.0), at(1.0, k + 1, 1.0)]);
|
||||
let edges = json!([{"corners": [1, 4], "arc": {"center": [0, 0, 0]}}, {"corners": [2, 3], "arc": {"center": [0, 0, 0]}}, {"corners": [5, 8], "arc": {"center": [0, 0, 1]}}, {"corners": [6, 7], "arc": {"center": [0, 0, 1]}}]);
|
||||
ring.execute("mesh.block", &json!({"body": "Ring", "kind": "hex20", "corners": corners, "divisions": [2, 3, 2], "edges": edges})).unwrap();
|
||||
}
|
||||
assert_eq!(ring.model.elements().len(), 48);
|
||||
assert!(t.execute("mesh.block", &json!({"body": "B", "kind": "hex8", "corners": cube(1.0), "divisions": [1, 1, 1], "merge": 0})).is_ok());
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "modelcraft-testkit"
|
||||
description = "ModelCraft test tooling: the meshing test corpus (SolidCraft recipes and analytic shapes)"
|
||||
description = "ModelCraft test tooling: the meshing test corpus (SolidCraft recipes and analytic shapes) and the verification oracle"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
@@ -12,5 +12,6 @@ workspace = true
|
||||
[dependencies]
|
||||
modelcraft-geometry = { workspace = true, features = ["scripts"] }
|
||||
modelcraft-mesh = { workspace = true }
|
||||
modelcraft-engine = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@@ -0,0 +1,43 @@
|
||||
//! `oracle [--case NAME]... [--bless NAME|all]... [--quick] [--no-write]`: see
|
||||
//! [`modelcraft_testkit::oracle`]; run it as `cargo xtask oracle`.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use std::process::ExitCode;
|
||||
|
||||
use modelcraft_testkit::oracle::{Options, run};
|
||||
|
||||
const USAGE: &str = "usage: oracle [--case NAME]... [--bless NAME|all]... [--quick] [--no-write]";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let mut options = Options { write: true, ..Options::default() };
|
||||
let mut it = args.iter();
|
||||
while let Some(a) = it.next() {
|
||||
let value = |it: &mut std::slice::Iter<'_, String>| it.next().cloned().ok_or_else(|| format!("{a} needs a value\n{USAGE}"));
|
||||
let step = match a.as_str() {
|
||||
"--case" => value(&mut it).map(|v| options.cases.push(v)),
|
||||
"--bless" => value(&mut it).map(|v| options.bless.push(v)),
|
||||
"--quick" => {
|
||||
options.quick = true;
|
||||
Ok(())
|
||||
}
|
||||
"--no-write" => {
|
||||
options.write = false;
|
||||
Ok(())
|
||||
}
|
||||
other => Err(format!("unknown argument `{other}`\n{USAGE}")),
|
||||
};
|
||||
if let Err(e) = step {
|
||||
eprintln!("error: {e}");
|
||||
return ExitCode::FAILURE;
|
||||
}
|
||||
}
|
||||
match run(&modelcraft_testkit::corpus::workspace_root(), &options) {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
Err(e) => {
|
||||
eprintln!("error: {e}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,12 @@
|
||||
//! `tests/corpus/<name>/recipe.json`, and analytic shapes generated in code with closed-form
|
||||
//! volumes. Meshing tests load their geometry from here; [`runner`] (the `corpus` binary,
|
||||
//! `cargo xtask corpus`) builds and meshes every part and writes docs/meshing/corpus.md.
|
||||
//!
|
||||
//! [`oracle`] (the `oracle` binary, `cargo xtask oracle`) runs the verification and validation
|
||||
//! cases of `tests/oracle` against their references and fixtures and writes docs/oracle.md.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod corpus;
|
||||
pub mod oracle;
|
||||
pub mod runner;
|
||||
@@ -0,0 +1,688 @@
|
||||
//! The verification and validation oracle (docs/verification.md): every case in
|
||||
//! `tests/oracle/<case>/` is our own model of a problem with a known answer, run through the
|
||||
//! command engine and compared with its reference values and with its frozen fixture.
|
||||
//!
|
||||
//! A case directory holds:
|
||||
//! - `expected.json`: the case's kind (theory, experiment, benchmark), source, status and its
|
||||
//! quantities: how each is measured, the reference value, the tolerance and the reason for it;
|
||||
//! - `recipe.json`: the model as ModelCraft commands (a JSON array of `{"command", "params"}`,
|
||||
//! each step optionally with an `id` that measures can read its output by); absent while a
|
||||
//! case is pending;
|
||||
//! - `design.json` (optional): a SolidCraft script that builds the geometry, opened first;
|
||||
//! - `fixture.json`: our accepted values, written by `--bless` once the case passes;
|
||||
//! - `README.md`: the problem in our own words.
|
||||
//!
|
||||
//! [`run`] (the `oracle` binary, `cargo xtask oracle`) runs the cases, prints and checks the
|
||||
//! results and writes `docs/oracle.md` after a full run.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use modelcraft_engine::Session;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// Largest case file read.
|
||||
const MAX_FILE_BYTES: u64 = 16 << 20;
|
||||
|
||||
/// Fixture band: a result may move by this fraction of its size (and a thousandth of the
|
||||
/// reference's absolute tolerance) before CI calls it a change. Results are deterministic on one
|
||||
/// platform; the band only absorbs last-digit differences of the system maths library.
|
||||
pub const FIXTURE_RELATIVE: f64 = 1e-9;
|
||||
|
||||
/// The kinds of reference (docs/verification.md).
|
||||
pub const KINDS: [&str; 3] = ["theory", "benchmark", "experiment"];
|
||||
|
||||
/// Options of one run.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Options {
|
||||
/// Run only these cases (all when empty).
|
||||
pub cases: Vec<String>,
|
||||
/// Write `fixture.json` for these cases when they pass (`all` for every passing case).
|
||||
pub bless: Vec<String>,
|
||||
/// Skip cases marked `slow`.
|
||||
pub quick: bool,
|
||||
/// Write docs/oracle.md (only after a run of every case).
|
||||
pub write: bool,
|
||||
}
|
||||
|
||||
/// A quantity's tolerance: |ours − reference| ≤ relative·|reference| + absolute.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Default)]
|
||||
pub struct Tolerance {
|
||||
pub relative: f64,
|
||||
pub absolute: f64,
|
||||
}
|
||||
|
||||
impl Tolerance {
|
||||
fn from_json(v: &Value) -> Result<Tolerance, String> {
|
||||
let get = |k: &str| -> Result<f64, String> {
|
||||
match v.get(k) {
|
||||
None => Ok(0.0),
|
||||
Some(x) => x.as_f64().filter(|x| x.is_finite() && *x >= 0.0).ok_or_else(|| format!("tolerance `{k}` must be a number ≥ 0")),
|
||||
}
|
||||
};
|
||||
let t = Tolerance { relative: get("relative")?, absolute: get("absolute")? };
|
||||
if t.relative == 0.0 && t.absolute == 0.0 && v.get("exact") != Some(&Value::Bool(true)) {
|
||||
return Err("a tolerance gives `relative`, `absolute` or `\"exact\": true`".into());
|
||||
}
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
pub fn allows(&self, reference: f64, ours: f64) -> bool {
|
||||
(ours - reference).abs() <= self.relative * reference.abs() + self.absolute
|
||||
}
|
||||
|
||||
fn describe(&self) -> String {
|
||||
match (self.relative > 0.0, self.absolute > 0.0) {
|
||||
(true, false) => fraction(self.relative),
|
||||
(false, true) => format!("± {}", number(self.absolute)),
|
||||
(true, true) => format!("{} + {}", fraction(self.relative), number(self.absolute)),
|
||||
(false, false) => "exact".into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One quantity of a case.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Quantity {
|
||||
pub id: String,
|
||||
pub description: String,
|
||||
pub measure: Value,
|
||||
pub reference: Option<f64>,
|
||||
pub tolerance: Tolerance,
|
||||
pub reason: String,
|
||||
pub known_failure: Option<String>,
|
||||
}
|
||||
|
||||
/// One case's `expected.json`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Expected {
|
||||
pub title: String,
|
||||
pub kind: String,
|
||||
pub source: String,
|
||||
pub pending: Option<String>,
|
||||
pub slow: bool,
|
||||
pub quantities: Vec<Quantity>,
|
||||
}
|
||||
|
||||
fn text(v: &Value, key: &str) -> Result<String, String> {
|
||||
v.get(key).and_then(Value::as_str).filter(|s| !s.trim().is_empty()).map(str::to_string).ok_or_else(|| format!("`{key}` must be a non-empty string"))
|
||||
}
|
||||
|
||||
impl Expected {
|
||||
pub fn parse(v: &Value) -> Result<Expected, String> {
|
||||
let kind = text(v, "kind")?;
|
||||
if !KINDS.contains(&kind.as_str()) {
|
||||
return Err(format!("`kind` must be one of {KINDS:?}"));
|
||||
}
|
||||
let pending = match v.get("status").and_then(Value::as_str) {
|
||||
None | Some("active") => None,
|
||||
Some("pending") => Some(text(v, "requires")?),
|
||||
Some(other) => return Err(format!("`status` must be active or pending, not `{other}`")),
|
||||
};
|
||||
let list = v.get("quantities").and_then(Value::as_array).filter(|q| !q.is_empty()).ok_or("`quantities` must be a non-empty list")?;
|
||||
let mut quantities = Vec::with_capacity(list.len());
|
||||
for q in list {
|
||||
let id = text(q, "id")?;
|
||||
let context = |e: String| format!("quantity `{id}`: {e}");
|
||||
let reference = match q.get("value") {
|
||||
None | Some(Value::Null) if pending.is_some() => None,
|
||||
Some(x) => Some(x.as_f64().filter(|x| x.is_finite()).ok_or_else(|| context("`value` must be a finite number".into()))?),
|
||||
None => return Err(context("`value` is required".into())),
|
||||
};
|
||||
let tolerance = match q.get("tolerance") {
|
||||
None if pending.is_some() => Tolerance::default(),
|
||||
Some(t) => Tolerance::from_json(t).map_err(context)?,
|
||||
None => return Err(context("`tolerance` is required".into())),
|
||||
};
|
||||
let reason = if pending.is_some() { q.get("reason").and_then(Value::as_str).unwrap_or_default().to_string() } else { text(q, "reason").map_err(context)? };
|
||||
let measure = q.get("measure").cloned().unwrap_or(Value::Null);
|
||||
if pending.is_none() && measure.get("pointer").and_then(Value::as_str).is_none() {
|
||||
return Err(context("`measure` needs a JSON `pointer` into a step's output or a command's".into()));
|
||||
}
|
||||
if quantities.iter().any(|x: &Quantity| x.id == id) {
|
||||
return Err(format!("quantity `{id}` is listed twice"));
|
||||
}
|
||||
quantities.push(Quantity {
|
||||
description: text(q, "quantity").map_err(context)?,
|
||||
known_failure: q.get("known_failure").and_then(Value::as_str).map(str::to_string),
|
||||
id,
|
||||
measure,
|
||||
reference,
|
||||
tolerance,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
Ok(Expected {
|
||||
title: text(v, "title")?,
|
||||
kind,
|
||||
source: text(v, "source")?,
|
||||
pending,
|
||||
slow: v.get("slow").and_then(Value::as_bool).unwrap_or(false),
|
||||
quantities,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A frozen value and how far it may move.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct Frozen {
|
||||
pub value: f64,
|
||||
pub band: f64,
|
||||
}
|
||||
|
||||
/// The band a fixture value gets when it is blessed.
|
||||
pub fn fixture_band(value: f64, reference: f64, tolerance: Tolerance) -> f64 {
|
||||
FIXTURE_RELATIVE * value.abs().max(reference.abs()) + 1e-3 * tolerance.absolute
|
||||
}
|
||||
|
||||
fn read_json(path: &Path) -> Result<Value, String> {
|
||||
let size = std::fs::metadata(path).map_err(|e| format!("{}: {e}", path.display()))?.len();
|
||||
if size > MAX_FILE_BYTES {
|
||||
return Err(format!("{}: larger than {MAX_FILE_BYTES} bytes", path.display()));
|
||||
}
|
||||
let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))
|
||||
}
|
||||
|
||||
fn read_fixture(path: &Path) -> Result<Option<BTreeMap<String, Frozen>>, String> {
|
||||
if !path.is_file() {
|
||||
return Ok(None);
|
||||
}
|
||||
let v = read_json(path)?;
|
||||
let values = v.get("values").and_then(Value::as_object).ok_or_else(|| format!("{}: no `values`", path.display()))?;
|
||||
values
|
||||
.iter()
|
||||
.map(|(k, f)| {
|
||||
let value = f.get("value").and_then(Value::as_f64);
|
||||
let band = f.get("band").and_then(Value::as_f64).filter(|b| *b >= 0.0);
|
||||
match (value, band) {
|
||||
(Some(value), Some(band)) => Ok((k.clone(), Frozen { value, band })),
|
||||
_ => Err(format!("{}: `{k}` needs a `value` and a `band` ≥ 0", path.display())),
|
||||
}
|
||||
})
|
||||
.collect::<Result<_, _>>()
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
/// The result of measuring one quantity.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Measured {
|
||||
pub id: String,
|
||||
pub value: Result<f64, String>,
|
||||
}
|
||||
|
||||
/// Run a case's recipe and measure its quantities. A failing command fails every quantity.
|
||||
pub fn run_case(dir: &Path, expected: &Expected) -> Result<Vec<Measured>, String> {
|
||||
let mut session = Session::default();
|
||||
let design = dir.join("design.json");
|
||||
if design.is_file() {
|
||||
let script = std::fs::read_to_string(&design).map_err(|e| format!("{}: {e}", design.display()))?;
|
||||
let bytes = modelcraft_geometry::script_design(&script).map_err(|e| format!("design.json: {e}"))?;
|
||||
let opened = modelcraft_geometry::open_design(&bytes).map_err(|e| format!("design.json: {e}"))?;
|
||||
if let Some(e) = opened.errors.first() {
|
||||
return Err(format!("design.json: feature {}: {}", e.feature, e.message));
|
||||
}
|
||||
session.design = Some(Arc::new(opened));
|
||||
}
|
||||
let recipe = read_json(&dir.join("recipe.json"))?;
|
||||
let steps = recipe.as_array().ok_or("recipe.json must be a JSON array of steps")?;
|
||||
let mut outputs: BTreeMap<String, Value> = BTreeMap::new();
|
||||
for (i, step) in steps.iter().enumerate() {
|
||||
let command = step.get("command").and_then(Value::as_str).ok_or_else(|| format!("recipe step {}: no `command`", i + 1))?;
|
||||
let params = step.get("params").cloned().unwrap_or_else(|| json!({}));
|
||||
let out = session.execute(command, ¶ms).map_err(|e| format!("recipe step {} ({command}): {e}", i + 1))?;
|
||||
if let Some(id) = step.get("id").and_then(Value::as_str) {
|
||||
outputs.insert(id.to_string(), out);
|
||||
}
|
||||
}
|
||||
Ok(expected
|
||||
.quantities
|
||||
.iter()
|
||||
.map(|q| Measured { id: q.id.clone(), value: measure(&mut session, &outputs, &q.measure) })
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Read one number: from a recipe step's output (`"from": id`) or a command run now
|
||||
/// (`"command"`, `"params"`), at a JSON pointer.
|
||||
fn measure(session: &mut Session, outputs: &BTreeMap<String, Value>, m: &Value) -> Result<f64, String> {
|
||||
let pointer = m.get("pointer").and_then(Value::as_str).ok_or("no `pointer`")?;
|
||||
let fresh;
|
||||
let out = match (m.get("from").and_then(Value::as_str), m.get("command").and_then(Value::as_str)) {
|
||||
(Some(id), None) => outputs.get(id).ok_or_else(|| format!("no recipe step has id `{id}`"))?,
|
||||
(None, Some(command)) => {
|
||||
let params = m.get("params").cloned().unwrap_or_else(|| json!({}));
|
||||
fresh = session.execute(command, ¶ms).map_err(|e| format!("{command}: {e}"))?;
|
||||
&fresh
|
||||
}
|
||||
_ => return Err("a measure gives either `from` (a recipe step id) or `command`".into()),
|
||||
};
|
||||
let v = out.pointer(pointer).ok_or_else(|| format!("nothing at `{pointer}`"))?;
|
||||
let x = v.as_f64().ok_or_else(|| format!("`{pointer}` is {v}, not a number"))?;
|
||||
let x = x * m.get("scale").and_then(Value::as_f64).unwrap_or(1.0);
|
||||
if x.is_finite() { Ok(x) } else { Err(format!("`{pointer}` is not finite")) }
|
||||
}
|
||||
|
||||
/// How one quantity came out.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Verdict {
|
||||
Pass,
|
||||
Fail,
|
||||
/// Fails, and expected.json says why.
|
||||
Known,
|
||||
/// Marked as a known failure but passes: remove the mark.
|
||||
Fixed,
|
||||
/// Could not be measured.
|
||||
Error(String),
|
||||
Pending,
|
||||
}
|
||||
|
||||
/// The fixture check of one quantity.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum FixtureCheck {
|
||||
Matches,
|
||||
Moved { frozen: f64 },
|
||||
Missing,
|
||||
/// No fixture is kept (pending, failing or known-failing quantity).
|
||||
NotKept,
|
||||
Blessed,
|
||||
}
|
||||
|
||||
/// One row of the report.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Row {
|
||||
pub case: String,
|
||||
pub quantity: Quantity,
|
||||
pub ours: Option<f64>,
|
||||
pub verdict: Verdict,
|
||||
pub fixture: FixtureCheck,
|
||||
}
|
||||
|
||||
/// Judge a measured value against the reference and the fixture.
|
||||
pub fn judge(q: &Quantity, ours: &Result<f64, String>, frozen: Option<&Frozen>) -> (Verdict, FixtureCheck) {
|
||||
let (Some(reference), Ok(ours)) = (q.reference, ours) else {
|
||||
return match ours {
|
||||
Err(e) => (Verdict::Error(e.clone()), FixtureCheck::NotKept),
|
||||
Ok(_) => (Verdict::Pending, FixtureCheck::NotKept),
|
||||
};
|
||||
};
|
||||
let pass = q.tolerance.allows(reference, *ours);
|
||||
let verdict = match (pass, &q.known_failure) {
|
||||
(true, None) => Verdict::Pass,
|
||||
(true, Some(_)) => Verdict::Fixed,
|
||||
(false, None) => Verdict::Fail,
|
||||
(false, Some(_)) => Verdict::Known,
|
||||
};
|
||||
let fixture = match frozen {
|
||||
Some(f) if (ours - f.value).abs() <= f.band => FixtureCheck::Matches,
|
||||
Some(f) => FixtureCheck::Moved { frozen: f.value },
|
||||
None if verdict == Verdict::Pass => FixtureCheck::Missing,
|
||||
None => FixtureCheck::NotKept,
|
||||
};
|
||||
(verdict, fixture)
|
||||
}
|
||||
|
||||
/// The case directories under `tests/oracle`, sorted.
|
||||
pub fn case_dirs(root: &Path) -> Result<Vec<PathBuf>, String> {
|
||||
let dir = root.join("tests").join("oracle");
|
||||
let mut cases: Vec<PathBuf> = std::fs::read_dir(&dir)
|
||||
.map_err(|e| format!("{}: {e}", dir.display()))?
|
||||
.flatten()
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.join("expected.json").is_file())
|
||||
.collect();
|
||||
cases.sort();
|
||||
Ok(cases)
|
||||
}
|
||||
|
||||
fn name_of(dir: &Path) -> String {
|
||||
dir.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Run the oracle. Fails when a quantity fails its reference, a fixture moved or is missing,
|
||||
/// a known failure now passes, or a case cannot run.
|
||||
pub fn run(root: &Path, options: &Options) -> Result<(), String> {
|
||||
let all = case_dirs(root)?;
|
||||
for want in options.cases.iter().chain(options.bless.iter().filter(|b| *b != "all")) {
|
||||
if !all.iter().any(|d| name_of(d) == *want) {
|
||||
return Err(format!("no case `{want}` in tests/oracle"));
|
||||
}
|
||||
}
|
||||
let selected: Vec<&PathBuf> = all.iter().filter(|d| options.cases.is_empty() || options.cases.contains(&name_of(d))).collect();
|
||||
let mut rows = Vec::new();
|
||||
let mut cases: Vec<(String, Expected)> = Vec::new();
|
||||
let mut problems = Vec::new();
|
||||
let mut skipped = Vec::new();
|
||||
for dir in selected {
|
||||
let case = name_of(dir);
|
||||
let expected = Expected::parse(&read_json(&dir.join("expected.json"))?).map_err(|e| format!("{case}/expected.json: {e}"))?;
|
||||
if expected.slow && options.quick {
|
||||
skipped.push(case);
|
||||
continue;
|
||||
}
|
||||
let measured: Vec<Measured> = if expected.pending.is_some() {
|
||||
expected.quantities.iter().map(|q| Measured { id: q.id.clone(), value: Err("pending".into()) }).collect()
|
||||
} else {
|
||||
let start = Instant::now();
|
||||
let m = run_case(dir, &expected).unwrap_or_else(|e| expected.quantities.iter().map(|q| Measured { id: q.id.clone(), value: Err(e.clone()) }).collect());
|
||||
eprintln!("oracle: {case} ({:.2} s)", start.elapsed().as_secs_f64());
|
||||
m
|
||||
};
|
||||
let fixture_path = dir.join("fixture.json");
|
||||
let fixture = read_fixture(&fixture_path)?;
|
||||
let bless = options.bless.iter().any(|b| *b == case || b == "all");
|
||||
let mut case_rows = Vec::new();
|
||||
for (q, m) in expected.quantities.iter().zip(&measured) {
|
||||
let (verdict, check) = if expected.pending.is_some() {
|
||||
(Verdict::Pending, FixtureCheck::NotKept)
|
||||
} else {
|
||||
judge(q, &m.value, fixture.as_ref().and_then(|f| f.get(&q.id)))
|
||||
};
|
||||
case_rows.push(Row { case: case.clone(), quantity: q.clone(), ours: m.value.as_ref().ok().copied(), verdict, fixture: check });
|
||||
}
|
||||
if bless && expected.pending.is_none() {
|
||||
let passing = case_rows.iter().all(|r| matches!(r.verdict, Verdict::Pass | Verdict::Known));
|
||||
if !passing {
|
||||
problems.push(format!("{case}: not blessed, it does not pass against expected.json"));
|
||||
} else {
|
||||
let mut values = serde_json::Map::new();
|
||||
for r in case_rows.iter_mut().filter(|r| r.verdict == Verdict::Pass) {
|
||||
if let (Some(ours), Some(reference)) = (r.ours, r.quantity.reference) {
|
||||
values.insert(r.quantity.id.clone(), json!({"value": ours, "band": fixture_band(ours, reference, r.quantity.tolerance)}));
|
||||
r.fixture = FixtureCheck::Blessed;
|
||||
}
|
||||
}
|
||||
let text = serde_json::to_string_pretty(&json!({
|
||||
"note": "ModelCraft's accepted results for this case, written by `cargo xtask oracle --bless`. A result outside its band fails CI.",
|
||||
"values": values,
|
||||
}))
|
||||
.map_err(|e| e.to_string())?;
|
||||
std::fs::write(&fixture_path, text + "\n").map_err(|e| format!("{}: {e}", fixture_path.display()))?;
|
||||
}
|
||||
}
|
||||
for r in &case_rows {
|
||||
let id = format!("{case}/{}", r.quantity.id);
|
||||
match &r.verdict {
|
||||
Verdict::Fail => problems.push(format!("{id}: {} vs reference {} ({})", show(r.ours), show(r.quantity.reference), r.quantity.tolerance.describe())),
|
||||
Verdict::Fixed => problems.push(format!("{id}: passes now; remove its `known_failure` mark")),
|
||||
Verdict::Error(e) => problems.push(format!("{id}: {e}")),
|
||||
_ => {}
|
||||
}
|
||||
match &r.fixture {
|
||||
FixtureCheck::Moved { frozen } => problems.push(format!("{id}: moved from the fixture {} to {}", number(*frozen), show(r.ours))),
|
||||
FixtureCheck::Missing => problems.push(format!("{id}: no fixture; bless the case (`cargo xtask oracle --bless {case}`)")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
rows.extend(case_rows);
|
||||
cases.push((case, expected));
|
||||
}
|
||||
print_summary(&rows);
|
||||
if options.write && options.cases.is_empty() && !options.quick {
|
||||
let path = root.join("docs").join("oracle.md");
|
||||
std::fs::write(&path, report(&cases, &rows)).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
eprintln!("oracle: wrote docs/oracle.md");
|
||||
}
|
||||
if !skipped.is_empty() {
|
||||
eprintln!("oracle: skipped slow cases: {}", skipped.join(", "));
|
||||
}
|
||||
if problems.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{} problem(s):\n {}", problems.len(), problems.join("\n ")))
|
||||
}
|
||||
}
|
||||
|
||||
fn print_summary(rows: &[Row]) {
|
||||
for r in rows {
|
||||
if r.verdict == Verdict::Pending {
|
||||
continue;
|
||||
}
|
||||
println!(
|
||||
"{:<34} {:<22} ours {:<14} ref {:<14} err {:<12} {:<14} {}",
|
||||
r.case,
|
||||
r.quantity.id,
|
||||
show(r.ours),
|
||||
show(r.quantity.reference),
|
||||
error_of(r),
|
||||
verdict_text(&r.verdict),
|
||||
fixture_text(&r.fixture)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A number with five significant digits.
|
||||
pub fn number(x: f64) -> String {
|
||||
if x == 0.0 {
|
||||
return "0".into();
|
||||
}
|
||||
let a = x.abs();
|
||||
if (1e-3..1e5).contains(&a) {
|
||||
let digits = 4 - a.log10().floor() as i32;
|
||||
let s = format!("{x:.*}", digits.max(0) as usize);
|
||||
if s.contains('.') { s.trim_end_matches('0').trim_end_matches('.').to_string() } else { s }
|
||||
} else {
|
||||
format!("{x:.4e}")
|
||||
}
|
||||
}
|
||||
|
||||
/// A fraction as a percentage with three significant digits.
|
||||
fn percent(r: f64) -> String {
|
||||
let p = r * 100.0;
|
||||
if p == 0.0 {
|
||||
return "0%".into();
|
||||
}
|
||||
let digits = (2 - p.abs().log10().floor() as i32).max(0) as usize;
|
||||
let s = format!("{p:.digits$}");
|
||||
let s = if s.contains('.') { s.trim_end_matches('0').trim_end_matches('.').to_string() } else { s };
|
||||
format!("{s}%")
|
||||
}
|
||||
|
||||
/// A tolerance's relative part: a percentage, or a plain fraction when it only admits rounding.
|
||||
fn fraction(r: f64) -> String {
|
||||
if r >= 1e-4 { percent(r) } else { format!("{r:.0e} rel.") }
|
||||
}
|
||||
|
||||
fn show(x: Option<f64>) -> String {
|
||||
x.map(number).unwrap_or_else(|| "—".into())
|
||||
}
|
||||
|
||||
/// The error against the reference: relative where the reference is not zero.
|
||||
fn error_of(r: &Row) -> String {
|
||||
let (Some(ours), Some(reference)) = (r.ours, r.quantity.reference) else { return "—".into() };
|
||||
let d = ours - reference;
|
||||
if reference != 0.0 && r.quantity.tolerance.relative > 0.0 {
|
||||
let rel = d / reference.abs();
|
||||
if rel.abs() < 1e-12 { "< 1e-10%".into() } else { percent(rel) }
|
||||
} else if d.abs() < 1e-12 {
|
||||
"< 1e-12".into()
|
||||
} else {
|
||||
format!("{d:.1e}")
|
||||
}
|
||||
}
|
||||
|
||||
fn verdict_text(v: &Verdict) -> String {
|
||||
match v {
|
||||
Verdict::Pass => "pass".into(),
|
||||
Verdict::Fail => "**FAIL**".into(),
|
||||
Verdict::Known => "known failure".into(),
|
||||
Verdict::Fixed => "**passes: remove known_failure**".into(),
|
||||
Verdict::Error(_) => "**ERROR**".into(),
|
||||
Verdict::Pending => "pending".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_text(f: &FixtureCheck) -> &'static str {
|
||||
match f {
|
||||
FixtureCheck::Matches => "matches",
|
||||
FixtureCheck::Moved { .. } => "**MOVED**",
|
||||
FixtureCheck::Missing => "**missing**",
|
||||
FixtureCheck::NotKept => "—",
|
||||
FixtureCheck::Blessed => "blessed",
|
||||
}
|
||||
}
|
||||
|
||||
fn cell(s: &str) -> String {
|
||||
s.replace('|', "/").replace('\n', " ")
|
||||
}
|
||||
|
||||
/// docs/oracle.md.
|
||||
pub fn report(cases: &[(String, Expected)], rows: &[Row]) -> String {
|
||||
let active: Vec<&(String, Expected)> = cases.iter().filter(|(_, e)| e.pending.is_none()).collect();
|
||||
let count = |kind: &str| active.iter().filter(|(_, e)| e.kind == kind).count();
|
||||
let quantities = rows.iter().filter(|r| r.verdict != Verdict::Pending).count();
|
||||
let passing = rows.iter().filter(|r| r.verdict == Verdict::Pass).count();
|
||||
let known = rows.iter().filter(|r| r.verdict == Verdict::Known).count();
|
||||
let mut md = String::new();
|
||||
let _ = write!(
|
||||
md,
|
||||
"# Verification and validation oracle\n\n\
|
||||
Generated by `cargo xtask oracle` from [`tests/oracle/`](../tests/oracle/); the policy is [verification.md](verification.md). \
|
||||
Every case is our own model, built with ModelCraft commands from the problem's published description, and compared with \
|
||||
the reference value its `expected.json` cites, within the stated tolerance and for the stated reason. Accepted results are \
|
||||
frozen in each case's `fixture.json`: a later change that moves one beyond its band (10⁻⁹ of its size) fails CI even while \
|
||||
the reference still passes. No reference value was produced with a commercial product.\n\n\
|
||||
**{passing} of {quantities} quantities pass** ({known} known failure(s)) in {} active cases: {} theory, {} published \
|
||||
benchmark, {} experiment; {} pending cases wait for capabilities still being built.\n\n",
|
||||
active.len(),
|
||||
count("theory"),
|
||||
count("benchmark"),
|
||||
count("experiment"),
|
||||
cases.len() - active.len(),
|
||||
);
|
||||
for kind in KINDS {
|
||||
let in_kind: Vec<&&(String, Expected)> = active.iter().filter(|(_, e)| e.kind == kind).collect();
|
||||
if in_kind.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let heading = match kind {
|
||||
"theory" => "Theory: closed-form solutions, patch tests and mesh invariants",
|
||||
"benchmark" => "Published benchmarks",
|
||||
_ => "Experiments",
|
||||
};
|
||||
let _ = write!(md, "## {heading}\n\n| Case | Quantity | Reference | ModelCraft | Error | Tolerance | Result | Fixture |\n|---|---|---|---|---|---|---|---|\n");
|
||||
for (case, _) in &in_kind {
|
||||
for r in rows.iter().filter(|r| &r.case == case) {
|
||||
let mut result = verdict_text(&r.verdict);
|
||||
if let Some(k) = &r.quantity.known_failure {
|
||||
result = format!("{result}: {k}");
|
||||
}
|
||||
if let Verdict::Error(e) = &r.verdict {
|
||||
result = format!("{result}: {e}");
|
||||
}
|
||||
let _ = writeln!(
|
||||
md,
|
||||
"| [{case}](../tests/oracle/{case}/) | {} | {} | {} | {} | {} | {} | {} |",
|
||||
cell(&r.quantity.description),
|
||||
show(r.quantity.reference),
|
||||
show(r.ours),
|
||||
error_of(r),
|
||||
r.quantity.tolerance.describe(),
|
||||
cell(&result),
|
||||
fixture_text(&r.fixture),
|
||||
);
|
||||
}
|
||||
}
|
||||
md.push('\n');
|
||||
}
|
||||
md += "## Sources and tolerances\n\n";
|
||||
for (case, e) in &active {
|
||||
let _ = write!(md, "- **[{case}](../tests/oracle/{case}/)** ({}): {}. Source: {}.", e.kind, cell(&e.title), cell(&e.source));
|
||||
let reasons: Vec<String> = e.quantities.iter().map(|q| format!("{}: {}", q.id, cell(&q.reason))).collect();
|
||||
let _ = writeln!(md, " Tolerances — {}", reasons.join("; "));
|
||||
}
|
||||
let pending: Vec<&(String, Expected)> = cases.iter().filter(|(_, e)| e.pending.is_some()).collect();
|
||||
if !pending.is_empty() {
|
||||
md += "\n## Pending cases\n\nReference values already cited; each case gets its recipe when the capability it needs lands.\n\n| Case | Kind | Problem | Needs | Reference values | Source |\n|---|---|---|---|---|---|\n";
|
||||
for (case, e) in pending {
|
||||
let values: Vec<String> =
|
||||
e.quantities.iter().map(|q| format!("{} = {}", cell(&q.description), q.reference.map(number).unwrap_or_else(|| "to be transcribed".into()))).collect();
|
||||
let _ = writeln!(
|
||||
md,
|
||||
"| [{case}](../tests/oracle/{case}/) | {} | {} | {} | {} | {} |",
|
||||
e.kind,
|
||||
cell(&e.title),
|
||||
cell(e.pending.as_deref().unwrap_or_default()),
|
||||
values.join("; "),
|
||||
cell(&e.source)
|
||||
);
|
||||
}
|
||||
}
|
||||
md
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn quantity(reference: f64, tolerance: Tolerance, known: Option<&str>) -> Quantity {
|
||||
Quantity {
|
||||
id: "q".into(),
|
||||
description: "q".into(),
|
||||
measure: json!({"from": "s", "pointer": "/x"}),
|
||||
reference: Some(reference),
|
||||
tolerance,
|
||||
reason: "r".into(),
|
||||
known_failure: known.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verdicts_and_fixtures() {
|
||||
let t = Tolerance { relative: 0.01, absolute: 0.0 };
|
||||
let q = quantity(2.0, t, None);
|
||||
let frozen = Frozen { value: 2.01, band: fixture_band(2.01, 2.0, t) };
|
||||
assert_eq!(judge(&q, &Ok(2.01), Some(&frozen)), (Verdict::Pass, FixtureCheck::Matches));
|
||||
assert_eq!(judge(&q, &Ok(2.011), Some(&frozen)), (Verdict::Pass, FixtureCheck::Moved { frozen: 2.01 }));
|
||||
assert_eq!(judge(&q, &Ok(2.01), None), (Verdict::Pass, FixtureCheck::Missing));
|
||||
assert_eq!(judge(&q, &Ok(2.1), None).0, Verdict::Fail);
|
||||
assert_eq!(judge(&quantity(2.0, t, Some("why")), &Ok(2.1), None), (Verdict::Known, FixtureCheck::NotKept));
|
||||
assert_eq!(judge(&quantity(2.0, t, Some("why")), &Ok(2.0), None).0, Verdict::Fixed);
|
||||
assert!(matches!(judge(&q, &Err("boom".into()), None).0, Verdict::Error(_)));
|
||||
// Zero references use the absolute tolerance, and the band keeps a floor from it.
|
||||
let a = Tolerance { relative: 0.0, absolute: 1e-9 };
|
||||
assert!(a.allows(0.0, 5e-10) && !a.allows(0.0, 2e-9));
|
||||
assert!(fixture_band(1e-16, 0.0, a) >= 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expected_files_are_checked() {
|
||||
let good = json!({"title": "t", "kind": "theory", "source": "s", "quantities": [
|
||||
{"id": "a", "quantity": "a", "measure": {"from": "x", "pointer": "/a"}, "value": 1.0, "tolerance": {"relative": 0.1}, "reason": "r"}]});
|
||||
assert!(Expected::parse(&good).is_ok());
|
||||
let pending = json!({"title": "t", "kind": "benchmark", "source": "s", "status": "pending", "requires": "shells",
|
||||
"quantities": [{"id": "a", "quantity": "a", "value": null}]});
|
||||
assert!(Expected::parse(&pending).unwrap().pending.is_some());
|
||||
for (key, bad) in [
|
||||
("kind", json!("guess")),
|
||||
("quantities", json!([])),
|
||||
("status", json!("maybe")),
|
||||
("title", json!("")),
|
||||
] {
|
||||
let mut v = good.clone();
|
||||
v[key] = bad;
|
||||
assert!(Expected::parse(&v).is_err(), "{key}");
|
||||
}
|
||||
for (key, bad) in [("tolerance", json!({})), ("tolerance", json!({"relative": -1})), ("value", json!("x")), ("reason", json!("")), ("measure", json!({}))] {
|
||||
let mut v = good.clone();
|
||||
v["quantities"][0][key] = bad;
|
||||
assert!(Expected::parse(&v).is_err(), "{key}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numbers_print_with_five_digits() {
|
||||
assert_eq!(number(92.7), "92.7");
|
||||
assert_eq!(number(-4.0241234), "-4.0241");
|
||||
assert_eq!(number(0.30241), "0.30241");
|
||||
assert_eq!(number(1.8248e-5), "1.8248e-5");
|
||||
assert_eq!(number(210000.0), "2.1000e5");
|
||||
assert_eq!(number(0.0), "0");
|
||||
assert_eq!(percent(-0.0050317), "-0.503%");
|
||||
assert_eq!(percent(0.000132), "0.0132%");
|
||||
assert_eq!(percent(0.02), "2%");
|
||||
assert_eq!(fraction(1e-9), "1e-9 rel.");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user