Action matrix: nightly shards cut by measured cost, a fresh baseline, and a determinism check (#1256)

* Action matrix: nightly shards cut by measured cost, heavy designs in slices, and a determinism check

The full matrix ran as four fixed quarters of the corpus and two of them hit the 60-minute
job limit. Now action_matrix_shard runs shard k of n (ACTION_MATRIX_SHARD=k/n), planned from
tests/action_matrix/costs.json: designs that take more than five minutes are split into slices,
and the pieces go largest first to the least loaded shard. The nightly runs 12 shards and uploads
each one's outcomes and timings; cargo xtask action-matrix --from DIR --bless records them in the
baseline and costs.json without running the matrix again. action_matrix_determinism runs a sample
twice and lists any action whose outcome changed.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

* Action matrix: costs measured on runners, 10 shards, determinism on the lighter designs

Co-Authored-By: Claude Opus 5.5 <[email protected]>

* Action matrix: baseline and costs from a full run on main, 16 shards, timeouts listed

The baseline is regenerated from the 10 shards' results of a full run on 01b1b705
(cargo xtask action-matrix --from DIR --bless): 8 new designs, 302 actions that pass now,
names that changed, and 40 actions that passed before and fail now, each listed in #1255
(22 fillets refused since #949, 4 hinge blends since #1101, 9 on the loft cut, 5 single).
costs.json now holds every design's measured time, and the nightly runs 16 shards of about
17 minutes. An action that passes in the baseline and times out is listed but not judged,
and stays passing when blessed: two runs of the same build differed only on 10 actions that
took 50-84 s against the 90 s limit.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

* Action matrix: slice heavy designs by a hash of each action's name

Slicing by index gave some slices only fillets on oracle 41 (28 slices, a multiple of the
actions per edge), three times the planned time; two shards ran past their limit.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

---------

Co-authored-by: Claude Opus 5.5 <[email protected]>

.github/workflows/ci.yml

@@ -205,16 +205,17 @@ jobs:
   action-matrix:
     # Every ordinary action on every face, edge, vertex and body of the design corpus, against
     # tests/action_matrix/baseline.json (docs/action-matrix.md). Every test run checks a sample;
-    # the whole matrix runs nightly (or by hand), a quarter per job.
-    name: Action matrix (${{ matrix.part }}/4)
+    # the whole matrix runs nightly (or by hand) in shards of about equal cost, cut by
+    # tests/action_matrix/costs.json. Keep SHARDS in step with the matrix below.
+    name: Action matrix (${{ matrix.shard }}/16)
     needs: build-tests
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-24.04
-    timeout-minutes: 60
+    timeout-minutes: 55
     strategy:
       fail-fast: false
       matrix:
-        part: [1, 2, 3, 4]
+        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28
@@ -223,12 +224,48 @@ jobs:
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: nextest-archive
-      - name: Run a quarter of the matrix
+      - name: Run one shard of the matrix
+        # A step limit below the job's, so the outcomes so far are still uploaded.
+        timeout-minutes: 45
         env:
-          PART: ${{ matrix.part }}
+          SHARDS: 16
+          SHARD: ${{ matrix.shard }}
+          OUT: ${{ github.workspace }}/action-matrix
+        run: |
+          mkdir -p "$OUT"
+          export ACTION_MATRIX_SHARD="$SHARD/$SHARDS" ACTION_MATRIX_RESULTS="$OUT/results-$SHARD.json" \
+            ACTION_MATRIX_TIMINGS="$OUT/timings-$SHARD.jsonl" ACTION_MATRIX_WORKDIR="$OUT/raw-$SHARD"
+          cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \
+            --profile ci --success-output final --run-ignored only -E "binary(action_matrix) & test(=action_matrix_shard)"
+      - name: Upload the outcomes and timings
+        # `cargo xtask action-matrix --from DIR --bless` (with every shard's files in DIR)
+        # records them in the baseline and costs.json.
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: action-matrix-${{ matrix.shard }}
+          path: action-matrix
+          retention-days: 14
+
+  action-matrix-determinism:
+    # A sample of the matrix run twice on one runner: every action must come out the same.
+    name: Action matrix determinism
+    needs: build-tests
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28
+        with:
+          tool: cargo-nextest
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: nextest-archive
+      - name: Run a sample twice
         run: |
           cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \
-            --profile ci --run-ignored only -E "binary(action_matrix) & test(=action_matrix_full_$PART)"
+            --profile ci --success-output final --run-ignored only -E "binary(action_matrix) & test(=action_matrix_determinism)"
 
   doc-tests:
     name: Doc tests

apps/solidcraft-cli/tests/action_matrix.rs

@@ -13,8 +13,10 @@
 //! fails the test (a regression). Newly passing actions are reported; record them with
 //! `cargo xtask action-matrix --bless` (or `ACTION_MATRIX_BLESS=1` on [`action_matrix_all`]).
 //! Every CI run checks a fixed sample ([`action_matrix_sample`], under a minute); the full matrix
-//! is four ignored tests (`action_matrix_full_N`) that the nightly "Action matrix" job runs side by
-//! side. See docs/action-matrix.md.
+//! is the ignored [`action_matrix_shard`], which the nightly "Action matrix" jobs run one shard
+//! each (`ACTION_MATRIX_SHARD=k/n`), cut by what each design costs (`tests/action_matrix/costs.json`).
+//! [`action_matrix_determinism`] runs a sample twice and lists any action whose outcome changed.
+//! See docs/action-matrix.md.
 //!
 //! Each design runs in a child process (this test binary, [`action_matrix_worker`]) so an action
 //! that hangs or aborts is recorded as such and the rest of the design still runs.
@@ -33,6 +35,10 @@ use solidcraft_engine::geom::Vec3;
 
 /// One in this many actions runs on pull requests (by a stable hash of its key).
 const PR_SAMPLE: u64 = 40;
+/// One in this many actions runs twice in the nightly determinism check, on the designs whose
+/// whole matrix takes at most [`DETERMINISM_MAX_SECS`] on a runner (most of the corpus).
+const DETERMINISM_SAMPLE: u64 = 8;
+const DETERMINISM_MAX_SECS: f64 = 300.0;
 /// Designs that take seconds to build: the full matrix only.
 const SLOW: [&str; 7] = [
     "oracle/41-tube-radial-holes",
@@ -623,17 +629,25 @@ fn action_matrix_worker() {
         out.flush().unwrap();
     };
     let Some((_, path)) = designs().into_iter().find(|(n, _)| *n == design) else { panic!("no design {design}") };
+    // A slice of a heavy design (`j/n`): the actions whose name hashes to `j` modulo `n` (by
+    // index, slices would take whole kinds of action when n shares a factor with the number of
+    // actions per entity, and a slice of only fillets runs three times as long); the first
+    // slice records the load.
+    let slice: Option<(usize, usize)> = std::env::var("ACTION_MATRIX_SLICE").ok().and_then(|v| parse_part(&v));
+    let first = slice.is_none_or(|(j, _)| j == 0);
     let t0 = Instant::now();
     let s = match load(&path) {
         Ok(s) => s,
         Err(e) => {
-            emit(outcome(&design, "design", "load", &Err(("load".into(), e)), t0.elapsed().as_millis()));
+            if first {
+                emit(outcome(&design, "design", "load", &Err(("load".into(), e)), t0.elapsed().as_millis()));
+            }
             emit(json!({"done": true}));
             return;
         }
     };
     let before = errors(&s);
-    if from == 0 {
+    if from == 0 && first {
         let r = if before.is_empty() { Ok(()) } else { Err(("timeline_error".into(), before.values().cloned().collect::<Vec<_>>().join("; "))) };
         emit(outcome(&design, "design", "load", &r, t0.elapsed().as_millis()));
     }
@@ -647,6 +661,9 @@ fn action_matrix_worker() {
         if only.as_ref().is_some_and(|o| !o.iter().any(|x| x == a.action)) {
             continue;
         }
+        if slice.is_some_and(|(j, n)| stable_hash(&format!("{} {}", a.entity, a.action)) % n as u64 != j as u64) {
+            continue;
+        }
         // Whole families of pieces are sampled together (see [`family`]).
         let k = key(&design, &family(&a.entity), a.action);
         if !stable_hash(&k).is_multiple_of(sample) {
@@ -708,46 +725,155 @@ fn action_matrix_worker() {
     emit(json!({"done": true, "actions": list.len()}));
 }
 
-/// Run the actions (one in `sample`) of every design in part `part` of `parts`, in child
-/// processes, `jobs` at a time.
-fn run_matrix(sample: u64, part: usize, parts: usize) -> Vec<Value> {
+/// `"k/n"` (1-based `k`) as `(k - 1, n)`.
+fn parse_part(v: &str) -> Option<(usize, usize)> {
+    let (k, n) = v.split_once('/')?;
+    let (k, n): (usize, usize) = (k.trim().parse().ok()?, n.trim().parse().ok()?);
+    (k >= 1 && k <= n).then_some((k - 1, n))
+}
+
+/// What one design costs on a CI runner (`tests/action_matrix/costs.json`, measured by the
+/// nightly shards): the whole run and the load before its first action, in seconds.
+#[derive(Clone, Copy, Debug)]
+struct Cost {
+    secs: f64,
+    load: f64,
+}
+
+fn costs_path() -> PathBuf {
+    root().join("tests/action_matrix/costs.json")
+}
+
+fn read_costs() -> BTreeMap<String, Cost> {
+    let Some(v) = std::fs::read(costs_path()).ok().and_then(|b| serde_json::from_slice::<Value>(&b).ok()) else { return BTreeMap::new() };
+    v["designs"]
+        .as_object()
+        .into_iter()
+        .flatten()
+        .map(|(d, c)| (d.clone(), Cost { secs: c["secs"].as_f64().unwrap_or(0.0), load: c["load"].as_f64().unwrap_or(0.0) }))
+        .collect()
+}
+
+/// A design new to the corpus (not in costs.json yet) is planned as this.
+const UNKNOWN_COST: Cost = Cost { secs: 120.0, load: 5.0 };
+/// A design that takes longer than this on a runner is split into slices of about this much, or
+/// five loads if that is more (each slice loads the design again), so no shard waits on one design.
+const SLICE_SECS: f64 = 600.0;
+/// Designs that take longer than this to load on a runner stay out of the pull request sample.
+const SAMPLE_MAX_LOAD_SECS: f64 = 8.0;
+
+/// A piece of work: one design, or a slice `(j, n)` of a heavy one, and what it should cost.
+#[derive(Clone, Debug)]
+struct Unit {
+    design: String,
+    slice: Option<(usize, usize)>,
+    cost: f64,
+}
+
+impl Unit {
+    fn name(&self) -> String {
+        match self.slice {
+            Some((j, n)) => format!("{} [{}/{n}]", self.design, j + 1),
+            None => self.design.clone(),
+        }
+    }
+}
+
+/// The whole matrix split into `parts` shards of about equal cost: heavy designs in slices, then
+/// the largest pieces first, each to the shard with the least so far. The same corpus and
+/// costs.json always give the same shards.
+fn plan(designs: &[String], costs: &BTreeMap<String, Cost>, parts: usize) -> Vec<Vec<Unit>> {
+    let mut units = Vec::new();
+    for d in designs {
+        let c = costs.get(d).copied().unwrap_or(UNKNOWN_COST);
+        let n = ((c.secs / SLICE_SECS.max(5.0 * c.load)).ceil() as usize).max(1);
+        if n == 1 {
+            units.push(Unit { design: d.clone(), slice: None, cost: c.secs });
+        } else {
+            let each = c.load + (c.secs - c.load).max(0.0) / n as f64;
+            units.extend((0..n).map(|j| Unit { design: d.clone(), slice: Some((j, n)), cost: each }));
+        }
+    }
+    units.sort_by(|a, b| b.cost.total_cmp(&a.cost).then_with(|| a.name().cmp(&b.name())));
+    let mut shards: Vec<(f64, Vec<Unit>)> = vec![(0.0, Vec::new()); parts];
+    for u in units {
+        let i = (0..parts).min_by(|&a, &b| shards[a].0.total_cmp(&shards[b].0)).unwrap_or(0);
+        shards[i].0 += u.cost;
+        shards[i].1.push(u);
+    }
+    shards.into_iter().map(|(_, v)| v).collect()
+}
+
+/// Every design's name, with `ACTION_MATRIX_FILTER` applied.
+fn design_names() -> Vec<String> {
     let designs = designs();
     assert!(designs.len() > 80, "only {} designs", designs.len());
-    let dir = std::env::temp_dir().join(format!("action-matrix-{}-{part}", std::process::id()));
+    let filter = std::env::var("ACTION_MATRIX_FILTER").ok();
+    designs.into_iter().map(|(n, _)| n).filter(|n| filter.as_deref().is_none_or(|f| n.contains(f))).collect()
+}
+
+/// Run the actions (one in `sample`) of these units in child processes, `jobs` at a time.
+///
+/// `ACTION_MATRIX_WORKDIR` keeps each design's raw outcomes there (one JSON line each, written as
+/// they come, so a run cut short still shows what it did) and `ACTION_MATRIX_TIMINGS` appends
+/// each unit's time to that file (JSON lines).
+fn run_units(sample: u64, queue: &[Unit]) -> Vec<Value> {
+    let keep = std::env::var_os("ACTION_MATRIX_WORKDIR").map(PathBuf::from);
+    static RUN: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
+    let run = RUN.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    let dir = keep.clone().unwrap_or_else(|| std::env::temp_dir().join(format!("action-matrix-{}-{run}", std::process::id())));
     std::fs::create_dir_all(&dir).unwrap();
+    let timings = std::env::var_os("ACTION_MATRIX_TIMINGS").map(PathBuf::from);
     let jobs: usize = std::env::var("ACTION_MATRIX_JOBS")
         .ok()
         .and_then(|v| v.parse().ok())
         .unwrap_or_else(|| std::thread::available_parallelism().map_or(2, |n| n.get()))
         .max(1);
-    let filter = std::env::var("ACTION_MATRIX_FILTER").ok();
-    let queue: Vec<(String, PathBuf)> = designs
-        .into_iter()
-        .enumerate()
-        .filter(|(i, (n, _))| i % parts == part && filter.as_deref().is_none_or(|f| n.contains(f)) && (sample == 1 || !SLOW.contains(&n.as_str())))
-        .map(|(_, d)| d)
-        .collect();
     let next = std::sync::atomic::AtomicUsize::new(0);
     let results = std::sync::Mutex::new(Vec::new());
     std::thread::scope(|scope| {
         for _ in 0..jobs {
             scope.spawn(|| {
-                while let Some((name, _)) = queue.get(next.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) {
-                    let out = dir.join(format!("{}.jsonl", name.replace('/', "__")));
-                    let lines = run_design(name, &out, sample);
+                while let Some(u) = queue.get(next.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) {
+                    let file = u.name().replace('/', "__").replace([' ', '[', ']'], "_");
+                    let out = dir.join(format!("{file}.r{run}.jsonl"));
+                    let t = Instant::now();
+                    let lines = run_design(&u.design, &out, sample, u.slice);
+                    let secs = t.elapsed().as_secs_f64();
+                    let load = lines.iter().find(|o| o["entity"] == "design" && o["action"] == "load").and_then(|o| o["ms"].as_f64()).unwrap_or(0.0) / 1000.0;
+                    if let Some(p) = &timings {
+                        let line = json!({"design": u.design, "slice": u.slice.map(|(j, n)| format!("{}/{n}", j + 1)), "secs": round3(secs), "load": round3(load), "actions": lines.len()});
+                        let mut f = std::fs::OpenOptions::new().create(true).append(true).open(p).unwrap();
+                        writeln!(f, "{line}").unwrap();
+                    }
+                    eprintln!("action matrix: {} took {secs:.1} s ({} outcomes, load {load:.1} s)", u.name(), lines.len());
                     results.lock().unwrap().extend(lines);
                 }
             });
         }
     });
-    let _ = std::fs::remove_dir_all(&dir);
+    if keep.is_none() {
+        let _ = std::fs::remove_dir_all(&dir);
+    }
     let mut v = results.into_inner().unwrap();
     v.sort_by_key(|o| key(o["design"].as_str().unwrap_or(""), o["entity"].as_str().unwrap_or(""), o["action"].as_str().unwrap_or("")));
     v
 }
 
+/// Run the actions (one in `sample`) of every design (the pull request sample leaves out the
+/// slow-loading ones).
+fn run_matrix(sample: u64) -> Vec<Value> {
+    let costs = read_costs();
+    let queue: Vec<Unit> = design_names()
+        .into_iter()
+        .filter(|n| sample == 1 || !(SLOW.contains(&n.as_str()) || costs.get(n).is_some_and(|c| c.load > SAMPLE_MAX_LOAD_SECS)))
+        .map(|design| Unit { design, slice: None, cost: 0.0 })
+        .collect();
+    run_units(sample, &queue)
+}
+
 /// One design in child processes, restarting after an action that hangs or aborts.
-fn run_design(design: &str, out: &Path, sample: u64) -> Vec<Value> {
+fn run_design(design: &str, out: &Path, sample: u64, slice: Option<(usize, usize)>) -> Vec<Value> {
     let _ = std::fs::remove_file(out);
     let mut from = 0usize;
     let mut results = Vec::new();
@@ -759,6 +885,7 @@ fn run_design(design: &str, out: &Path, sample: u64) -> Vec<Value> {
             .env("ACTION_MATRIX_OUT", out)
             .env("ACTION_MATRIX_FROM", from.to_string())
             .env("ACTION_MATRIX_SAMPLE", sample.to_string())
+            .env("ACTION_MATRIX_SLICE", slice.map(|(j, n)| format!("{}/{n}", j + 1)).unwrap_or_default())
             .stdout(Stdio::null())
             .stderr(Stdio::null())
             .spawn()
@@ -796,6 +923,9 @@ fn run_design(design: &str, out: &Path, sample: u64) -> Vec<Value> {
         }
         // It stopped inside action `last_start`: record it and go on after it.
         let Some(i) = last_start else {
+            if slice.is_some_and(|(j, _)| j > 0) {
+                break;
+            }
             extra.push(
                 json!({"design": design, "entity": "design", "action": "load", "ok": false, "kind": if timed_out { "timeout" } else { "crash" },
                               "message": format!("the design did not load ({status:?})")}),
@@ -863,9 +993,12 @@ fn write_baseline(results: &[Value]) {
             m.entry(design).or_default().entry(entity).or_default().extend(actions.as_str().unwrap_or("").split_whitespace().map(str::to_string));
         }
     }
+    // A timeout of an action that passed keeps it passing (see [`ratchet`]).
+    let base = read_baseline();
     for o in results {
         let (design, entity, action) = (o["design"].as_str().unwrap_or(""), o["entity"].as_str().unwrap_or(""), o["action"].as_str().unwrap_or(""));
-        let mark = if o["ok"].as_bool() == Some(true) { "" } else { "!" };
+        let kept = o["kind"] == "timeout" && base.get(&key(design, entity, action)) == Some(&true);
+        let mark = if o["ok"].as_bool() == Some(true) || kept { "" } else { "!" };
         m.entry(design).or_default().entry(entity).or_default().push(format!("{mark}{action}"));
     }
     // One entity per line, sorted: small diffs when a few actions change.
@@ -894,10 +1027,18 @@ fn ratchet(results: &[Value], full: bool) {
     // baseline, over the same names this run tried, and in this run.
     let mut fam: BTreeMap<String, (usize, usize, Vec<String>, Vec<String>)> = BTreeMap::new();
     let mut fresh = 0usize;
+    // An action that passed in the baseline and now runs past ACTION_TIMEOUT is listed but not
+    // judged: on a busy runner the slowest actions (40-80 s on oracle 41's tube) land either side
+    // of the limit from one night to the next. An error or a crash is judged as usual.
+    let mut slow = Vec::new();
     for o in results {
         let (design, entity, action) = (o["design"].as_str().unwrap_or(""), o["entity"].as_str().unwrap_or(""), o["action"].as_str().unwrap_or(""));
         let k = key(design, entity, action);
         let ok = o["ok"].as_bool() == Some(true);
+        if o["kind"] == "timeout" && base.get(&k) == Some(&true) {
+            slow.push(k);
+            continue;
+        }
         let f = fam.entry(key(design, &family(entity), action)).or_default();
         match base.get(&k) {
             Some(was) => {
@@ -934,6 +1075,14 @@ fn ratchet(results: &[Value], full: bool) {
         eprintln!("baseline written: {}", baseline_path().display());
         return;
     }
+    if !slow.is_empty() {
+        eprintln!(
+            "{} actions that pass in the baseline ran past the {} s limit this time (not judged):\n{}",
+            slow.len(),
+            ACTION_TIMEOUT.as_secs(),
+            slow.join("\n")
+        );
+    }
     if !fixed.is_empty() || fresh > 0 {
         eprintln!(
             "{} actions pass now that failed in the baseline, and {fresh} are not in it: record them with `cargo xtask action-matrix --bless`:\n{}",
@@ -952,51 +1101,156 @@ fn ratchet(results: &[Value], full: bool) {
 /// Pull requests: a fixed sample of the matrix (one action in [`PR_SAMPLE`]) against the ratchet.
 #[test]
 fn action_matrix_sample() {
-    let results = run_matrix(PR_SAMPLE, 0, 1);
+    let results = run_matrix(PR_SAMPLE);
     assert!(results.len() > 300, "only {} actions ran", results.len());
     ratchet(&results, false);
 }
 
-/// The whole matrix against the ratchet, in four parts that the nightly CI job runs side by side
-/// (`--run-ignored only`).
-fn full_part(part: usize) {
-    let results = run_matrix(1, part, 4);
+/// One shard of the whole matrix against the ratchet: `ACTION_MATRIX_SHARD=k/n` (the nightly
+/// "Action matrix" CI jobs run the n shards side by side, `--run-ignored only`). Shards are cut by
+/// what each design costs (tests/action_matrix/costs.json), see [`plan`].
+/// `ACTION_MATRIX_SHARD_LIST=1` only prints the plan.
+#[test]
+#[ignore = "heavy: the nightly Action matrix CI jobs run it, one shard each"]
+fn action_matrix_shard() {
+    let Some((k, n)) = std::env::var("ACTION_MATRIX_SHARD").ok().and_then(|v| parse_part(&v)) else { panic!("set ACTION_MATRIX_SHARD=k/n") };
+    let shards = plan(&design_names(), &read_costs(), n);
+    for (i, sh) in shards.iter().enumerate() {
+        let total: f64 = sh.iter().map(|u| u.cost).sum();
+        eprintln!("shard {}/{n}: {} pieces, about {:.0} s of work{}", i + 1, sh.len(), total, if i == k { " (this one)" } else { "" });
+    }
+    eprintln!("this shard: {}", shards[k].iter().map(Unit::name).collect::<Vec<_>>().join(", "));
+    if std::env::var_os("ACTION_MATRIX_SHARD_LIST").is_some() {
+        return;
+    }
+    let results = run_units(1, &shards[k]);
     ratchet(&results, false);
 }
 
+/// Determinism: a sample of the matrix (one family in [`DETERMINISM_SAMPLE`], the heaviest
+/// designs left out) run twice must give the same outcome for every action. Lists each action
+/// whose outcome changed between the runs.
 #[test]
-#[ignore = "heavy: the nightly Action matrix CI job runs it"]
-fn action_matrix_full_1() {
-    full_part(0);
-}
-
-#[test]
-#[ignore = "heavy: the nightly Action matrix CI job runs it"]
-fn action_matrix_full_2() {
-    full_part(1);
-}
-
-#[test]
-#[ignore = "heavy: the nightly Action matrix CI job runs it"]
-fn action_matrix_full_3() {
-    full_part(2);
-}
-
-#[test]
-#[ignore = "heavy: the nightly Action matrix CI job runs it"]
-fn action_matrix_full_4() {
-    full_part(3);
+#[ignore = "heavy: the nightly Action matrix determinism CI job runs it"]
+fn action_matrix_determinism() {
+    let outcomes = |v: Vec<Value>| -> BTreeMap<String, String> {
+        v.into_iter()
+            .map(|o| {
+                let k = key(o["design"].as_str().unwrap_or(""), o["entity"].as_str().unwrap_or(""), o["action"].as_str().unwrap_or(""));
+                let r = if o["ok"].as_bool() == Some(true) {
+                    "pass".to_string()
+                } else {
+                    format!("{}: {}", o["kind"].as_str().unwrap_or(""), o["message"].as_str().unwrap_or(""))
+                };
+                (k, r)
+            })
+            .collect()
+    };
+    let costs = read_costs();
+    let queue: Vec<Unit> = design_names()
+        .into_iter()
+        .filter(|n| costs.get(n).copied().unwrap_or(UNKNOWN_COST).secs <= DETERMINISM_MAX_SECS)
+        .map(|design| Unit { design, slice: None, cost: 0.0 })
+        .collect();
+    let a = outcomes(run_units(DETERMINISM_SAMPLE, &queue));
+    let b = outcomes(run_units(DETERMINISM_SAMPLE, &queue));
+    let keys: BTreeSet<&String> = a.keys().chain(b.keys()).collect();
+    let mut changed = Vec::new();
+    // Pieces that share a name can swap numbers between runs (#569): compare passes per family
+    // too, and report a family whose pass count differs.
+    let mut fam: BTreeMap<String, (usize, usize)> = BTreeMap::new();
+    for k in &keys {
+        let (x, y) = (a.get(*k).map_or("missing", String::as_str), b.get(*k).map_or("missing", String::as_str));
+        let mut parts = k.splitn(3, " :: ");
+        let (d, e, act) = (parts.next().unwrap_or(""), parts.next().unwrap_or(""), parts.next().unwrap_or(""));
+        let f = fam.entry(key(d, &family(e), act)).or_default();
+        f.0 += usize::from(x == "pass");
+        f.1 += usize::from(y == "pass");
+        if x != y {
+            changed.push(format!("{k}\n    first:  {x}\n    second: {y}"));
+        }
+    }
+    let families: Vec<String> = fam.into_iter().filter(|(_, (x, y))| x != y).map(|(k, (x, y))| format!("{k}: {x} then {y} pass")).collect();
+    eprintln!(
+        "action matrix determinism: {} actions, {} changed between two runs, {} families pass a different number of times",
+        keys.len(),
+        changed.len(),
+        families.len()
+    );
+    for c in &changed {
+        eprintln!("  {c}");
+    }
+    assert!(
+        families.is_empty(),
+        "{} families of actions pass a different number of times in two runs of the same build:\n{}",
+        families.len(),
+        families.join("\n")
+    );
 }
 
 /// The whole matrix at once (`cargo xtask action-matrix`): `ACTION_MATRIX_BLESS=1` rewrites the
 /// baseline; `ACTION_MATRIX_RESULTS=file.json` keeps every outcome with its message.
+/// `ACTION_MATRIX_MERGE=DIR` runs nothing and takes the outcomes from the `results-*.json` files
+/// in DIR instead (the nightly shards' artifacts), and the timings from its `timings-*.jsonl`
+/// files, which `ACTION_MATRIX_BLESS` then also writes to tests/action_matrix/costs.json.
 #[test]
-#[ignore = "cargo xtask action-matrix runs it (CI runs it as the four action_matrix_full parts)"]
+#[ignore = "cargo xtask action-matrix runs it (CI runs it as the action_matrix_shard jobs)"]
 fn action_matrix_all() {
-    let results = run_matrix(1, 0, 1);
+    let Some(dir) = std::env::var_os("ACTION_MATRIX_MERGE").map(PathBuf::from) else {
+        let results = run_matrix(1);
+        ratchet(&results, true);
+        return;
+    };
+    let mut results: Vec<Value> = Vec::new();
+    let mut timings: Vec<Value> = Vec::new();
+    let mut files: Vec<PathBuf> = std::fs::read_dir(&dir).unwrap().flatten().map(|e| e.path()).collect();
+    files.sort();
+    for f in files {
+        let name = f.file_name().unwrap_or_default().to_string_lossy().to_string();
+        let text = std::fs::read_to_string(&f).unwrap_or_default();
+        if name.starts_with("results-") && name.ends_with(".json") {
+            results.extend(serde_json::from_str::<Vec<Value>>(&text).unwrap_or_else(|e| panic!("{name}: {e}")));
+        } else if name.starts_with("timings-") {
+            timings.extend(text.lines().filter_map(|l| serde_json::from_str(l).ok()));
+        }
+    }
+    let ran: BTreeSet<&str> = results.iter().filter_map(|o| o["design"].as_str()).collect();
+    let missing: Vec<String> = design_names().into_iter().filter(|d| !ran.contains(d.as_str())).collect();
+    assert!(missing.is_empty(), "no results for {}", missing.join(", "));
+    results.sort_by_key(|o| key(o["design"].as_str().unwrap_or(""), o["entity"].as_str().unwrap_or(""), o["action"].as_str().unwrap_or("")));
+    if std::env::var_os("ACTION_MATRIX_BLESS").is_some() && !timings.is_empty() {
+        write_costs(&timings);
+    }
     ratchet(&results, true);
 }
 
+/// tests/action_matrix/costs.json from the shards' timings: per design, its whole time (a sliced
+/// design's slices added up, less the loads all but one of them repeat) and its load.
+fn write_costs(timings: &[Value]) {
+    let mut m: BTreeMap<String, (f64, f64, usize)> = BTreeMap::new();
+    for t in timings {
+        let e = m.entry(t["design"].as_str().unwrap_or("").to_string()).or_default();
+        e.0 += t["secs"].as_f64().unwrap_or(0.0);
+        e.1 = e.1.max(t["load"].as_f64().unwrap_or(0.0));
+        e.2 += 1;
+    }
+    // Designs these timings don't cover keep their old costs.
+    for (d, c) in read_costs() {
+        m.entry(d).or_insert((c.secs, c.load, 1));
+    }
+    let mut s = String::from(
+        "{\n \"about\": \"What each action matrix design costs on a CI runner, in seconds (secs: the whole design, load: before its first action), for cutting the nightly shards (apps/solidcraft-cli/tests/action_matrix.rs, plan). Written with the baseline by cargo xtask action-matrix --bless --from DIR.\",\n \"designs\": {\n",
+    );
+    let n = m.len();
+    for (i, (d, (secs, load, pieces))) in m.into_iter().enumerate() {
+        let secs = (secs - load * pieces.saturating_sub(1) as f64).max(load);
+        s += &format!("  {}: {{\"secs\": {:.1}, \"load\": {:.1}}}{}\n", json!(d), secs, load, if i + 1 < n { "," } else { "" });
+    }
+    s += " }\n}\n";
+    std::fs::write(costs_path(), s).unwrap();
+    eprintln!("costs written: {}", costs_path().display());
+}
+
 // ---------------------------------------------------------------------------------------------
 // The matrix's causes: one small reproduction per open issue, ignored until it is fixed (the
 // issue says how many matrix actions it accounts for). Each runs commands as the desktop sends

docs/action-matrix.md

@@ -9,11 +9,21 @@ The test is [`apps/solidcraft-cli/tests/action_matrix.rs`](../apps/solidcraft-cl
 ## Running it
 
 - **Every CI run** (pull requests too) checks a fixed sample, one action in 40 by a stable hash of
-  its name, on every design except the seven slowest to build (`action_matrix_sample`, under a
-  minute).
-- **Nightly** (and by hand, Run workflow), the "Action matrix" job runs the whole matrix in four
-  parts (`action_matrix_full_1` to `_4`).
-- **Locally**: `cargo xtask action-matrix` runs it whole (about 22 minutes with six designs at a time on a 16-core machine). Add `--bless` to
+  its name, on every design except the slowest to build (the `SLOW` list, and any design that
+  takes more than 8 s to load in costs.json; `action_matrix_sample`, a few minutes).
+- **Nightly** (and by hand, Run workflow), the "Action matrix" jobs run the whole matrix in 16
+  shards (`action_matrix_shard` with `ACTION_MATRIX_SHARD=k/16`) of about equal cost, cut by
+  [`tests/action_matrix/costs.json`](../tests/action_matrix/costs.json): what each design took on a
+  runner. A design that takes more than ten minutes is split into slices (actions by a hash of their name, each
+  slice loading the design again), and the pieces go largest first to the shard with the least
+  work. A design not in costs.json yet counts as two minutes. Each shard uploads its outcomes and
+  timings (artifact `action-matrix-k`); with all sixteen in one folder,
+  `cargo xtask action-matrix --from DIR --bless` writes the baseline and costs.json from them
+  without running anything. `ACTION_MATRIX_SHARD=k/n ACTION_MATRIX_SHARD_LIST=1` prints the plan.
+- **Determinism** (nightly, "Action matrix determinism"): `action_matrix_determinism` runs one
+  action in 8 of every design that takes at most five minutes twice on one runner and lists every action whose outcome changed; it fails when a
+  family of pieces passes a different number of times.
+- **Locally**: `cargo xtask action-matrix` runs it whole (a few hours on one machine now; the nightly shards' artifacts and `--from` are quicker). Add `--bless` to
   rewrite the baseline after a fix (newly passing actions are listed when you don't), and
   `--results FILE` to keep every outcome with its message and the command that was sent.
   `ACTION_MATRIX_FILTER=oracle/23` runs only the designs whose name contains it;
@@ -28,7 +38,9 @@ The test is [`apps/solidcraft-cli/tests/action_matrix.rs`](../apps/solidcraft-cl
   (from `apps/solidcraft-cli`).
 
 Each design runs in its own process, so an action that hangs (90 s without a result) or aborts is
-recorded as `timeout` or `crash` and the rest of the design still runs. Loading a design, before
+recorded as `timeout` or `crash` and the rest of the design still runs. An action that passes in the baseline and
+times out is listed but not judged (and stays passing when blessed): the slowest actions take 40 to
+80 s on a busy runner and land either side of the limit from night to night. Loading a design, before
 its first action, may take up to 300 s (the largest take several times longer on a loaded CI
 runner than here). None did in this run.
 

tests/action_matrix/costs.json

@@ -0,0 +1,124 @@
+{
+ "about": "What each action matrix design costs on a CI runner, in seconds (secs: the whole design, load: before its first action), for cutting the nightly shards (apps/solidcraft-cli/tests/action_matrix.rs, plan). Written with the baseline by cargo xtask action-matrix --bless --from DIR.",
+ "designs": {
+  "corpus/components-arm": {"secs": 48.0, "load": 0.3},
+  "corpus/issue-489-stacked-join": {"secs": 2.8, "load": 0.0},
+  "corpus/issue-616-lid-ribs": {"secs": 257.6, "load": 0.7},
+  "corpus/joints-hinge": {"secs": 23.4, "load": 0.0},
+  "corpus/m0-box-fillet-hole": {"secs": 25.2, "load": 0.1},
+  "corpus/m0-revolve-primitives": {"secs": 86.9, "load": 0.5},
+  "corpus/modern-path-pattern": {"secs": 73.6, "load": 0.4},
+  "corpus/modern-plastic": {"secs": 617.5, "load": 1.4},
+  "corpus/modern-sheet-metal": {"secs": 47.9, "load": 0.0},
+  "corpus/move-bodies": {"secs": 2.7, "load": 0.0},
+  "corpus/naming-split-pieces": {"secs": 15.6, "load": 0.1},
+  "corpus/split-body-pieces": {"secs": 12.4, "load": 0.1},
+  "demo/crazy-geom-cuts": {"secs": 21.4, "load": 0.1},
+  "demo/enclosure_steps": {"secs": 655.0, "load": 15.4},
+  "demo/logo-up-down-join": {"secs": 5.8, "load": 0.0},
+  "demo/pi-case": {"secs": 3753.2, "load": 5.2},
+  "demo/pi-case-0.8": {"secs": 13979.6, "load": 10.8},
+  "examples/bracket": {"secs": 14.2, "load": 0.1},
+  "examples/sensor_enclosure": {"secs": 8289.0, "load": 16.3},
+  "oracle/01-box": {"secs": 0.9, "load": 0.0},
+  "oracle/02-cylinder": {"secs": 1.4, "load": 0.0},
+  "oracle/03-box-through-hole": {"secs": 18.8, "load": 0.0},
+  "oracle/04-revolve-vase": {"secs": 8.0, "load": 0.0},
+  "oracle/05-taper-extrude": {"secs": 5.4, "load": 0.0},
+  "oracle/06-symmetric-extrude": {"secs": 0.9, "load": 0.0},
+  "oracle/07-fillet-all-edges": {"secs": 54.0, "load": 0.2},
+  "oracle/08-chamfer": {"secs": 9.4, "load": 0.1},
+  "oracle/09-shell": {"secs": 2.2, "load": 0.0},
+  "oracle/10-rect-pattern-holes": {"secs": 135.3, "load": 0.3},
+  "oracle/11-circular-pattern-flange": {"secs": 24.2, "load": 0.3},
+  "oracle/12-mirror-feature": {"secs": 30.5, "load": 0.2},
+  "oracle/13-combine-join": {"secs": 29.3, "load": 0.1},
+  "oracle/14-combine-cut": {"secs": 123.6, "load": 2.4},
+  "oracle/15-combine-intersect": {"secs": 41.4, "load": 12.3},
+  "oracle/16-sweep": {"secs": 29.7, "load": 0.0},
+  "oracle/17-loft": {"secs": 33.6, "load": 0.0},
+  "oracle/18-hole-features": {"secs": 63.5, "load": 1.8},
+  "oracle/19-l-bracket-constrained": {"secs": 1.6, "load": 0.0},
+  "oracle/20-slot-constrained": {"secs": 3.0, "load": 0.0},
+  "oracle/21-draft": {"secs": 5.4, "load": 0.0},
+  "oracle/22-split-body": {"secs": 392.0, "load": 4.4},
+  "oracle/23-bracket-multifeature": {"secs": 149.8, "load": 4.3},
+  "oracle/24-user-parameters": {"secs": 13.4, "load": 0.0},
+  "oracle/25-sketch-triangle": {"secs": 4.9, "load": 0.0},
+  "oracle/26-sketch-belt": {"secs": 9.3, "load": 0.0},
+  "oracle/27-sketch-trapezoid": {"secs": 32.9, "load": 0.0},
+  "oracle/28-blind-pocket-fillets": {"secs": 143.7, "load": 0.6},
+  "oracle/29-two-sided-and-partial-revolve": {"secs": 48.5, "load": 2.1},
+  "oracle/30-fillet-after-shell": {"secs": 151.5, "load": 1.0},
+  "oracle/31-fillet-l-all-edges": {"secs": 129.9, "load": 2.2},
+  "oracle/32-boss-plate-fillets": {"secs": 122.0, "load": 0.4},
+  "oracle/33-fillet-tangent-chain": {"secs": 86.5, "load": 0.3},
+  "oracle/34-chamfer-then-fillet": {"secs": 71.0, "load": 0.3},
+  "oracle/35-fillet-curved-curved": {"secs": 38.8, "load": 19.0},
+  "oracle/36-corner-blend-two-radii": {"secs": 27.5, "load": 0.1},
+  "oracle/37-pyramid-apex-blend": {"secs": 18.3, "load": 0.1},
+  "oracle/38-coincident-cylinders": {"secs": 3.2, "load": 1.2},
+  "oracle/39-tee-pipe": {"secs": 880.7, "load": 26.6},
+  "oracle/40-box-sphere-torus": {"secs": 352.8, "load": 9.4},
+  "oracle/41-tube-radial-holes": {"secs": 18429.5, "load": 134.7},
+  "oracle/42-overlapping-patterns": {"secs": 1466.4, "load": 2.4},
+  "oracle/43-cut-splits-body": {"secs": 3.3, "load": 0.0},
+  "oracle/44-shell-l-bracket": {"secs": 3.7, "load": 0.0},
+  "oracle/45-shell-boss-plate": {"secs": 83.9, "load": 0.8},
+  "oracle/46-shell-two-open-faces": {"secs": 2.1, "load": 0.0},
+  "oracle/47-draft-curved-pocket": {"secs": 30.7, "load": 0.1},
+  "oracle/48-pulley": {"secs": 138.2, "load": 0.1},
+  "oracle/49-helical-sweep": {"secs": 715.3, "load": 0.3},
+  "oracle/50-sweep-tight-bend": {"secs": 24.9, "load": 0.0},
+  "oracle/51-loft-rect-circle-offset": {"secs": 22.0, "load": 0.0},
+  "oracle/52-circular-pattern-fillet": {"secs": 143.8, "load": 8.3},
+  "oracle/53-path-pattern": {"secs": 50.3, "load": 0.3},
+  "oracle/54-mirror-body-fillets": {"secs": 17.6, "load": 0.1},
+  "oracle/55-tapped-hole-cosmetic": {"secs": 29.4, "load": 0.8},
+  "oracle/56-holes-on-curved-face": {"secs": 721.8, "load": 12.0},
+  "oracle/57-hole-pattern-sketch-points": {"secs": 168.2, "load": 1.5},
+  "oracle/58-phone-stand": {"secs": 45.6, "load": 0.5},
+  "oracle/59-enclosure-base": {"secs": 274.7, "load": 3.3},
+  "oracle/60-pipe-flange": {"secs": 213.4, "load": 13.6},
+  "oracle/61-shaft-keyway": {"secs": 1938.2, "load": 11.1},
+  "oracle/62-heat-sink": {"secs": 54.7, "load": 0.7},
+  "oracle/63-hinge-knuckle": {"secs": 107.9, "load": 0.4},
+  "oracle/64-sketch-tangent-arc-chain": {"secs": 36.5, "load": 0.0},
+  "oracle/65-sketch-gear-tooth": {"secs": 18.7, "load": 0.0},
+  "oracle/66-sketch-linkage": {"secs": 24.2, "load": 0.0},
+  "oracle/67-sm-base-flange": {"secs": 11.4, "load": 0.0},
+  "oracle/68-sm-edge-flange": {"secs": 261.8, "load": 0.0},
+  "oracle/69-sm-tray": {"secs": 1449.7, "load": 0.1},
+  "oracle/70-sm-flange-45": {"secs": 194.4, "load": 0.0},
+  "oracle/71-sm-custom-rule": {"secs": 200.7, "load": 0.0},
+  "oracle/72-sm-contour-flange": {"secs": 63.6, "load": 0.0},
+  "oracle/73-sm-hem": {"secs": 465.7, "load": 0.0},
+  "oracle/74-sm-unfold-refold": {"secs": 205.2, "load": 0.0},
+  "oracle/75-sm-bracket-holes": {"secs": 672.8, "load": 0.7},
+  "oracle/76-sm-thickness-variant": {"secs": 171.0, "load": 0.1},
+  "oracle/77-fillet-variable-linear": {"secs": 5.6, "load": 0.0},
+  "oracle/78-fillet-variable-to-zero": {"secs": 8.6, "load": 0.0},
+  "oracle/79-fillet-chord-length": {"secs": 5.7, "load": 0.0},
+  "oracle/80-fillet-taper-to-fit": {"secs": 25.5, "load": 0.0},
+  "oracle/81-fillet-spline-radius": {"secs": 82.2, "load": 0.0},
+  "oracle/82-thin-extrude-frame": {"secs": 4.2, "load": 0.0},
+  "regressions/blind_hole_in_long_d_profile": {"secs": 54.3, "load": 0.3},
+  "regressions/join_struts_on_bore_wall": {"secs": 333.4, "load": 0.7},
+  "regressions/lid_ribs_flush_with_window": {"secs": 234.8, "load": 0.7},
+  "regressions/loft_cut_beside_round_hole": {"secs": 79.4, "load": 0.3},
+  "regressions/oblique_cut_of_cylinder": {"secs": 162.2, "load": 0.1},
+  "regressions/oblique_cut_of_cylinder_hang": {"secs": 244.9, "load": 0.4},
+  "regressions/partial_face_contact_pattern": {"secs": 269.4, "load": 1.7},
+  "regressions/ring_cut_below_recess_thin_floor": {"secs": 7.4, "load": 0.1},
+  "regressions/shallow_loft_cut_from_top_face": {"secs": 40.8, "load": 1.3},
+  "typical/filleted-shelled-box": {"secs": 43.4, "load": 0.4},
+  "typical/holes-pattern-chamfer": {"secs": 525.7, "load": 3.1},
+  "typical/loft": {"secs": 69.9, "load": 0.3},
+  "typical/multi-body-join-cut": {"secs": 39.1, "load": 0.1},
+  "typical/revolved-part": {"secs": 12.1, "load": 0.0},
+  "typical/sheet-metal-bracket": {"secs": 140.1, "load": 0.8},
+  "typical/stacked-extrude": {"secs": 37.1, "load": 0.5},
+  "typical/sweep": {"secs": 57.9, "load": 0.0},
+  "typical/up-down-join": {"secs": 4.9, "load": 0.0}
+ }
+}

xtask/src/main.rs

@@ -35,10 +35,12 @@ commands:
                   replay the reference tests (tests/oracle/*/recipe.json), compare with each
                   case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
                   already built, instead of cargo run --release)
-  action-matrix [--bless] [--results FILE]
+  action-matrix [--bless] [--results FILE] [--from DIR]
                   try every ordinary action on every face, edge, vertex and body of the design
                   corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;
-                  --results keeps every outcome with its message). See docs/action-matrix.md
+                  --results keeps every outcome with its message; --from takes the outcomes of the
+                  nightly shards' artifacts in DIR instead of running, and --bless then also
+                  rewrites tests/action_matrix/costs.json). See docs/action-matrix.md
   book [--no-build]
                   generate docs/book's command reference and screenshots, then mdbook build
                   into target/book
@@ -71,7 +73,8 @@ fn main() -> ExitCode {
         Some("i18n") => i18n::run(&root(), &rest),
         Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()),
         Some("action-matrix") => {
-            cmd_action_matrix(rest.contains(&"--bless"), rest.iter().position(|a| *a == "--results").and_then(|i| rest.get(i + 1)).copied())
+            let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();
+            cmd_action_matrix(rest.contains(&"--bless"), arg("--results"), arg("--from"))
         }
         Some("book") => book::run(&root(), !rest.contains(&"--no-build")),
         Some("licences") => licences::run(&root(), rest.contains(&"--check")),
@@ -128,7 +131,7 @@ pub fn metadata() -> Result<serde_json::Value, String> {
 }
 
 /// The full action matrix (apps/solidcraft-cli/tests/action_matrix.rs), with the ci profile.
-fn cmd_action_matrix(bless: bool, results: Option<&str>) -> Result<(), String> {
+fn cmd_action_matrix(bless: bool, results: Option<&str>, from: Option<&str>) -> Result<(), String> {
     let mut c = cargo();
     c.args([
         "test",
@@ -151,6 +154,10 @@ fn cmd_action_matrix(bless: bool, results: Option<&str>) -> Result<(), String> {
         let p = std::path::absolute(r).map_err(|e| format!("{r}: {e}"))?;
         c.env("ACTION_MATRIX_RESULTS", p);
     }
+    if let Some(d) = from {
+        let p = std::path::absolute(d).map_err(|e| format!("{d}: {e}"))?;
+        c.env("ACTION_MATRIX_MERGE", p);
+    }
     run(c, "cargo test --profile ci -p solidcraft-cli --test action_matrix action_matrix_all")
 }