Deterministic CPU multithreading: supernodal Cholesky, parallel elements, LBM blocks (#50)

* parallel: deterministic thread pool helpers for the solvers (fixed partitions, ordered results)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* solver: supernodal left-looking Cholesky, multithreaded with a fixed operation order

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* solver: parallel element matrices, stresses, Lanczos block solves and products (sums in the sequential order)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* explicit: element updates in parallel, nodal forces added in element order (bit-identical to the sequential loop)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* lbm: collision and streaming by fixed cell blocks in parallel, wall forces in link order (bit-identical)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* engine: threads setting for the solve commands (session default from MODELCRAFT_THREADS or the cores); bit-identity test on 1, 2 and 7 threads

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

* docs: rayon threads in ADR 0001, the parallel crate, supernodal Cholesky

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01NmAttHhUZTokgPeEJia8sm

---------

Co-authored-by: jyviko <[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>

crates/solver/src/analysis.rs

@@ -9,6 +9,8 @@ use modelcraft_fem::{DofKind, DofLayout, PointStress, von_mises};
 use modelcraft_model::{ElementKind, Model, Procedure, Resolved, Step};
 use serde::Serialize;
 
+use modelcraft_parallel as parallel;
+
 use crate::cholesky::{Factor, FactorError, UpperColumns};
 use crate::constraints::{Budget, Reduction, coupling_rows};
 use crate::eigen::{Operator, Wanted, lanczos};
@@ -164,21 +166,46 @@ struct Assembled {
     m: Option<Csr>,
 }
 
+/// Elements whose matrices are computed together before they are added.
+const ROUND: usize = 4096;
+
+/// Elements per parallel task.
+const GRAIN: usize = 16;
+
 /// Assemble the stiffness (and the mass when `mass` is set) on the mesh's node graph.
 fn assemble(mesh: &Mesh, graph: &NodeGraph, mass: Option<MassKind>) -> Result<Assembled> {
     let layout = &mesh.layout;
     let mut k = graph.zero_matrix(layout);
     let mut m = mass.map(|_| graph.zero_matrix(layout));
-    for e in &mesh.elements {
-        let kernel = e.kernel()?;
-        let kinds = kernel.dofs().node;
-        let at = |err| Error::element(e.label, err);
-        add_block(&mut k, layout, &e.nodes, kinds, &kernel.stiffness(&e.d).map_err(at)?.data)?;
-        if let (Some(mm), Some(kind)) = (m.as_mut(), mass) {
-            let rho = e.density.ok_or_else(|| Error::Invalid(format!("element {} has no density", e.label)))?;
-            match kind {
-                MassKind::Consistent => add_scalar(mm, layout, &e.nodes, kinds, &kernel.mass(rho).map_err(at)?.data)?,
-                MassKind::Lumped => add_diagonal(mm, layout, &e.nodes, kinds, &kernel.lumped_mass(rho).map_err(at)?)?,
+    // Element matrices are computed in parallel, a round at a time, and added in label order
+    // (the sums are those of a sequential loop).
+    for round in parallel::blocks(mesh.elements.len(), ROUND) {
+        let elements = mesh.elements.get(round).unwrap_or_default();
+        let matrices = parallel::map_grained(elements.len(), GRAIN, |i| -> Result<_> {
+            let Some(e) = elements.get(i) else { return Err(Error::Internal("element index".into())) };
+            let kernel = e.kernel()?;
+            let at = |err| Error::element(e.label, err);
+            let stiffness = kernel.stiffness(&e.d).map_err(at)?.data;
+            let element_mass = match mass {
+                None => None,
+                Some(kind) => {
+                    let rho = e.density.ok_or_else(|| Error::Invalid(format!("element {} has no density", e.label)))?;
+                    Some(match kind {
+                        MassKind::Consistent => kernel.mass(rho).map_err(at)?.data,
+                        MassKind::Lumped => kernel.lumped_mass(rho).map_err(at)?,
+                    })
+                }
+            };
+            Ok((kernel.dofs().node, stiffness, element_mass))
+        });
+        for (e, computed) in elements.iter().zip(matrices) {
+            let (kinds, stiffness, element_mass) = computed?;
+            add_block(&mut k, layout, &e.nodes, kinds, &stiffness)?;
+            if let (Some(mm), Some(kind), Some(data)) = (m.as_mut(), mass, element_mass) {
+                match kind {
+                    MassKind::Consistent => add_scalar(mm, layout, &e.nodes, kinds, &data)?,
+                    MassKind::Lumped => add_diagonal(mm, layout, &e.nodes, kinds, &data)?,
+                }
             }
         }
     }
@@ -224,6 +251,8 @@ fn factor_error(e: FactorError, mesh: &Mesh, reduction: &Reduction) -> Error {
         FactorError::TooLarge { entries, limit } => {
             Error::Limit(format!("the factorization needs {entries} entries; the limit is {limit} (refine less or raise the limit)"))
         }
+        FactorError::OutOfMemory { entries } => Error::Limit(format!("not enough memory for the factorization ({entries} entries)")),
+        FactorError::Internal(what) => Error::Internal(what),
         FactorError::NotPositiveDefinite { column } => {
             let at = reduction.eq_dof.get(column).map_or_else(|| "an unknown degree of freedom".to_string(), |d| mesh.describe_dof(*d));
             Error::Singular(format!(
@@ -370,14 +399,18 @@ pub fn solve_static(model: &Model, step: Option<&str>, limits: &Limits) -> Resul
     let mut elements = Vec::with_capacity(mesh.elements.len());
     let mut sum = vec![[0.0; 6]; mesh.labels.len()];
     let mut count = vec![0u32; mesh.labels.len()];
-    for e in &mesh.elements {
+    let all_points = parallel::map_grained(mesh.elements.len(), GRAIN, |i| -> Result<_> {
+        let Some(e) = mesh.elements.get(i) else { return Err(Error::Internal("element index".into())) };
         let kernel = e.kernel()?;
-        let at = |err| Error::element(e.label, err);
         let ue: Vec<[f64; 3]> = e.nodes.iter().map(|n| displacement.get(*n).copied().unwrap_or_default()).collect();
         let thermal = field.as_deref().and_then(|t| e.thermal_strain(t));
-        let points = kernel.stresses_with(&e.d, &ue, thermal.as_ref()).map_err(at)?;
+        let points = kernel.stresses_with(&e.d, &ue, thermal.as_ref()).map_err(|err| Error::element(e.label, err))?;
+        Ok((points, kernel.ty(), kernel.rule()))
+    });
+    for (e, computed) in mesh.elements.iter().zip(all_points) {
+        let (points, ty, rule) = computed?;
+        let at = |err| Error::element(e.label, err);
         let values: Vec<[f64; 6]> = points.iter().map(|p| p.stress).collect();
-        let (ty, rule) = (kernel.ty(), kernel.rule());
         let cached = extrapolations.iter().position(|(t, r, _)| std::ptr::eq(*t, ty) && *r == rule);
         let index = match cached {
             Some(i) => i,

crates/solver/src/eigen.rs

@@ -184,7 +184,7 @@ pub struct Wanted {
 }
 
 /// The operators the Lanczos iteration needs.
-pub trait Operator {
+pub trait Operator: Sync {
     fn n(&self) -> usize;
     /// y = M x.
     fn mass(&self, x: &[f64], y: &mut [f64]);
@@ -285,10 +285,12 @@ pub fn lanczos(op: &impl Operator, shift: f64, wanted: Wanted, block: usize, max
         let j0 = q.len();
         // W = Op Q_j.
         let mut w: Vec<Vec<f64>> = Vec::with_capacity(b);
-        for mx in &mcur {
-            let mut y = mx.clone();
-            op.solve(&mut y)?;
-            w.push(y);
+        // The block's solves are independent: one task each.
+        for y in modelcraft_parallel::map(mcur.len(), |i| {
+            let mut y = mcur.get(i).cloned().unwrap_or_default();
+            op.solve(&mut y).map(|()| y)
+        }) {
+            w.push(y?);
         }
         // A_j = Q_jᵀ M W (symmetrized), W −= Q_j A_j (the earlier blocks go in the full
         // reorthogonalization below).

crates/solver/src/heat.rs

@@ -308,6 +308,8 @@ fn factor(a: &Csr, limits: &Limits, mesh: &HeatMesh, eqs: &Equations) -> Result<
         FactorError::TooLarge { entries, limit } => {
             Error::Limit(format!("the factorization needs {entries} entries; the limit is {limit} (refine less or raise the limit)"))
         }
+        FactorError::OutOfMemory { entries } => Error::Limit(format!("not enough memory for the factorization ({entries} entries)")),
+        FactorError::Internal(what) => Error::Internal(what),
         FactorError::NotPositiveDefinite { column } => {
             let label = eqs.node.get(column).and_then(|n| mesh.labels.get(*n)).copied().unwrap_or(0);
             Error::Singular(format!(

crates/solver/src/lib.rs

@@ -15,8 +15,11 @@
 //!    von Mises. Frequency analysis ([`eigen`]) runs shift-invert block Lanczos on the same
 //!    kind of factorization.
 //!
-//! Everything is sequential and in a fixed order, so the same model gives the same numbers bit
-//! for bit; sizes and iterations are capped by [`Limits`].
+//! Element matrices and stresses, the factorization's subtrees and dense tiles, the Lanczos
+//! block solves and matrix products run on threads ([`modelcraft_parallel`]), but every number
+//! is computed by one task in an order fixed by the model alone and sums are taken in a fixed
+//! order, so the same model gives the same numbers bit for bit on any number of threads and on
+//! every platform; sizes and iterations are capped by [`Limits`].
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 #![forbid(unsafe_code)]
 // Matrix kernels read as the textbook formulas with index loops.

crates/solver/src/sparse.rs

@@ -11,6 +11,9 @@ use modelcraft_fem::{DofLayout, DofSet};
 
 use crate::{Error, Result};
 
+/// Rows per parallel block of a product.
+const ROWS: usize = 4096;
+
 /// A square sparse matrix in compressed rows with sorted column indices.
 #[derive(Debug, Clone, PartialEq)]
 pub struct Csr {
@@ -28,11 +31,15 @@ impl Csr {
     }
 
     /// y = A x.
+    /// Rows are computed in parallel blocks; each row's sum runs in column order.
     pub fn mul(&self, x: &[f64], y: &mut [f64]) {
-        for (r, yr) in y.iter_mut().enumerate().take(self.n) {
-            let (cols, vals) = self.row(r);
-            *yr = cols.iter().zip(vals).map(|(c, v)| v * x.get(*c).copied().unwrap_or(0.0)).sum();
-        }
+        let n = self.n.min(y.len());
+        modelcraft_parallel::chunks_mut(y.get_mut(..n).unwrap_or_default(), ROWS, |b, block| {
+            for (i, yr) in block.iter_mut().enumerate() {
+                let (cols, vals) = self.row(b * ROWS + i);
+                *yr = cols.iter().zip(vals).map(|(c, v)| v * x.get(*c).copied().unwrap_or(0.0)).sum();
+            }
+        });
     }
 
     pub fn diagonal(&self) -> Vec<f64> {