diff --git a/crates/kernel/src/lib.rs b/crates/kernel/src/lib.rs index d99415e9..3204bc59 100644 --- a/crates/kernel/src/lib.rs +++ b/crates/kernel/src/lib.rs @@ -140,7 +140,9 @@ pub use prism_measure::certified_prism_equivalent; pub use sew::{EdgeSpec, FaceSpec, SurfSpec, sew, sew_touching}; pub use splitface::{SplitTool, split_body, split_faces, split_faces_with_map}; pub use step::step_export; -pub use step_in::{ImportedBody, StepImport, StepNode, step_cache_alive, step_import, step_import_shared, step_orientation_errors, step_validate}; +pub use step_in::{ + ImportedBody, StepImport, StepNode, StepReading, step_cache_alive, step_import, step_import_shared, step_orientation_errors, step_validate, +}; pub use step_out::{ExportBody, ExportProduct, StepHeader, step_export_bodies, step_export_products}; pub use surface_extend::{ExtendLimit, ExtendMode, ExtendOptions, extend_surface}; pub use surface_kind::{EdgeCircle, FaceSurface, edge_circle, face_surface}; diff --git a/crates/kernel/src/step_in/mod.rs b/crates/kernel/src/step_in/mod.rs index 36c9e658..6bf8a91d 100644 --- a/crates/kernel/src/step_in/mod.rs +++ b/crates/kernel/src/step_in/mod.rs @@ -88,6 +88,21 @@ pub struct StepImport { pub system: String, } +/// A solid the walk found, converted later. +struct Planned { + item: u64, + units: Units, + batch: usize, +} + +/// Where the solids of a batch go: their placement and names. +struct Batch { + m: mt::Matrix4, + path: Vec, + product: String, + instance: Option, +} + /// Reading context: the file and the units of the representation being read. pub(crate) struct Ctx<'a> { pub ex: &'a Exchange, @@ -228,8 +243,8 @@ fn rigid_inverse(m: &mt::Matrix4) -> mt::Matrix4 { /// faces the file had, and the file face of each kernel face (band faces split into two). type SolidEntry = (mt::Solid, bool, usize, Vec); -struct Reader<'a> { - ex: &'a Exchange, +struct Reader { + ex: Arc, units: HashMap, warnings: BTreeMap, /// Representation → its items and context. @@ -263,6 +278,15 @@ struct Reader<'a> { /// Items whose meshing was checked. unmeshed_checked: HashSet, bodies: Vec, + /// Solids the walk found, in file order, to be converted into bodies afterwards (so the + /// conversion, the long part, can be done a piece at a time: [`StepReading`]). + plan: Vec, + /// What the solids of one batch (consecutive solids of one representation) share. + batches: Vec, + /// Planned solids converted so far. + converted: usize, + /// The body each converted planned solid became (`None`: it failed). + made: Vec>, workers: usize, /// Whoever waits for the import (progress, Cancel). watch: Option>, @@ -272,8 +296,8 @@ struct Reader<'a> { cancelled: bool, } -impl<'a> Reader<'a> { - fn new(ex: &'a Exchange, workers: usize) -> Self { +impl Reader { + fn new(ex: Arc, workers: usize) -> Self { Self { ex, units: HashMap::new(), @@ -293,6 +317,10 @@ impl<'a> Reader<'a> { visits: 0, unmeshed_checked: HashSet::new(), bodies: Vec::new(), + plan: Vec::new(), + batches: Vec::new(), + converted: 0, + made: Vec::new(), workers, watch: None, solid_items: 0, @@ -310,7 +338,7 @@ impl<'a> Reader<'a> { self.cancelled = true; return true; } - w.progress("converting", self.bodies.len().min(self.solid_items), self.solid_items); + w.progress("converting", self.converted.min(self.solid_items), self.solid_items); } false } @@ -320,7 +348,8 @@ impl<'a> Reader<'a> { } fn index(&mut self) { - let ex = self.ex; + let ex = self.ex.clone(); + let ex = &*ex; let mut pds: HashMap = HashMap::new(); // PDS → definition let mut sdrs: Vec<(u64, u64)> = Vec::new(); // (PDS, rep) let mut cdsrs: Vec<(u64, u64)> = Vec::new(); // (relationship, PDS) @@ -522,11 +551,11 @@ impl<'a> Reader<'a> { fn units_of(&mut self, rep: u64) -> Units { let ctx = self.reps.get(&rep).and_then(|r| r.1); - context_units(self.ex, ctx, &mut self.units) + context_units(&self.ex, ctx, &mut self.units) } fn placement_matrix(&mut self, item: u64, u: Units) -> R { - let cx = Ctx { ex: self.ex, len: u.len, ang: u.ang, tol: u.tol }; + let cx = Ctx { ex: &self.ex, len: u.len, ang: u.ang, tol: u.tol }; let e = geom::entity(&cx, item)?; if e.name() == "CARTESIAN_TRANSFORMATION_OPERATOR_3D" { let p = e.params(); @@ -569,7 +598,7 @@ impl<'a> Reader<'a> { if let Some(r) = self.solids.get(&item) { return r.as_ref().ok().cloned(); } - let cx = Ctx { ex: self.ex, len: u.len, ang: u.ang, tol: u.tol }; + let cx = Ctx { ex: &self.ex, len: u.len, ang: u.ang, tol: u.tol }; let r = match guard("step brep", || brep::read_item(&cx, item).map_err(KernelError::Failed)) { Ok(out) => { let label = self.label(item); @@ -625,7 +654,7 @@ impl<'a> Reader<'a> { // Only batch consecutive solids: mapped items must retain traversal and // first-use unit/cache order. Bound temporary geometry independently of file size. let mut batch = vec![item]; - let capacity = MAX_BODIES.saturating_sub(self.bodies.len()).min(64); + let capacity = MAX_BODIES.saturating_sub(self.bodies_so_far()).min(64); while batch.len() < capacity && items.peek().is_some_and(|id| { self.ex.get(*id).is_some_and(|e| { @@ -637,84 +666,20 @@ impl<'a> Reader<'a> { batch.push(next); } } - let mut sources = Vec::new(); - let mut solids = Vec::new(); + let id = self.batches.len(); + self.batches.push(Batch { m: *m, path: path.to_vec(), product: product.to_string(), instance: instance.map(str::to_string) }); for item in batch { self.used_items.insert(item); if !self.node_items.insert(item) { continue; } - if self.bodies.len() >= MAX_BODIES { + if self.bodies_so_far() >= MAX_BODIES { self.warn(format!("more than {MAX_BODIES} bodies: the rest are left out")); return; } - let Some(entry) = self.item_solid(item, u) else { continue }; - // A one-use item is the only reference to its entry, so the solid moves - // out; an item whose conversion is cached for a later instance needs its - // own copy. - let (solid, closed, file_faces, face_origin) = match Arc::try_unwrap(entry) { - Ok(entry) => entry, - Err(shared) => (*shared).clone(), - }; - sources.push((item, closed, file_faces, face_origin)); - solids.push(parallel::Job { solid, check_mesh: !self.unmeshed_checked.contains(&item) }); - } - let prepared = parallel::bodies(&solids, m, self.workers); - for ((item, closed, file_faces, face_origin), body) in sources.into_iter().zip(prepared) { - let Some(e) = self.ex.get(item) else { continue }; - match body { - Ok((body, missing)) => { - let own = e.params().first().and_then(Param::as_str).unwrap_or("").trim().to_string(); - let name = if own.is_empty() { product.to_string() } else { own }; - // Faces the mesher gives up on (or that the budget cut short) are reported. - if self.unmeshed_checked.insert(item) && missing > 0 { - self.warn(format!("solid '{name}': {missing} face(s) could not be meshed and are not shown")); - } - out.push(self.bodies.len()); - let color = self.colors.get(&item).copied(); - // See-through bodies and faces with colours of their own. - let faces = face_origin - .iter() - .enumerate() - .filter_map(|(i, f)| { - let c = self.colors.get(f)?; - Some(crate::FacePaint { face: i, color: *c, opacity: self.opacity.get(f).copied().unwrap_or(1.0) }) - }) - .collect(); - let paint = crate::Paint { opacity: self.opacity.get(&item).copied().unwrap_or(1.0), faces }; - let body = body.with_color(color).with_paint(Some(paint)); - let identities = face_origin - .iter() - .map(|face| Some(crate::FaceSource::Imported { item, face: *face, payload_sha256: None, producer: None })) - .collect(); - let body = body.with_face_sources(Some(std::sync::Arc::new(identities))); - // A later instance of an item shows its first instance's mesh, placed. - let body = match self.shown.get(&item) { - Some((shared, first)) => body.with_display_of(shared.clone(), to_array(&mat_mul(m, &rigid_inverse(first)))), - None => { - if !self.once.contains(&item) { - self.shown.insert(item, (Arc::new(crate::body::SharedDisplay::new(body.clone())), *m)); - } - body - } - }; - self.bodies.push(ImportedBody { - name, - body, - color, - path: path.to_vec(), - instance: instance.map(str::to_string), - closed, - file_faces, - source_item: item, - face_origin, - }); - } - Err(err) => { - let label = self.label(item); - self.warn(format!("{label}: not imported: {err}")); - } - } + // The node lists the planned solid; it becomes a body index once converted. + out.push(self.plan.len()); + self.plan.push(Planned { item, units: u, batch: id }); } } "MAPPED_ITEM" => { @@ -742,6 +707,121 @@ impl<'a> Reader<'a> { } } + /// Bodies made, and planned ones still to convert (the most a failure can leave out). + fn bodies_so_far(&self) -> usize { + self.bodies.len() + self.plan.len() - self.converted + } + + /// Convert the next batch of planned solids into bodies; false once every one is done. + /// `single`: one solid at a time (the caller wants short steps and has no worker threads). + fn convert_next(&mut self, single: bool) -> bool { + if self.converted >= self.plan.len() { + return false; + } + if self.stop() { + return false; + } + let first = self.converted; + let id = self.plan[first].batch; + let mut last = first + 1; + while !single && last < self.plan.len() && self.plan[last].batch == id { + last += 1; + } + self.converted = last; + let ex = self.ex.clone(); + let mut sources = Vec::new(); + let mut solids = Vec::new(); + for i in first..last { + let (item, u) = (self.plan[i].item, self.plan[i].units); + self.made.push(None); + let Some(entry) = self.item_solid(item, u) else { continue }; + // A one-use item is the only reference to its entry, so the solid moves + // out; an item whose conversion is cached for a later instance needs its + // own copy. + let (solid, closed, file_faces, face_origin) = match Arc::try_unwrap(entry) { + Ok(entry) => entry, + Err(shared) => (*shared).clone(), + }; + sources.push((i, item, closed, file_faces, face_origin)); + solids.push(parallel::Job { solid, check_mesh: !self.unmeshed_checked.contains(&item) }); + } + let Some(batch) = self.batches.get(id) else { return true }; + let (m, path, product, instance) = (batch.m, batch.path.clone(), batch.product.clone(), batch.instance.clone()); + let prepared = parallel::bodies(&solids, &m, self.workers); + for ((i, item, closed, file_faces, face_origin), body) in sources.into_iter().zip(prepared) { + let Some(e) = ex.get(item) else { continue }; + match body { + Ok((body, missing)) => { + let own = e.params().first().and_then(Param::as_str).unwrap_or("").trim().to_string(); + let name = if own.is_empty() { product.clone() } else { own }; + // Faces the mesher gives up on (or that the budget cut short) are reported. + if self.unmeshed_checked.insert(item) && missing > 0 { + self.warn(format!("solid '{name}': {missing} face(s) could not be meshed and are not shown")); + } + if let Some(slot) = self.made.get_mut(i) { + *slot = Some(self.bodies.len()); + } + let color = self.colors.get(&item).copied(); + // See-through bodies and faces with colours of their own. + let faces = face_origin + .iter() + .enumerate() + .filter_map(|(i, f)| { + let c = self.colors.get(f)?; + Some(crate::FacePaint { face: i, color: *c, opacity: self.opacity.get(f).copied().unwrap_or(1.0) }) + }) + .collect(); + let paint = crate::Paint { opacity: self.opacity.get(&item).copied().unwrap_or(1.0), faces }; + let body = body.with_color(color).with_paint(Some(paint)); + let identities = face_origin + .iter() + .map(|face| Some(crate::FaceSource::Imported { item, face: *face, payload_sha256: None, producer: None })) + .collect(); + let body = body.with_face_sources(Some(std::sync::Arc::new(identities))); + // A later instance of an item shows its first instance's mesh, placed. + let body = match self.shown.get(&item) { + Some((shared, first)) => body.with_display_of(shared.clone(), to_array(&mat_mul(&m, &rigid_inverse(first)))), + None => { + if !self.once.contains(&item) { + self.shown.insert(item, (Arc::new(crate::body::SharedDisplay::new(body.clone())), m)); + } + body + } + }; + self.bodies.push(ImportedBody { + name, + body, + color, + path: path.clone(), + instance: instance.clone(), + closed, + file_faces, + source_item: item, + face_origin, + }); + } + Err(err) => { + let label = self.label(item); + self.warn(format!("{label}: not imported: {err}")); + } + } + } + true + } + + /// Planned-solid indices (a node's `bodies` after the walk) as body indices, leaving out + /// the ones that failed. + fn remap(&self, planned: &[usize]) -> Vec { + planned.iter().filter_map(|i| self.made.get(*i).copied().flatten()).collect() + } + + fn remap_node(&self, n: &mut StepNode) { + n.bodies = self.remap(&n.bodies); + for c in &mut n.children { + self.remap_node(c); + } + } + /// The name of an assembly occurrence: the usage's name, else its id. fn occurrence_name(&self, nauo: u64) -> Option { let p = self.ex.get(nauo)?.params(); @@ -763,8 +843,9 @@ impl<'a> Reader<'a> { /// Placement of a child occurrence in its parent. fn occurrence_matrix(&mut self, nauo: u64, child_reps: &[u64]) -> mt::Matrix4 { let Some(&rel) = self.placements.get(&nauo) else { return mt::Matrix4::from_scale(1.0) }; + let ex = self.ex.clone(); let r = (|| -> R { - let e = self.ex.get(rel).ok_or("missing placement")?; + let e = ex.get(rel).ok_or("missing placement")?; let rr = e.record("REPRESENTATION_RELATIONSHIP").ok_or("placement without representations")?; let (r1, r2) = (rr.get(2).and_then(Param::as_ref_id).ok_or("bad placement")?, rr.get(3).and_then(Param::as_ref_id).ok_or("bad placement")?); @@ -773,7 +854,7 @@ impl<'a> Reader<'a> { .and_then(|x| x.first()) .and_then(Param::as_ref_id) .ok_or("placement without transformation")?; - let ope = self.ex.get(op).ok_or("missing transformation")?; + let ope = ex.get(op).ok_or("missing transformation")?; let (u1, u2) = (self.units_of(r1), self.units_of(r2)); if ope.name() == "ITEM_DEFINED_TRANSFORMATION" { let p = ope.params(); @@ -1019,25 +1100,34 @@ const KEEP_SMALL_BYTES: usize = 1 << 20; /// [`step_import`] through a small cache of recent results (keyed by the text). Keep the result /// while the same text is still to be read elsewhere (in the timeline): that read reuses it. pub fn step_import_shared(text: &str) -> Result> { - use std::hash::{Hash, Hasher}; - let mut h = std::collections::hash_map::DefaultHasher::new(); - text.hash(&mut h); - let key = (h.finish(), text.len()); + let key = text_key(text); if let Ok(cache) = RECENT.lock() && let Some(imp) = cache.iter().filter(|r| r.key == key).find_map(|r| r.import.upgrade()) { return Ok(imp); } let imp = std::sync::Arc::new(step_import(text)?); + remember(key, &imp, text.len() <= KEEP_SMALL_BYTES); + Ok(imp) +} + +/// The cache key of a STEP text: its hash and length. +fn text_key(text: &str) -> (u64, usize) { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut h); + (h.finish(), text.len()) +} + +/// Keep `imp` in the recent imports (strongly only when `small`). +fn remember(key: (u64, usize), imp: &Arc, small: bool) { if let Ok(mut cache) = RECENT.lock() { cache.retain(|r| r.key != key && r.import.strong_count() > 0); - let small = (text.len() <= KEEP_SMALL_BYTES).then(|| imp.clone()); - cache.push(Recent { key, import: Arc::downgrade(&imp), _small: small }); + cache.push(Recent { key, import: Arc::downgrade(imp), _small: small.then(|| imp.clone()) }); while cache.len() > 4 { cache.remove(0); } } - Ok(imp) } /// Imports the cache still holds alive, and how many STEP bytes they came from (tests and @@ -1052,72 +1142,130 @@ pub fn step_import(text: &str) -> Result { } fn step_import_with_workers(text: &str, workers: usize) -> Result { - let watch = crate::watch::current(); - let cancelled = || watch.as_ref().is_some_and(|w| w.cancelled()); - let ex = match p21::parse(text) { - Ok(ex) => ex, - Err(_) if cancelled() => return Err(crate::watch::cancelled_error("STEP import")), - Err(e) => return Err(KernelError::Invalid(e)), - }; - let mut rd = Reader::new(&ex, workers); - rd.solid_items = ex - .entities - .values() - .filter(|e| matches!(e.name(), "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL")) - .count(); - rd.watch = watch.clone(); - rd.index(); - // Roots: product definitions that are nobody's component. - let child_pds: HashSet = - rd.children.values().flatten().filter_map(|n| ex.get(*n).and_then(|e| e.params().get(4)).and_then(Param::as_ref_id)).collect(); - let mut roots: Vec = - ex.entities.iter().filter(|(id, e)| e.name() == "PRODUCT_DEFINITION" && !child_pds.contains(id)).map(|(id, _)| *id).collect(); - roots.sort(); - let identity = mt::Matrix4::from_scale(1.0); - let mut tree = Vec::new(); - for pd in roots { - let n = rd.node(pd, identity, identity, &[], None, 0, &mut Vec::new()); - if !n.bodies.is_empty() || !n.children.is_empty() { - tree.push(n); - } + let mut reading = StepReading::with_workers(text, workers)?; + while reading.step() {} + reading.finish() +} + +/// A STEP file being read a piece at a time, for a caller that can't wait for the whole read +/// in one go (the web app, which has one thread and must keep drawing): [`StepReading::new`] +/// parses the file and walks its structure, each [`StepReading::step`] converts the next few +/// solids into bodies, and [`StepReading::finish_shared`] gives the import, cached as +/// [`step_import_shared`] would, so a command reading the same text next finds it read. +pub struct StepReading { + rd: Reader, + tree: Vec, + key: (u64, usize), + small: bool, + /// One solid per step (no worker threads to share a batch between). + single: bool, +} + +impl StepReading { + /// Parse `text` and walk it; the solids are converted by [`Self::step`]. + pub fn new(text: &str) -> Result { + Self::with_workers(text, parallel::workers()) } - // Solids outside any product structure (bare geometry files). - let mut loose: Vec = rd - .reps - .iter() - .flat_map(|(r, (items, _))| items.iter().map(move |i| (*r, *i))) - .filter(|(_, i)| !rd.used_items.contains(i)) - .filter(|(_, i)| { - ex.get(*i).is_some_and(|e| matches!(e.name(), "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL")) - }) - .map(|(r, _)| r) - .collect(); - loose.sort(); - loose.dedup(); - if !loose.is_empty() { - let mut bodies = Vec::new(); - rd.node_items = rd.used_items.clone(); - rd.visits = 0; - for r in loose { - rd.rep_bodies(r, &identity, &[], "Import", None, &mut bodies, 0); + + fn with_workers(text: &str, workers: usize) -> Result { + let key = text_key(text); + let watch = crate::watch::current(); + let cancelled = || watch.as_ref().is_some_and(|w| w.cancelled()); + let ex = match p21::parse(text) { + Ok(ex) => Arc::new(ex), + Err(_) if cancelled() => return Err(crate::watch::cancelled_error("STEP import")), + Err(e) => return Err(KernelError::Invalid(e)), + }; + let mut rd = Reader::new(ex.clone(), workers); + rd.solid_items = ex + .entities + .values() + .filter(|e| matches!(e.name(), "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL")) + .count(); + rd.watch = watch.clone(); + rd.index(); + // Roots: product definitions that are nobody's component. + let child_pds: HashSet = + rd.children.values().flatten().filter_map(|n| ex.get(*n).and_then(|e| e.params().get(4)).and_then(Param::as_ref_id)).collect(); + let mut roots: Vec = + ex.entities.iter().filter(|(id, e)| e.name() == "PRODUCT_DEFINITION" && !child_pds.contains(id)).map(|(id, _)| *id).collect(); + roots.sort(); + let identity = mt::Matrix4::from_scale(1.0); + let mut tree = Vec::new(); + for pd in roots { + tree.push(rd.node(pd, identity, identity, &[], None, 0, &mut Vec::new())); } - if !bodies.is_empty() { + // Solids outside any product structure (bare geometry files). + let mut loose: Vec = rd + .reps + .iter() + .flat_map(|(r, (items, _))| items.iter().map(move |i| (*r, *i))) + .filter(|(_, i)| !rd.used_items.contains(i)) + .filter(|(_, i)| { + ex.get(*i) + .is_some_and(|e| matches!(e.name(), "MANIFOLD_SOLID_BREP" | "BREP_WITH_VOIDS" | "FACETED_BREP" | "SHELL_BASED_SURFACE_MODEL")) + }) + .map(|(r, _)| r) + .collect(); + loose.sort(); + loose.dedup(); + if !loose.is_empty() { + let mut bodies = Vec::new(); + rd.node_items = rd.used_items.clone(); + rd.visits = 0; + for r in loose { + rd.rep_bodies(r, &identity, &[], "Import", None, &mut bodies, 0); + } tree.push(StepNode { name: "Import".into(), transform: to_array(&identity), bodies, children: Vec::new() }); } + let single = workers <= 1; + Ok(StepReading { rd, tree, key, small: text.len() <= KEEP_SMALL_BYTES, single }) } - // Cancelled: what was read so far is dropped here, not cached. - if rd.cancelled || cancelled() { - return Err(crate::watch::cancelled_error("STEP import")); + + /// Convert the next solid (or batch of solids); false once all are converted (or the + /// watch cancelled the import). + pub fn step(&mut self) -> bool { + self.rd.convert_next(self.single) } - if let Some(first) = ex.duplicates.first() { - rd.warn(format!("{} entity ids are defined twice (first #{first}); the first definitions were used", ex.duplicates.len())); + + /// Solids converted, of the solids to convert. + pub fn progress(&self) -> (usize, usize) { + (self.rd.converted, self.rd.plan.len()) } - let warnings: Vec = rd.warnings.iter().map(|(m, n)| if *n > 1 { format!("{m} (×{n})") } else { m.clone() }).collect(); - if rd.bodies.is_empty() { - let why = warnings.first().cloned().unwrap_or_else(|| "the file has no solids".into()); - return Err(KernelError::Failed(format!("STEP import: no bodies ({why})"))); + + /// The import, once [`Self::step`] has converted every solid. + pub fn finish(mut self) -> Result { + while self.step() {} + let rd = &mut self.rd; + // Cancelled: what was read so far is dropped here, not cached. + if rd.cancelled || rd.watch.as_ref().is_some_and(|w| w.cancelled()) { + return Err(crate::watch::cancelled_error("STEP import")); + } + let mut tree = std::mem::take(&mut self.tree); + for n in &mut tree { + rd.remap_node(n); + } + tree.retain(|n| !n.bodies.is_empty() || !n.children.is_empty()); + let ex = rd.ex.clone(); + if let Some(first) = ex.duplicates.first() { + rd.warn(format!("{} entity ids are defined twice (first #{first}); the first definitions were used", ex.duplicates.len())); + } + let warnings: Vec = rd.warnings.iter().map(|(m, n)| if *n > 1 { format!("{m} (×{n})") } else { m.clone() }).collect(); + if rd.bodies.is_empty() { + let why = warnings.first().cloned().unwrap_or_else(|| "the file has no solids".into()); + return Err(KernelError::Failed(format!("STEP import: no bodies ({why})"))); + } + let bodies = std::mem::take(&mut rd.bodies); + Ok(StepImport { bodies, tree, warnings, schema: ex.schemas.join(", "), system: ex.originating_system.clone() }) + } + + /// [`Self::finish`], and the import is cached for the next read of the same text. + pub fn finish_shared(self) -> Result> { + let (key, small) = (self.key, self.small); + let imp = Arc::new(self.finish()?); + remember(key, &imp, small); + Ok(imp) } - Ok(StepImport { bodies: rd.bodies, tree, warnings, schema: ex.schemas.join(", "), system: ex.originating_system.clone() }) } #[cfg(test)] diff --git a/crates/kernel/src/step_in/tests.rs b/crates/kernel/src/step_in/tests.rs index 1ff99f93..9bad51c6 100644 --- a/crates/kernel/src/step_in/tests.rs +++ b/crates/kernel/src/step_in/tests.rs @@ -175,16 +175,19 @@ fn parallel_import_does_not_prepare_past_body_budget_with_duplicates_or_mapped_i "{HEAD}{}#2000=MANIFOLD_SOLID_BREP('Duplicate',#100);\n#2001=MANIFOLD_SOLID_BREP('First',#100);\n#2002=MANIFOLD_SOLID_BREP('Second',#100);\n#2003=MANIFOLD_SOLID_BREP('Beyond budget',#999999);\n#3000=MAPPED_ITEM('',#999998,#999997);\n#60=SHAPE_REPRESENTATION('',(#2000,#2000,#2001,#3000,#2002,#2003),$);\n{TAIL}", cube_entities() ); - let ex = p21::parse(&text).unwrap(); + let ex = std::sync::Arc::new(p21::parse(&text).unwrap()); let seed = step_import_with_workers(&format!("{HEAD}{}#60=SHAPE_REPRESENTATION('',(#101),$);\n{TAIL}", cube_entities()), 1).unwrap().bodies.remove(0); for workers in [1, 8] { - let mut reader = Reader::new(&ex, workers); + let mut reader = Reader::new(ex.clone(), workers); reader.index(); reader.bodies = vec![seed.clone(); MAX_BODIES - 2]; reader.node_items.insert(2000); let mut out = Vec::new(); reader.rep_bodies(60, &mt::Matrix4::from_scale(1.0), &[], "Budget", None, &mut out, 0); + assert!(reader.solids.is_empty(), "the walk only plans the solids"); + while reader.convert_next(workers == 1) {} + let out = reader.remap(&out); assert_eq!(reader.bodies.len(), MAX_BODIES); assert_eq!(out, vec![MAX_BODIES - 2, MAX_BODIES - 1]); assert_eq!(reader.bodies.last().unwrap().name, "Second"); @@ -598,8 +601,8 @@ fn repeated_items_are_shared_and_one_use_items_are_not() { let a = box_solid(Vec3::ZERO, Vec3::new(10.0, 10.0, 10.0)).unwrap(); let b = box_solid(Vec3::new(20.0, 0.0, 0.0), Vec3::new(30.0, 10.0, 10.0)).unwrap(); let flat = step_export(&[&a, &b], "test").unwrap(); - let ex = p21::parse(&flat).unwrap(); - let mut rd = Reader::new(&ex, 1); + let ex = std::sync::Arc::new(p21::parse(&flat).unwrap()); + let mut rd = Reader::new(ex.clone(), 1); rd.index(); assert_eq!(rd.once.len(), 2, "each solid of a flat file is converted once"); let imp = step_import(&flat).unwrap(); @@ -609,8 +612,8 @@ fn repeated_items_are_shared_and_one_use_items_are_not() { assert!(rel(vol, 2000.0) < 1e-6, "{vol}"); let assembly = two_instance_assembly(); - let ex = p21::parse(&assembly).unwrap(); - let mut rd = Reader::new(&ex, 1); + let ex = std::sync::Arc::new(p21::parse(&assembly).unwrap()); + let mut rd = Reader::new(ex.clone(), 1); rd.index(); assert!(rd.once.is_empty(), "an assembly may convert an item again for the second instance"); let imp = step_import(&assembly).unwrap(); diff --git a/crates/ui-egui/src/file_load.rs b/crates/ui-egui/src/file_load.rs index 7f88cd2a..1beeac50 100644 --- a/crates/ui-egui/src/file_load.rs +++ b/crates/ui-egui/src/file_load.rs @@ -95,6 +95,41 @@ struct Running { partial_rx: Option>, /// The picture shown is of the first this many features (`None`: the saved preview). built: Option, + /// A STEP file read a piece per frame on this thread (the web app), instead of on a worker. + staged: Option, +} + +/// A STEP import done a piece per frame, for the web app: it has one thread, which must keep +/// drawing (and offer Cancel) while a large file comes in. The file is parsed in one frame, +/// then its solids are converted a few at a time, each frame for about [`STAGED_FRAME_MS`]; +/// the import is then cached, and the command that inserts or opens the file finds it read. +struct Staged { + tx: Sender, + session: Option, + command: &'static str, + params: Value, + reading: Option, + /// The command's result once it has run; then the bodies' meshes are made, a few a frame, + /// so the first frame that draws the design doesn't make them all at once. + ran: Option, + /// Bodies meshed so far, and their triangles. + warmed: usize, + triangles: usize, + /// The overlay has been drawn once (the first frame does no work, so it shows at once). + shown: bool, +} + +/// Designs up to this many triangles get a thumbnail (a picture of more takes a while). +const STAGED_THUMBNAIL_TRIANGLES: usize = 1_000_000; + +/// How long a frame spends on a staged STEP import. +const STAGED_FRAME_MS: f64 = 30.0; + +/// Files read in stages: STEP (not IGES, which is first restated as STEP) to insert or open. +#[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))] +fn staged_kind(request: &Request) -> bool { + let p = &request.path; + matches!(request.kind, Kind::Insert | Kind::Open) && solidcraft_engine::io::is_step_path(p) && !solidcraft_engine::io::is_iges_path(p) } impl Running { @@ -147,6 +182,11 @@ pub fn begin(app: &mut SolveApp, path: &str, insert: bool) { pub fn begin_request(app: &mut SolveApp, request: Request) -> Result<(), String> { #[cfg(target_arch = "wasm32")] { + if staged_kind(&request) { + app.file_load.queued.push_back(request); + start_staged(app); + return Ok(()); + } let r = match &request.kind { Kind::Insert => { app.insert_path(&request.path); @@ -332,6 +372,7 @@ fn start(app: &mut SolveApp) { preview: None, partial_rx, built: None, + staged: None, }) } Err(e) => { @@ -342,6 +383,139 @@ fn start(app: &mut SolveApp) { } } +/// Start the next queued STEP file as a [`Staged`] import (see [`step_staged`]). +#[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))] +fn start_staged(app: &mut SolveApp) { + if app.file_load.running.is_some() { + return; + } + let Some(request) = app.file_load.queued.pop_front() else { return }; + let insert = request.insert(); + let before = if insert { app.session.world_state().bodies.iter().map(|b| b.name.clone()).collect() } else { Vec::new() }; + let mut session = if insert { app.session.scratch() } else { Session::default() }; + if insert { + session.selection = app.session.selection.clone(); + } + let (command, params) = request.command(); + let (tx, rx) = std::sync::mpsc::channel(); + let progress = Arc::new(Progress::default()); + #[cfg(test)] + if let Some(i) = app.file_load.hold_at.take() { + progress.hold_at(i); + } + app.file_load.running = Some(Running { + base: app.session.doc.clone(), + base_revision: app.session.revision, + before, + rx, + phase: Arc::new(AtomicU8::new(0)), + progress, + preview_rx: None, + preview: None, + partial_rx: None, + built: None, + staged: Some(Staged { tx, session: Some(session), command, params, reading: None, ran: None, warmed: 0, triangles: 0, shown: false }), + request, + }); +} + +/// One frame of a staged STEP import: parse the file, or convert solids for a frame's worth of +/// time, or, once every solid is in, run the command (which finds the file read) and hand its +/// result to [`tick`] as a worker would. +fn step_staged(running: &mut Running) { + let Some(staged) = running.staged.as_mut() else { return }; + let progress = running.progress.clone(); + let scope = running.request.scope.clone(); + let path = running.request.path.clone(); + let send = |staged: &mut Staged, job: Job| { + let _ = staged.tx.send(job); + staged.session = None; + staged.reading = None; + }; + if staged.session.is_none() { + return; + } + if progress.is_cancelled() { + send(staged, Err("file operation cancelled".into())); + return; + } + if !staged.shown { + staged.shown = true; + return; + } + let began = web_time::Instant::now(); + let in_budget = || began.elapsed().as_secs_f64() * 1000.0 < STAGED_FRAME_MS; + if staged.ran.is_some() { + let Some(session) = staged.session.as_ref() else { return }; + let world = session.world_state(); + while staged.warmed < world.bodies.len() && in_budget() { + if let Some(body) = world.bodies.get(staged.warmed) { + staged.triangles += body.mesh().triangles.len(); + } + staged.warmed += 1; + } + if staged.warmed < world.bodies.len() { + return; + } + let (Some(session), Some(result)) = (staged.session.take(), staged.ran.take()) else { return }; + let thumbnail = staged.command == "doc.open" && staged.triangles <= STAGED_THUMBNAIL_TRIANGLES; + let image = if thumbnail { crate::home::remember(&session) } else { None }; + send(staged, Ok(Made { session, result, image })); + return; + } + let Some(reading) = staged.reading.as_mut() else { + // Parse and walk the file. The reading keeps the watch: its steps report progress and + // stop on Cancel. + let read = solidcraft_engine::io::vfs::scoped(scope, || { + let bytes = solidcraft_engine::io::vfs::read(&path, solidcraft_engine::io::MAX_STEP_BYTES).map_err(|e| format!("{path}: {e}"))?; + // The same text the import command makes of the bytes, so it finds this reading. + let text = String::from_utf8_lossy(&bytes).into_owned(); + solidcraft_engine::doc::progress::watch(progress.clone(), || solidcraft_engine::kernel::StepReading::new(&text)) + .map_err(|e| format!("{path}: {e}")) + }); + match read { + Ok(reading) => staged.reading = Some(reading), + Err(e) => send(staged, Err(e)), + } + return; + }; + while in_budget() { + if !reading.step() { + break; + } + } + let (done, total) = reading.progress(); + if done < total && !progress.is_cancelled() { + return; + } + let Some(reading) = staged.reading.take() else { return }; + // Held until the command has read the file from the cache. + let import = reading.finish_shared(); + let import = match import { + _ if progress.is_cancelled() => Err("file operation cancelled".to_string()), + Ok(import) => Ok(import), + Err(e) => Err(format!("{path}: {e}")), + }; + let (Some(mut session), Ok(import)) = (staged.session.take(), import.clone()) else { + send(staged, Err(import.err().unwrap_or_else(|| "file operation cancelled".into()))); + return; + }; + let (command, params) = (staged.command, staged.params.clone()); + let result = solidcraft_engine::io::vfs::scoped(running.request.scope.clone(), || { + solidcraft_engine::doc::progress::watch(progress.clone(), || session.execute(command, ¶ms)) + }); + drop(import); + match result { + Ok(result) => { + // Next frames: the meshes ("Preparing the view"). + running.phase.store(1, Ordering::Relaxed); + staged.session = Some(session); + staged.ran = Some(result); + } + Err(e) => send(staged, Err(e.to_string())), + } +} + /// A small picture of the bodies built so far (their display meshes, made here and kept for /// the finished model), or `None` when there is nothing to show yet, too much to draw, or the /// load ended meanwhile (`stop`). @@ -520,6 +694,9 @@ fn compute( } pub fn tick(app: &mut SolveApp, ctx: &egui::Context) { + if let Some(running) = &mut app.file_load.running { + step_staged(running); + } if let Some(running) = &mut app.file_load.running { if !running.cancelled() && running.matches(&app.session) { if let Some(image) = running.preview_rx.as_ref().and_then(|rx| rx.try_recv().ok()) { @@ -598,6 +775,8 @@ pub fn tick(app: &mut SolveApp, ctx: &egui::Context) { } #[cfg(not(target_arch = "wasm32"))] start(app); + #[cfg(target_arch = "wasm32")] + start_staged(app); if app.file_load.busy() { ctx.request_repaint_after(std::time::Duration::from_millis(16)); } @@ -746,6 +925,7 @@ mod tests { preview: None, partial_rx: None, built: None, + staged: None, }); if change_document { app.session.execute("file.new", &serde_json::json!({"name":"Newer design"})).unwrap(); @@ -810,6 +990,7 @@ mod tests { preview: None, partial_rx: None, built: None, + staged: None, }); tick(&mut app, &egui::Context::default()); assert!(Arc::ptr_eq(&app.session.doc, &old)); @@ -835,6 +1016,7 @@ mod tests { preview: None, partial_rx: None, built: None, + staged: None, }); assert!(!app.file_load.busy()); assert!(app.file_load.running.is_some()); @@ -845,4 +1027,61 @@ mod tests { assert!(app.file_load.running.is_none()); assert!(Arc::ptr_eq(&old, &app.session.doc)); } + + /// #1376: the web app reads a STEP file a piece per frame. The first frame only shows the + /// overlay, the next parses, and the solids follow, frame by frame, until the insert + /// lands as a worker's would; Cancel between frames leaves the design as it was. + #[test] + fn a_staged_step_import_lands_over_frames_and_cancels_between_them() { + let dir = std::env::temp_dir().join(format!("solidcraft-staged-step-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("boxes.step").to_string_lossy().into_owned(); + let mut part = Session::default(); + for i in 0..6 { + part.execute("solid.box", &serde_json::json!({"length": 10, "width": 10, "height": 10, "corner": [i * 20, 0, 0]})).unwrap(); + } + part.execute("file.export", &serde_json::json!({"path": path})).unwrap(); + let ctx = egui::Context::default(); + let mut app = SolveApp::new(Session::default(), Default::default()); + let bodies = |app: &SolveApp| app.session.world_state().bodies.len(); + assert!(staged_kind(&Request::new(&path, Kind::Insert)) && staged_kind(&Request::new(&path, Kind::Open))); + assert!(!staged_kind(&Request::new("part.igs", Kind::Insert)) && !staged_kind(&Request::new("part.solidcraft", Kind::Open))); + app.file_load.queued.push_back(Request::new(&path, Kind::Insert)); + start_staged(&mut app); + tick(&mut app, &ctx); + assert!(app.file_load.busy() && bodies(&app) == 0, "the first frame only shows the overlay"); + tick(&mut app, &ctx); + let reading = app.file_load.running.as_ref().and_then(|r| r.staged.as_ref()).is_some_and(|s| s.reading.is_some()); + assert!(reading, "the next frame parses the file"); + let mut frames = 2; + while app.file_load.running.is_some() && frames < 1000 { + tick(&mut app, &ctx); + frames += 1; + } + assert_eq!(bodies(&app), 6, "{:?}", app.status); + assert!(app.status.as_ref().is_some_and(|s| s.0.contains("Insert done")), "{:?}", app.status); + // Cancelled after the parse: nothing is inserted, and nothing waits. + let before = app.session.doc.clone(); + app.file_load.queued.push_back(Request::new(&path, Kind::Insert)); + start_staged(&mut app); + tick(&mut app, &ctx); + tick(&mut app, &ctx); + app.file_load.cancel(); + tick(&mut app, &ctx); + assert!(app.file_load.running.is_none() && !app.file_load.busy()); + assert!(Arc::ptr_eq(&before, &app.session.doc)); + assert_eq!(bodies(&app), 6); + // A file that isn't STEP fails like the worker's would, and the design stays. + let junk = dir.join("junk.step").to_string_lossy().into_owned(); + std::fs::write(&junk, "not a step file").unwrap(); + app.file_load.queued.push_back(Request::new(&junk, Kind::Insert)); + start_staged(&mut app); + for _ in 0..4 { + tick(&mut app, &ctx); + } + assert!(app.file_load.running.is_none()); + assert!(app.status.as_ref().is_some_and(|s| s.2 && s.0.contains("junk.step")), "{:?}", app.status); + assert!(Arc::ptr_eq(&before, &app.session.doc)); + let _ = std::fs::remove_dir_all(&dir); + } }