1b-agent/OneCAD
1
0 Fork 0
forked from 1b-agent/solidcraft-upstream
Code Pull Requests Activity

Add design containers with deduplicated blobs and JSON compatibility (#327)

Browse Source 
* Add fast design containers with lossless deduplicated payloads

* Keep display caching opt-in to avoid default save overhead
This commit is contained in:
selune authored and GitHub committed 2026-10-12 19:15:41 -04:00
1 parent 8e0c3d2794
commit 006a23a767
30 files changed
+1921 -71
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/io/src/design.rs
+811
View File
@@ -0,0 +1,811 @@
//! Versioned design packages. Geometry is rebuilt from authoritative inputs; no
//! evaluated cache is trusted here. All archive access stays in memory, never on disk.
use std::collections::{BTreeMap, BTreeSet};
use std::io::{Cursor, Read, Write};
use serde_json::{Map, Value, json};
use sha2::{Digest, Sha256};
use solidcraft_doc::{Document, FeatureKind, MeshData};
use crate::{IoError, MAX_DESIGN_BYTES, Result, design_metadata::replace};
const VERSION: &str = "solidcraft-container/1";
const LEGACY_VERSION: &str = "solvecraft-container/1";
const PREVIEW_VERSION: &str = "solidcraft-preview/1";
const MAX_ENTRIES: usize = 10_000;
const MAX_EXPANDED: usize = 512 << 20;
const MESH_MAGIC: &[u8; 8] = b"SCMESH01";
const MAX_PREVIEW_BYTES: usize = 8 << 20;
const MAX_PREVIEW_TRIANGLES: usize = 100_000;
/// Untrusted, display-only evaluated meshes. Never used for editing, picking or measurement.
#[derive(Default)]
pub struct DesignPreview {
pub meshes: Vec<MeshData>,
}
/// Fast favors interactive file operations; Compact trades CPU time for space.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum DesignCompression {
#[default]
Fast,
Compact,
}
fn invalid(message: impl std::fmt::Display) -> IoError {
IoError::Invalid(format!("design container: {message}"))
}
/// Detect the package signature independently of the file name.
pub fn is_design_container(bytes: &[u8]) -> bool {
bytes.starts_with(b"PK\x03\x04")
}
fn fields(pairs: impl IntoIterator<Item = (&'static str, Value)>) -> Map<String, Value> {
pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()
}
fn digest(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
struct PackageWriter {
zip: zip::ZipWriter<Cursor<Vec<u8>>>,
names: BTreeSet<String>,
expanded: usize,
resolved: usize,
}
impl PackageWriter {
fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<()> {
if bytes.len() > MAX_DESIGN_BYTES {
return Err(invalid("part too large"));
}
if !self.names.insert(name.to_owned()) {
return Ok(());
}
self.expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
if self.expanded > MAX_EXPANDED || self.names.len() > MAX_ENTRIES {
return Err(invalid("package too large"));
}
let method = if compressed { zip::CompressionMethod::Deflated } else { zip::CompressionMethod::Stored };
let options = zip::write::SimpleFileOptions::default().compression_method(method).compression_level(compressed.then_some(1));
self.zip.start_file(name, options).map_err(invalid)?;
self.zip.write_all(bytes).map_err(invalid)
}
fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value> {
self.resolved =
self.resolved.checked_add(bytes.len()).filter(|&n| n <= MAX_EXPANDED).ok_or_else(|| invalid("resolved payloads too large"))?;
let name = format!("blobs/{}.{extension}", digest(bytes));
self.part(&name, bytes, compressed)?;
Ok(Value::String(name))
}
}
/// Write a compact container, deduplicating payloads by SHA-256. Mesh arrays are
/// stored directly for low save/open latency; small metadata uses level-one
/// Deflate. Use the explicit Compact option to compress STEP and meshes too.
pub fn write_design_container(doc: &Document) -> Result<Vec<u8>> {
write_design_container_with_compression(doc, DesignCompression::Fast)
}
/// Write a container with an explicit payload compression policy.
pub fn write_design_container_with_compression(doc: &Document, compression: DesignCompression) -> Result<Vec<u8>> {
write_design_container_with_preview(doc, compression, None)
}
/// Optional display cache failure never prevents saving the authoritative design.
pub fn write_design_container_with_preview(doc: &Document, compression: DesignCompression, preview: Option<&DesignPreview>) -> Result<Vec<u8>> {
let mut writer = PackageWriter { zip: zip::ZipWriter::new(Cursor::new(Vec::new())), names: BTreeSet::new(), expanded: 0, resolved: 0 };
let mut features = Vec::with_capacity(doc.features.len());
for feature in &doc.features {
let replacements = match &feature.kind {
FeatureKind::Import { step, .. } => {
let blob = writer.blob(step.as_bytes(), "step", compression == DesignCompression::Compact)?;
fields([("step", Value::String(String::new())), ("step_blob", blob)])
}
FeatureKind::MeshImport { meshes, .. } => {
let mut metadata = Vec::with_capacity(meshes.len());
for mesh in meshes {
let bytes = encode_mesh(mesh)?;
let blob = writer.blob(&bytes, "mesh", compression == DesignCompression::Compact)?;
let mut value = replace(mesh, fields([("positions", json!([])), ("triangles", json!([]))])).map_err(invalid)?;
value.as_object_mut().ok_or_else(|| invalid("mesh metadata"))?.insert("mesh_blob".into(), blob);
metadata.push(value);
}
fields([("meshes", Value::Array(metadata))])
}
_ => Map::new(),
};
// Extra reference fields are added after serializing existing fields.
let step_blob = replacements.get("step_blob").cloned();
let mut value = replace(feature, replacements).map_err(invalid)?;
if let Some(blob) = step_blob {
value.as_object_mut().ok_or_else(|| invalid("feature metadata"))?.insert("step_blob".into(), blob);
}
features.push(value);
}
let mut canvases = Vec::with_capacity(doc.canvases.len());
for canvas in &doc.canvases {
if canvas.data.len() > solidcraft_doc::canvas::MAX_CANVAS_BYTES.div_ceil(3) * 4 {
return Err(invalid("canvas too large"));
}
let bytes = canvas.bytes().ok_or_else(|| invalid("invalid canvas base64"))?;
if !matches!(canvas.format.as_str(), "png" | "jpeg") || bytes.len() > solidcraft_doc::canvas::MAX_CANVAS_BYTES {
return Err(invalid("invalid canvas format or size"));
}
let blob = writer.blob(&bytes, &canvas.format, false)?;
let mut value = replace(canvas, fields([("data", json!(""))])).map_err(invalid)?;
value.as_object_mut().ok_or_else(|| invalid("canvas metadata"))?.insert("image_blob".into(), blob);
canvases.push(value);
}
let metadata = replace(doc, fields([("features", Value::Array(features)), ("canvases", Value::Array(canvases))])).map_err(invalid)?;
let metadata = serde_json::to_vec(&metadata).map_err(invalid)?;
writer.part("document.json", &metadata, true)?;
writer.part("container.json", &serde_json::to_vec(&json!({"format": VERSION})).map_err(invalid)?, false)?;
if let Some((manifest, data)) = preview.and_then(|p| encode_preview(p, &metadata).ok())
&& writer.names.len().saturating_add(2) <= MAX_ENTRIES
&& writer
.expanded
.saturating_add(manifest.len())
.saturating_add(data.len())
.saturating_add(writer.names.len().saturating_mul(256))
.saturating_add(65_536)
<= MAX_DESIGN_BYTES
{
writer.part("cache/preview.json", &manifest, false)?;
writer.part("cache/preview.bin", &data, false)?;
}
let bytes = writer.zip.finish().map_err(invalid)?.into_inner();
if bytes.len() > MAX_DESIGN_BYTES {
return Err(invalid("compressed file too large"));
}
Ok(bytes)
}
fn validate_mesh(mesh: &MeshData) -> Result<()> {
if !mesh.positions.len().is_multiple_of(3)
|| !mesh.triangles.len().is_multiple_of(3)
|| mesh.positions.iter().any(|v| !v.is_finite() || v.abs() > 1e9)
|| mesh.triangles.iter().any(|&v| v as usize >= mesh.positions.len() / 3)
{
return Err(invalid("invalid mesh coordinates or indices"));
}
Ok(())
}
fn encode_mesh(mesh: &MeshData) -> Result<Vec<u8>> {
validate_mesh(mesh)?;
let size = mesh
.positions
.len()
.checked_mul(8)
.and_then(|v| mesh.triangles.len().checked_mul(4).and_then(|t| v.checked_add(t)))
.and_then(|v| v.checked_add(24))
.filter(|&n| n <= MAX_DESIGN_BYTES)
.ok_or_else(|| invalid("mesh too large"))?;
let mut bytes = Vec::with_capacity(size);
bytes.extend_from_slice(MESH_MAGIC);
bytes.extend_from_slice(&(mesh.positions.len() as u64).to_le_bytes());
bytes.extend_from_slice(&(mesh.triangles.len() as u64).to_le_bytes());
for value in &mesh.positions {
bytes.extend_from_slice(&value.to_le_bytes());
}
for value in &mesh.triangles {
bytes.extend_from_slice(&value.to_le_bytes());
}
Ok(bytes)
}
fn decode_mesh(bytes: &[u8], mesh: &mut MeshData) -> Result<()> {
if bytes.get(..8) != Some(MESH_MAGIC.as_slice()) {
return Err(invalid("unsupported mesh blob version"));
}
let count = |range: std::ops::Range<usize>| -> Result<usize> {
let array: [u8; 8] = bytes.get(range).ok_or_else(|| invalid("truncated mesh blob"))?.try_into().map_err(invalid)?;
usize::try_from(u64::from_le_bytes(array)).map_err(invalid)
};
let positions = count(8..16)?;
let triangles = count(16..24)?;
let middle = positions.checked_mul(8).and_then(|n| n.checked_add(24)).ok_or_else(|| invalid("mesh size overflow"))?;
let total = triangles.checked_mul(4).and_then(|n| n.checked_add(middle)).ok_or_else(|| invalid("mesh size overflow"))?;
if total != bytes.len() || !positions.is_multiple_of(3) || !triangles.is_multiple_of(3) {
return Err(invalid("invalid mesh blob lengths"));
}
mesh.positions =
bytes.get(24..middle).ok_or_else(|| invalid("truncated mesh"))?.as_chunks::<8>().0.iter().map(|&v| f64::from_le_bytes(v)).collect();
mesh.triangles =
bytes.get(middle..).ok_or_else(|| invalid("truncated mesh"))?.as_chunks::<4>().0.iter().map(|&v| u32::from_le_bytes(v)).collect();
validate_mesh(mesh)
}
struct PackageReader<'a> {
zip: zip::ZipArchive<Cursor<&'a [u8]>>,
blobs: BTreeMap<String, Vec<u8>>,
hydrated: usize,
}
impl<'a> PackageReader<'a> {
fn new(bytes: &'a [u8]) -> Result<Self> {
// zip's name index can coalesce duplicate central-directory entries. Our
// bounded format does not need ZIP64/multiple disks: verify the declared
// entry count as well so duplicate names cannot silently select a payload.
let tail = bytes.get(bytes.len().saturating_sub(65_557)..).ok_or_else(|| invalid("missing ZIP directory"))?;
let eocd = tail.windows(4).rposition(|v| v == b"PK\x05\x06").ok_or_else(|| invalid("missing ZIP directory"))?;
let end = tail.get(eocd..).filter(|v| v.len() >= 22).ok_or_else(|| invalid("truncated ZIP directory"))?;
let u16_at = |offset| -> Result<u16> {
let array: [u8; 2] = end.get(offset..offset + 2).ok_or_else(|| invalid("truncated ZIP directory"))?.try_into().map_err(invalid)?;
Ok(u16::from_le_bytes(array))
};
let declared = u16_at(10)? as usize;
if declared > MAX_ENTRIES || u16_at(4)? != 0 || u16_at(6)? != 0 || u16_at(8)? as usize != declared || end.len() != 22 + u16_at(20)? as usize {
return Err(invalid("too many entries or unsupported ZIP directory"));
}
let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(invalid)?;
if u16_at(4)? != 0
|| u16_at(6)? != 0
|| u16_at(8)? as usize != zip.len()
|| u16_at(10)? as usize != zip.len()
|| end.len() != 22 + u16_at(20)? as usize
{
return Err(invalid("duplicate entries or unsupported ZIP directory"));
}
let mut names = BTreeSet::new();
let mut total = 0u64;
for i in 0..zip.len() {
let part = zip.by_index_raw(i).map_err(invalid)?;
let name = part.name();
if name.is_empty()
|| name.contains('\\')
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|| !names.insert(name.to_owned())
{
return Err(invalid("duplicate or invalid archive entry"));
}
total = total.checked_add(part.size()).ok_or_else(|| invalid("expanded size overflow"))?;
if part.size() > MAX_DESIGN_BYTES as u64 || total > MAX_EXPANDED as u64 {
return Err(invalid("expanded package too large"));
}
}
Ok(Self { zip, blobs: BTreeMap::new(), hydrated: 0 })
}
fn part(&mut self, name: &str) -> Result<Vec<u8>> {
self.limited_part(name, MAX_DESIGN_BYTES)
}
fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>> {
let part = self.zip.by_name(name).map_err(|e| invalid(format!("required part `{name}`: {e}")))?;
let expected = part.size();
if expected > limit as u64 {
return Err(invalid("part too large"));
}
let mut bytes = Vec::with_capacity(expected as usize);
part.take(limit as u64 + 1).read_to_end(&mut bytes).map_err(invalid)?;
if bytes.len() > limit || bytes.len() as u64 != expected {
return Err(invalid("part too large or length mismatch"));
}
Ok(bytes)
}
fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8]> {
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
let hash = name
.strip_prefix("blobs/")
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
.ok_or_else(|| invalid("invalid blob reference"))?;
if !self.blobs.contains_key(name) {
let bytes = self.part(name)?;
if digest(&bytes) != hash {
return Err(invalid(format!("blob hash mismatch: {name}")));
}
self.blobs.insert(name.to_owned(), bytes);
}
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
// Count repeated references too: a tiny archive must not expand into unbounded
// owned document strings/arrays when the same blob is referenced repeatedly.
self.hydrated =
self.hydrated.checked_add(bytes.len()).filter(|&n| n <= MAX_EXPANDED).ok_or_else(|| invalid("resolved payloads too large"))?;
Ok(bytes)
}
}
fn encode_preview(preview: &DesignPreview, document: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
if preview.meshes.is_empty() || preview.meshes.len() > 1024 {
return Err(invalid("preview body count"));
}
let mut data = Vec::new();
let mut entries = Vec::new();
let mut triangles = 0usize;
for mesh in &preview.meshes {
triangles = triangles.saturating_add(mesh.triangles.len() / 3);
let size = mesh.positions.len().saturating_mul(8).saturating_add(mesh.triangles.len().saturating_mul(4)).saturating_add(24);
if data.len().saturating_add(size) > MAX_PREVIEW_BYTES || triangles > MAX_PREVIEW_TRIANGLES || mesh.name.len() > 1024 {
return Err(invalid("preview budget"));
}
let bytes = encode_mesh(mesh)?;
entries.push(json!({"name":mesh.name,"color":mesh.color,"offset":data.len(),"length":bytes.len()}));
data.extend_from_slice(&bytes);
}
let manifest = serde_json::to_vec(&json!({
"format":PREVIEW_VERSION, "build":env!("CARGO_PKG_VERSION"),
"document":digest(document), "payload":digest(&data), "meshes":entries
}))
.map_err(invalid)?;
if manifest.len() > 65_536 {
return Err(invalid("preview manifest budget"));
}
Ok((manifest, data))
}
/// Read only an optional provisional display cache; any corruption/incompatibility is a miss.
/// The normal design reader must still validate and rebuild before enabling the design.
pub fn read_design_preview(bytes: &[u8]) -> Option<DesignPreview> {
if bytes.len() > MAX_DESIGN_BYTES || !is_design_container(bytes) {
return None;
}
read_preview(bytes).ok()
}
fn read_preview(bytes: &[u8]) -> Result<DesignPreview> {
let mut reader = PackageReader::new(bytes)?;
let container: Value = serde_json::from_slice(&reader.limited_part("container.json", 4096)?).map_err(invalid)?;
if !matches!(container.get("format").and_then(Value::as_str), Some(VERSION | LEGACY_VERSION)) {
return Err(invalid("unsupported container version"));
}
let manifest: Value = serde_json::from_slice(&reader.limited_part("cache/preview.json", 65_536)?).map_err(invalid)?;
if manifest.get("format").and_then(Value::as_str) != Some(PREVIEW_VERSION)
|| manifest.get("build").and_then(Value::as_str) != Some(env!("CARGO_PKG_VERSION"))
{
return Err(invalid("incompatible preview"));
}
if manifest.get("document").and_then(Value::as_str) != Some(digest(&reader.part("document.json")?).as_str()) {
return Err(invalid("stale preview"));
}
let data = reader.limited_part("cache/preview.bin", MAX_PREVIEW_BYTES)?;
if manifest.get("payload").and_then(Value::as_str) != Some(digest(&data).as_str()) {
return Err(invalid("damaged preview"));
}
let entries =
manifest.get("meshes").and_then(Value::as_array).filter(|m| !m.is_empty() && m.len() <= 1024).ok_or_else(|| invalid("preview bodies"))?;
let mut preview = DesignPreview::default();
let mut end = 0usize;
let mut triangles = 0usize;
for entry in entries {
let offset = entry.get("offset").and_then(Value::as_u64).and_then(|n| usize::try_from(n).ok()).ok_or_else(|| invalid("preview offset"))?;
let length = entry.get("length").and_then(Value::as_u64).and_then(|n| usize::try_from(n).ok()).ok_or_else(|| invalid("preview length"))?;
if offset != end {
return Err(invalid("overlapping preview bodies"));
}
end = end.checked_add(length).ok_or_else(|| invalid("preview overflow"))?;
let mut mesh = MeshData { name: String::new(), positions: Vec::new(), triangles: Vec::new(), color: None };
decode_mesh(data.get(offset..end).ok_or_else(|| invalid("truncated preview"))?, &mut mesh)?;
triangles = triangles.saturating_add(mesh.triangles.len() / 3);
if triangles > MAX_PREVIEW_TRIANGLES {
return Err(invalid("preview triangles"));
}
mesh.name = entry.get("name").and_then(Value::as_str).filter(|n| n.len() <= 1024).ok_or_else(|| invalid("preview name"))?.into();
if !entry.get("color").is_none_or(Value::is_null) {
let color: [f32; 3] = serde_json::from_value(entry.get("color").cloned().unwrap_or(Value::Null)).map_err(invalid)?;
if color.iter().any(|v| !v.is_finite() || !(0.0..=1.0).contains(v)) {
return Err(invalid("preview color"));
}
mesh.color = Some(color);
}
preview.meshes.push(mesh);
}
if end != data.len() {
return Err(invalid("trailing preview data"));
}
Ok(preview)
}
pub(super) fn read(bytes: &[u8]) -> Result<Document> {
let mut reader = PackageReader::new(bytes)?;
let version: Value = serde_json::from_slice(&reader.part("container.json")?).map_err(invalid)?;
if !matches!(version.get("format").and_then(Value::as_str), Some(VERSION | LEGACY_VERSION)) {
return Err(invalid("unsupported container version"));
}
let bytes = reader.part("document.json")?;
let metadata: Value = serde_json::from_slice(&bytes).map_err(invalid)?;
let text = std::str::from_utf8(&bytes).map_err(invalid)?;
// Migrations and all document validation run on the small parametric metadata.
// Binary arrays have their own equivalent numeric/index validation below.
let mut doc = Document::from_json(text)?;
for (index, feature) in doc.features.iter_mut().enumerate() {
let value = metadata.get("features").and_then(|v| v.get(index)).ok_or_else(|| invalid("missing feature metadata"))?;
match &mut feature.kind {
FeatureKind::Import { step, .. } => {
if !step.is_empty() {
return Err(invalid("ambiguous embedded STEP and blob"));
}
*step = std::str::from_utf8(reader.blob(value, "step_blob", "step")?).map_err(invalid)?.to_owned();
}
FeatureKind::MeshImport { meshes, .. } => {
for (index, mesh) in meshes.iter_mut().enumerate() {
if !mesh.positions.is_empty() || !mesh.triangles.is_empty() {
return Err(invalid("ambiguous embedded mesh and blob"));
}
let value = value.get("meshes").and_then(|v| v.get(index)).ok_or_else(|| invalid("missing mesh metadata"))?;
decode_mesh(reader.blob(value, "mesh_blob", "mesh")?, mesh)?;
}
}
_ => {}
}
}
for (index, canvas) in doc.canvases.iter_mut().enumerate() {
if !canvas.data.is_empty() || !matches!(canvas.format.as_str(), "png" | "jpeg") {
return Err(invalid("invalid canvas metadata"));
}
let value = metadata.get("canvases").and_then(|v| v.get(index)).ok_or_else(|| invalid("missing canvas metadata"))?;
let bytes = reader.blob(value, "image_blob", &canvas.format)?;
if bytes.len() > solidcraft_doc::canvas::MAX_CANVAS_BYTES {
return Err(invalid("canvas too large"));
}
canvas.data = solidcraft_doc::canvas::base64_encode(bytes);
}
Ok(doc)
}
#[cfg(test)]
mod tests {
use super::*;
use solidcraft_geom::Vec2;
fn mesh() -> MeshData {
MeshData {
name: "Precise mesh".into(),
positions: vec![0.123456789012345, -0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0],
triangles: vec![0, 1, 2],
color: Some([0.25, 0.5, 0.75]),
}
}
fn document() -> Document {
let mut doc = Document::new("Payloads");
for _ in 0..2 {
doc.add_feature(
FeatureKind::Import { file: "example.step".into(), step: "ISO-10303-21;\nEND-ISO-10303-21;".into(), components: Vec::new() },
None,
)
.unwrap();
}
let mut other = mesh();
other.name = "Same triangles, separate appearance".into();
other.color = None;
doc.add_feature(FeatureKind::MeshImport { file: "example.obj".into(), meshes: vec![mesh(), other] }, None).unwrap();
doc.canvases.push(solidcraft_doc::canvas::Canvas {
id: 99,
name: "Reference".into(),
format: "png".into(),
data: solidcraft_doc::canvas::base64_encode(b"original image bytes"),
pixels: [1, 1],
plane: solidcraft_doc::PlaneRef::Origin { name: "XY".into() },
center: Vec2::new(0.0, 0.0),
width: 10.0,
angle: 0.0,
opacity: 0.5,
flip: false,
visible: true,
component: 0,
});
doc
}
fn parts(bytes: &[u8]) -> BTreeMap<String, Vec<u8>> {
let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();
(0..zip.len())
.map(|i| {
let mut entry = zip.by_index(i).unwrap();
let name = entry.name().to_owned();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
(name, bytes)
})
.collect()
}
fn package(parts: &BTreeMap<String, Vec<u8>>) -> Vec<u8> {
let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
for (name, bytes) in parts {
zip.start_file(name, zip::write::SimpleFileOptions::default()).unwrap();
zip.write_all(bytes).unwrap();
}
zip.finish().unwrap().into_inner()
}
#[test]
fn optional_preview_is_bounded_keyed_and_never_authoritative() {
let doc = Document::from_json(&document().to_json()).unwrap();
let preview = DesignPreview { meshes: vec![mesh()] };
let bytes = write_design_container_with_preview(&doc, DesignCompression::Fast, Some(&preview)).unwrap();
let loaded = read_design_preview(&bytes).unwrap();
assert_eq!(loaded.meshes, preview.meshes);
assert_eq!(crate::read_design(&bytes).unwrap(), doc);
let original = parts(&bytes);
for field in ["format", "build", "document", "payload"] {
let mut entries = original.clone();
let mut manifest: Value = serde_json::from_slice(&entries["cache/preview.json"]).unwrap();
manifest[field] = json!("obsolete");
entries.insert("cache/preview.json".into(), serde_json::to_vec(&manifest).unwrap());
let changed = package(&entries);
assert!(read_design_preview(&changed).is_none(), "{field}");
assert_eq!(crate::read_design(&changed).unwrap(), doc);
}
let mut entries = original.clone();
let mut metadata: Value = serde_json::from_slice(&entries["document.json"]).unwrap();
metadata["name"] = json!("Changed document");
entries.insert("document.json".into(), serde_json::to_vec(&metadata).unwrap());
assert!(read_design_preview(&package(&entries)).is_none());
for damaged in [vec![], vec![0; 128], vec![0; MAX_PREVIEW_BYTES + 1]] {
let mut entries = original.clone();
entries.insert("cache/preview.bin".into(), damaged);
let changed = package(&entries);
assert!(read_design_preview(&changed).is_none());
assert_eq!(crate::read_design(&changed).unwrap(), doc);
}
// Even a matching checksum does not excuse invalid indices or overlapping slices.
for overlap in [false, true] {
let mut entries = original.clone();
let mut manifest: Value = serde_json::from_slice(&entries["cache/preview.json"]).unwrap();
if overlap {
manifest["meshes"][0]["offset"] = json!(1);
} else {
let data = entries.get_mut("cache/preview.bin").unwrap();
let end = data.len();
data[end - 4..].copy_from_slice(&u32::MAX.to_le_bytes());
manifest["payload"] = json!(digest(data));
}
entries.insert("cache/preview.json".into(), serde_json::to_vec(&manifest).unwrap());
assert!(read_design_preview(&package(&entries)).is_none());
}
let mut too_large = mesh();
too_large.triangles = vec![0; (MAX_PREVIEW_TRIANGLES + 1) * 3];
let bytes = write_design_container_with_preview(&doc, DesignCompression::Fast, Some(&DesignPreview { meshes: vec![too_large] })).unwrap();
assert!(read_design_preview(&bytes).is_none());
assert_eq!(crate::read_design(&bytes).unwrap(), doc);
assert!(read_design_preview(&crate::write_design(&doc)).is_none());
}
#[test]
fn round_trip_deduplicates_and_preserves_binary_precision() {
let source = Document::from_json(&document().to_json()).unwrap();
let bytes = write_design_container(&source).unwrap();
assert!(is_design_container(&bytes));
let reopened = crate::read_design(&bytes).unwrap();
assert_eq!(source, reopened);
let FeatureKind::MeshImport { meshes, .. } = &reopened.features[2].kind else { panic!() };
assert_eq!(meshes[0].positions[1].to_bits(), (-0.0f64).to_bits());
let entries = parts(&bytes);
assert_eq!(entries.keys().filter(|n| n.ends_with(".step")).count(), 1);
assert_eq!(entries.keys().filter(|n| n.ends_with(".mesh")).count(), 1);
assert_eq!(entries.keys().filter(|n| n.ends_with(".png")).count(), 1);
let json: Value = serde_json::from_slice(&entries["document.json"]).unwrap();
assert_eq!(json["features"][2]["meshes"][0]["positions"], json!([]));
assert_eq!(crate::read_design(&crate::write_design(&reopened)).unwrap(), source);
assert_eq!(write_design_container(&source).unwrap(), bytes, "deterministic archive");
let compact = write_design_container_with_compression(&source, DesignCompression::Compact).unwrap();
assert_eq!(crate::read_design(&compact).unwrap(), source);
}
#[test]
fn draft_container_marker_remains_readable() {
let doc = Document::from_json(&document().to_json()).unwrap();
let mut entries = parts(&write_design_container(&doc).unwrap());
let marker: Value = serde_json::from_slice(&entries["container.json"]).unwrap();
assert_eq!(marker["format"], VERSION);
entries.insert("container.json".into(), serde_json::to_vec(&json!({"format": LEGACY_VERSION})).unwrap());
assert_eq!(crate::read_design(&package(&entries)).unwrap(), doc);
}
#[test]
fn required_blobs_must_exist_and_match_hashes() {
let bytes = write_design_container(&document()).unwrap();
let mut entries = parts(&bytes);
let name = entries.keys().find(|n| n.ends_with(".step")).unwrap().clone();
let original = entries.remove(&name).unwrap();
assert!(crate::read_design(&package(&entries)).unwrap_err().to_string().contains("required part"));
entries.insert(name.clone(), b"damaged".to_vec());
assert!(crate::read_design(&package(&entries)).unwrap_err().to_string().contains("hash mismatch"));
entries.insert(name, original);
entries.insert("cache/obsolete.bin".into(), b"untrusted optional cache".to_vec());
assert!(crate::read_design(&package(&entries)).is_ok());
}
#[test]
fn refuses_hostile_mesh_lengths_numbers_indices_and_versions() {
let valid = encode_mesh(&mesh()).unwrap();
for offset in [8, 16] {
let mut bytes = valid.clone();
bytes[offset..offset + 8].copy_from_slice(&u64::MAX.to_le_bytes());
assert!(decode_mesh(&bytes, &mut mesh()).is_err());
}
let mut bytes = valid.clone();
bytes[24..32].copy_from_slice(&f64::NAN.to_le_bytes());
assert!(decode_mesh(&bytes, &mut mesh()).is_err());
let mut bytes = valid.clone();
let offset = bytes.len() - 4;
bytes[offset..].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(decode_mesh(&bytes, &mut mesh()).is_err());
for length in 0..valid.len() {
assert!(decode_mesh(&valid[..length], &mut mesh()).is_err());
}
let mut bytes = valid;
bytes[7] = b'2';
assert!(decode_mesh(&bytes, &mut mesh()).is_err());
}
#[test]
fn hostile_archive_paths_versions_and_truncations_are_errors() {
let bytes = write_design_container(&document()).unwrap();
let mut entries = parts(&bytes);
entries.insert("../escape".into(), vec![]);
assert!(crate::read_design(&package(&entries)).is_err());
entries.remove("../escape");
entries.insert("container.json".into(), br#"{"format":"solidcraft-container/999"}"#.to_vec());
assert!(crate::read_design(&package(&entries)).unwrap_err().to_string().contains("unsupported container version"));
for length in (0..bytes.len()).step_by(17) {
assert!(crate::read_design(&bytes[..length]).is_err());
}
}
#[test]
fn rejects_duplicate_entries_and_declared_expansion_before_reading_payloads() {
let bytes = write_design_container(&document()).unwrap();
let mut entries = parts(&bytes);
entries.insert("cache/one.bin".into(), vec![1]);
entries.insert("cache/two.bin".into(), vec![2]);
let mut duplicate = package(&entries);
let offsets: Vec<_> = duplicate.windows(13).enumerate().filter_map(|(i, v)| (v == b"cache/two.bin").then_some(i)).collect();
// Both the local header and central-directory name are changed.
for i in offsets {
duplicate[i..i + 13].copy_from_slice(b"cache/one.bin");
}
assert!(crate::read_design(&duplicate).is_err());
let mut bomb = bytes.clone();
let central = bomb.windows(4).position(|v| v == b"PK\x01\x02").unwrap();
bomb[central + 24..central + 28].copy_from_slice(&((MAX_DESIGN_BYTES + 1) as u32).to_le_bytes());
assert!(crate::read_design(&bomb).unwrap_err().to_string().contains("expanded package too large"));
let mut excessive = bytes;
let end = excessive.windows(4).rposition(|v| v == b"PK\x05\x06").unwrap();
excessive[end + 8..end + 12].copy_from_slice(&[0xff; 4]);
assert!(crate::read_design(&excessive).is_err());
}
#[test]
fn rejects_invalid_references_and_repeated_payload_expansion() {
let bytes = write_design_container(&document()).unwrap();
let mut entries = parts(&bytes);
let mut metadata: Value = serde_json::from_slice(&entries["document.json"]).unwrap();
let reference = metadata["features"][0].clone();
let mut reader = PackageReader::new(&bytes).unwrap();
reader.hydrated = MAX_EXPANDED;
assert!(reader.blob(&reference, "step_blob", "step").unwrap_err().to_string().contains("resolved payloads too large"));
for name in ["../escape.step", "blobs/not-a-hash.step", "blobs/00.png"] {
metadata["features"][0]["step_blob"] = json!(name);
entries.insert("document.json".into(), serde_json::to_vec(&metadata).unwrap());
assert!(crate::read_design(&package(&entries)).is_err());
}
}
#[test]
fn deterministic_archive_mutations_never_panic() {
let preview = DesignPreview { meshes: vec![mesh()] };
let bytes = write_design_container_with_preview(&document(), DesignCompression::Fast, Some(&preview)).unwrap();
assert!(read_design_preview(&bytes).is_some());
for end in 0..bytes.len() {
assert!(
std::panic::catch_unwind(|| {
let _ = crate::read_design(&bytes[..end]);
let _ = read_design_preview(&bytes[..end]);
})
.is_ok(),
"truncated at {end}"
);
}
let mut seed = 0x303_u64;
for _ in 0..512 {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
let mut changed = bytes.clone();
let index = seed as usize % changed.len();
changed[index] ^= (seed >> 32) as u8 | 1;
assert!(
std::panic::catch_unwind(|| {
let _ = crate::read_design(&changed);
let _ = read_design_preview(&changed);
})
.is_ok()
);
}
}
#[test]
fn metadata_round_trip_preserves_new_and_existing_fields() {
let mut doc = Document::new("Empty");
doc.units = "in".into();
doc.named_views.push(solidcraft_doc::NamedView { name: "View".into(), ..Default::default() });
doc.param_comments.insert("test".into(), "metadata".into());
let expected = crate::read_design(&crate::write_design(&doc)).unwrap();
assert_eq!(crate::read_design(&write_design_container(&doc).unwrap()).unwrap(), expected);
}
/// Public-safe generated data, no geometry evaluation. Print medians without
/// timing assertions, which would be unreliable on shared build machines.
#[test]
#[ignore = "manual IO performance measurement"]
fn benchmark_design_storage() {
let mut doc = Document::new("Generated mesh benchmark");
let mut large = mesh();
large.positions = (0..300_000).map(|i| (i as f64 * 0.001234567890123).sin() * 100.0).collect();
large.triangles = (0..300_000).map(|i| (i % 100_000) as u32).collect();
doc.add_feature(FeatureKind::MeshImport { file: "generated.obj".into(), meshes: vec![large] }, None).unwrap();
measure_storage(&doc, "mesh");
}
#[test]
#[ignore = "manual IO performance measurement"]
fn benchmark_step_storage() {
use solidcraft_geom::Vec3;
let bodies: Vec<_> = (0..512)
.map(|i| {
let corner = Vec3::new((i % 32) as f64 * 12.0, (i / 32) as f64 * 12.0, 0.0);
solidcraft_kernel::box_solid(corner, corner + Vec3::new(10.0, 10.0, 10.0)).unwrap()
})
.collect();
let step = solidcraft_kernel::step_export(&bodies.iter().collect::<Vec<_>>(), "SolidCraft storage benchmark").unwrap();
let mut doc = Document::new("Generated STEP benchmark");
doc.add_feature(FeatureKind::Import { file: "generated.step".into(), step, components: Vec::new() }, None).unwrap();
measure_storage(&doc, "step");
}
fn measure_storage(doc: &Document, label: &str) {
let json = crate::write_design(doc);
let container = write_design_container(doc).unwrap();
let dir = std::env::var_os("CARGO_TARGET_DIR").map(std::path::PathBuf::from).unwrap_or_else(|| std::path::PathBuf::from("target"));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(format!("issue303-{label}.json")), &json).unwrap();
std::fs::write(dir.join(format!("issue303-{label}.solidcraft")), &container).unwrap();
let mut timings = [Vec::new(), Vec::new(), Vec::new(), Vec::new()];
for _ in 0..9 {
let start = std::time::Instant::now();
std::hint::black_box(crate::write_design(doc));
timings[0].push(start.elapsed().as_secs_f64() * 1000.0);
let start = std::time::Instant::now();
std::hint::black_box(write_design_container(doc).unwrap());
timings[1].push(start.elapsed().as_secs_f64() * 1000.0);
let start = std::time::Instant::now();
let plain = crate::read_design(&json).unwrap();
timings[2].push(start.elapsed().as_secs_f64() * 1000.0);
let start = std::time::Instant::now();
let zipped = crate::read_design(&container).unwrap();
timings[3].push(start.elapsed().as_secs_f64() * 1000.0);
assert_eq!(plain, zipped);
}
for times in &mut timings {
times.sort_by(f64::total_cmp);
}
eprintln!(
"{label}: bytes json={} container={}; median ms save_json={:.3} save_container={:.3} read_json={:.3} read_container={:.3}",
json.len(),
container.len(),
timings[0][4],
timings[1][4],
timings[2][4],
timings[3][4]
);
let mut saves = [Vec::new(), Vec::new()];
for round in 0..9 {
// Alternate ordering; include encoding, flushing, replacement and backup.
for index in if round % 2 == 0 { [0, 1] } else { [1, 0] } {
let start = std::time::Instant::now();
let bytes = if index == 0 { crate::write_design(doc) } else { write_design_container(doc).unwrap() };
crate::write_atomic(&dir.join(format!("issue303-{label}-atomic-{index}.solidcraft")), &bytes, true).unwrap();
saves[index].push(start.elapsed().as_secs_f64() * 1000.0);
}
}
for times in &mut saves {
times.sort_by(f64::total_cmp);
}
eprintln!("{label}: median atomic-save ms json={:.3} container={:.3}", saves[0][4], saves[1][4]);
}
}
crates/io/src/design_metadata.rs
+130
View File
@@ -0,0 +1,130 @@
//! Replace only top-level fields during serialization. This avoids cloning or turning
//! large embedded payloads into JSON values and automatically preserves future fields.
use serde::ser::{Serialize, SerializeMap, SerializeStruct, Serializer};
use serde_json::{Map, Value};
pub(super) fn replace<T: Serialize>(value: &T, fields: Map<String, Value>) -> serde_json::Result<Value> {
value.serialize(Replace(fields))
}
struct Replace(Map<String, Value>);
struct Fields {
replacements: Map<String, Value>,
values: Map<String, Value>,
}
impl Fields {
fn field<T: ?Sized + Serialize>(&mut self, key: &str, value: &T) -> serde_json::Result<()> {
let value = match self.replacements.remove(key) {
Some(value) => value,
None => serde_json::to_value(value)?,
};
self.values.insert(key.to_owned(), value);
Ok(())
}
}
impl SerializeStruct for Fields {
type Ok = Value;
type Error = serde_json::Error;
fn serialize_field<T: ?Sized + Serialize>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error> {
self.field(key, value)
}
fn end(self) -> Result<Value, Self::Error> {
Ok(Value::Object(self.values))
}
}
impl SerializeMap for Fields {
type Ok = Value;
type Error = serde_json::Error;
fn serialize_key<T: ?Sized + Serialize>(&mut self, _key: &T) -> Result<(), Self::Error> {
Err(serde::ser::Error::custom("metadata requires map entries"))
}
fn serialize_value<T: ?Sized + Serialize>(&mut self, _value: &T) -> Result<(), Self::Error> {
Err(serde::ser::Error::custom("metadata requires map entries"))
}
fn serialize_entry<K: ?Sized + Serialize, V: ?Sized + Serialize>(&mut self, key: &K, value: &V) -> Result<(), Self::Error> {
let key = serde_json::to_value(key)?;
let key = key.as_str().ok_or_else(|| <Self::Error as serde::ser::Error>::custom("metadata key must be a string"))?;
self.field(key, value)
}
fn end(self) -> Result<Value, Self::Error> {
Ok(Value::Object(self.values))
}
}
macro_rules! forward {
($($name:ident($($arg:ident: $ty:ty),*);)*) => {$(
fn $name(self, $($arg: $ty),*) -> Result<Self::Ok, Self::Error> {
serde_json::value::Serializer.$name($($arg),*)
}
)*};
}
impl Serializer for Replace {
type Ok = Value;
type Error = serde_json::Error;
type SerializeSeq = <serde_json::value::Serializer as Serializer>::SerializeSeq;
type SerializeTuple = <serde_json::value::Serializer as Serializer>::SerializeTuple;
type SerializeTupleStruct = <serde_json::value::Serializer as Serializer>::SerializeTupleStruct;
type SerializeTupleVariant = <serde_json::value::Serializer as Serializer>::SerializeTupleVariant;
type SerializeMap = Fields;
type SerializeStruct = Fields;
type SerializeStructVariant = <serde_json::value::Serializer as Serializer>::SerializeStructVariant;
forward! {
serialize_bool(v: bool); serialize_i8(v: i8); serialize_i16(v: i16); serialize_i32(v: i32); serialize_i64(v: i64);
serialize_u8(v: u8); serialize_u16(v: u16); serialize_u32(v: u32); serialize_u64(v: u64);
serialize_f32(v: f32); serialize_f64(v: f64); serialize_char(v: char); serialize_str(v: &str);
serialize_bytes(v: &[u8]); serialize_none(); serialize_unit(); serialize_unit_struct(name: &'static str);
serialize_unit_variant(name: &'static str, index: u32, variant: &'static str);
}
fn serialize_some<T: ?Sized + Serialize>(self, value: &T) -> Result<Value, Self::Error> {
serde_json::to_value(value)
}
fn serialize_newtype_struct<T: ?Sized + Serialize>(self, _name: &'static str, value: &T) -> Result<Value, Self::Error> {
serde_json::to_value(value)
}
fn serialize_newtype_variant<T: ?Sized + Serialize>(
self,
name: &'static str,
index: u32,
variant: &'static str,
value: &T,
) -> Result<Value, Self::Error> {
serde_json::value::Serializer.serialize_newtype_variant(name, index, variant, value)
}
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
serde_json::value::Serializer.serialize_seq(len)
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error> {
serde_json::value::Serializer.serialize_tuple(len)
}
fn serialize_tuple_struct(self, name: &'static str, len: usize) -> Result<Self::SerializeTupleStruct, Self::Error> {
serde_json::value::Serializer.serialize_tuple_struct(name, len)
}
fn serialize_tuple_variant(
self,
name: &'static str,
index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeTupleVariant, Self::Error> {
serde_json::value::Serializer.serialize_tuple_variant(name, index, variant, len)
}
fn serialize_struct_variant(
self,
name: &'static str,
index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeStructVariant, Self::Error> {
serde_json::value::Serializer.serialize_struct_variant(name, index, variant, len)
}
fn serialize_map(self, _len: Option<usize>) -> Result<Fields, Self::Error> {
Ok(Fields { replacements: self.0, values: Map::new() })
}
fn serialize_struct(self, _name: &'static str, _len: usize) -> Result<Fields, Self::Error> {
self.serialize_map(None)
}
}
crates/io/src/lib.rs
+26 -3
View File
@@ -1,11 +1,18 @@
//! File formats. Designs are saved as JSON (`.solidcraft`): the parametric document only, the
//! model is recomputed on load. Bodies export as STL (binary or ASCII), OBJ, STEP and 3MF; STEP
//! File formats. Designs use JSON or versioned containers (`.solidcraft`); the authoritative
//! document rebuilds on load and optional display caches remain provisional. Bodies export
//! as STL (binary or ASCII), OBJ, STEP and 3MF; STEP
//! files import as a base feature (see [`step_import_feature`]), 3MF and STL files as mesh bodies
//! (see [`mesh_import_feature`]).
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
mod design;
mod design_metadata;
mod dxf;
pub use design::{
DesignCompression, DesignPreview, is_design_container, read_design_preview, write_design_container, write_design_container_with_compression,
write_design_container_with_preview,
};
mod legacy_dir;
pub mod safe_write;
mod sketch2d;
@@ -35,7 +42,10 @@ pub const DESIGN_EXTS: [&str; 2] = [DESIGN_EXT, LEGACY_DESIGN_EXT];
/// Is this a design file name (`.solidcraft`, or the legacy extension), any case?
pub fn is_design_path(path: &str) -> bool {
std::path::Path::new(path).extension().and_then(|e| e.to_str()).is_some_and(|e| DESIGN_EXTS.iter().any(|x| e.eq_ignore_ascii_case(x)))
std::path::Path::new(path)
.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| DESIGN_EXTS.iter().chain(["json"].iter()).any(|x| e.eq_ignore_ascii_case(x)))
}
/// Largest design file we read.
pub const MAX_DESIGN_BYTES: usize = 256 << 20;
@@ -359,6 +369,9 @@ pub fn read_design(bytes: &[u8]) -> Result<Document> {
if bytes.len() > MAX_DESIGN_BYTES {
return Err(IoError::Invalid("design file too large".into()));
}
if is_design_container(bytes) {
return design::read(bytes);
}
let s = std::str::from_utf8(bytes).map_err(|_| IoError::Invalid("design file is not UTF-8 JSON".into()))?;
Ok(Document::from_json(s)?)
}
@@ -551,6 +564,16 @@ mod tests {
assert_eq!(f.body_names, ["Body1"]);
assert_eq!(f.body_names.len(), 1);
assert!(matches!(&f.kind, FeatureKind::Import { file, .. } if file == "My Part.step"));
let mut imported = Document::new("Imported STEP round trip");
imported.add_feature(f.kind, None).unwrap();
let packaged = read_design(&write_design_container(&imported).unwrap()).unwrap();
assert_eq!(packaged, imported);
let mut rebuilt = Model::new();
rebuilt.evaluate(&packaged);
assert!(rebuilt.results.iter().all(|r| r.error.is_none()));
let state = rebuilt.state();
assert_eq!(state.bodies.len(), 1);
assert!((solidcraft_kernel::measure(&state.bodies[0].body).unwrap().volume - 6000.0).abs() < 1e-6);
assert!(step_import_feature(b"ISO-10303-21;", "x.step").is_err());
assert!(step_import_feature(&[0xff, 0xfe, 0x00], "x.step").is_err());
assert!(Format::from_name("a.dwg").is_err());