feat(automation): replay pointer events on the canvas

The new pointer op sends moves (canvas pixels, or a world point projected
through the camera the cursor edits with), button presses/releases and
wheel steps in order, and reports the cursor's world point after each, so
grip drags and picks go through the same cursor path as the mouse.
click now projects through that editing camera too (it used the model
camera inside an entered viewport), and the grips read reports each
grip's constraint axis.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/app/control/actions.rs

@@ -841,6 +841,7 @@ impl OpenCADStudio {
                     "point": [grip.world.x, grip.world.y, grip.world.z],
                     "shape": format!("{:?}", grip.shape),
                     "moves_entity": grip.is_midpoint,
+                    "axis": grip.axis.map(|a| [a.x, a.y, a.z]),
                 })
             })
             .collect::<Vec<_>>();
@@ -911,27 +912,33 @@ impl OpenCADStudio {
 
     /// Click the viewport where `point` (world) shows, as the left button
     /// would, holding Ctrl and/or Shift when `modifiers` names them.
-    pub(super) fn control_click(&mut self, req: &Value) -> Result<Task<Message>, Value> {
-        let i = self.active_tab;
-        let at = point(req)?;
-        let modifiers = req["modifiers"].as_str().unwrap_or("").to_ascii_lowercase();
-        let (vw, vh) = self.tabs[i].scene.selection.borrow().view.vp_size;
-        let tile = self.tabs[i].scene.active_model_tile_bounds(vw, vh);
+    /// Where a world point shows on the drawing canvas, through the camera
+    /// the cursor edits with: an entered viewport's, else the active tile's.
+    fn control_world_to_canvas(&self, at: glam::DVec3) -> Result<iced::Point, Value> {
+        let scene = &self.tabs[self.active_tab].scene;
+        let (vw, vh) = scene.selection.borrow().view.vp_size;
+        let (camera, bounds) = scene.viewport_edit_frame((vw, vh)).unwrap_or_else(|| {
+            (scene.camera.borrow().clone(), scene.active_model_tile_bounds(vw, vh))
+        });
         let local = iced::Rectangle {
-            width: tile.width,
-            height: tile.height,
+            width: bounds.width,
+            height: bounds.height,
             ..iced::Rectangle::default()
         };
-        let screen = self.tabs[i]
-            .scene
-            .camera
-            .borrow()
+        camera
             .project(at, local)
-            .ok_or_else(|| failure("invalid_point", "The point is not in view"))?;
+            .filter(|p| p.x.is_finite() && p.y.is_finite())
+            .map(|p| iced::Point::new(p.x + bounds.x, p.y + bounds.y))
+            .ok_or_else(|| failure("invalid_point", "The point is not in view"))
+    }
+
+    pub(super) fn control_click(&mut self, req: &Value) -> Result<Task<Message>, Value> {
+        let at = point(req)?;
+        let modifiers = req["modifiers"].as_str().unwrap_or("").to_ascii_lowercase();
+        let cursor = self.control_world_to_canvas(at)?;
         let saved = (self.ctrl_down, self.shift_down);
         self.ctrl_down = modifiers.contains("ctrl");
         self.shift_down = modifiers.contains("shift");
-        let cursor = iced::Point::new(screen.x + tile.x, screen.y + tile.y);
         let tasks = [
             self.update(Message::ViewportMove(cursor)),
             self.update(Message::ViewportLeftPress),
@@ -941,6 +948,86 @@ impl OpenCADStudio {
         Ok(Task::batch(tasks))
     }
 
+    /// Replay pointer events on the drawing canvas in order, as the mouse
+    /// sends them: moves (canvas pixels, or a world point projected through
+    /// the editing camera), button presses and releases, and wheel steps.
+    /// Each step reports where the cursor landed in the world, so a drag
+    /// shows the points the interaction actually received.
+    pub(super) fn control_pointer(&mut self, req: &Value) -> Result<Task<Message>, Value> {
+        let i = self.active_tab;
+        let events = req["events"]
+            .as_array()
+            .filter(|events| !events.is_empty())
+            .ok_or_else(|| failure("invalid_request", "Supply a non-empty events array"))?;
+        let modifiers = req["modifiers"].as_str().unwrap_or("").to_ascii_lowercase();
+        let coords = |value: &Value, n: usize| -> Option<Vec<f64>> {
+            let values: Vec<f64> = value.as_array()?.iter().filter_map(Value::as_f64).collect();
+            (values.len() == n && values.iter().all(|v| v.is_finite())).then_some(values)
+        };
+        let saved = (self.ctrl_down, self.shift_down);
+        self.ctrl_down = modifiers.contains("ctrl");
+        self.shift_down = modifiers.contains("shift");
+        let mut tasks = Vec::new();
+        let mut steps = Vec::new();
+        let mut outcome = Ok(());
+        for event in events {
+            let message = if let Some(xy) = event.get("move").and_then(|v| coords(v, 2)) {
+                Message::ViewportMove(iced::Point::new(xy[0] as f32, xy[1] as f32))
+            } else if let Some(at) = event.get("move_world").and_then(|v| coords(v, 3)) {
+                match self.control_world_to_canvas(glam::DVec3::new(at[0], at[1], at[2])) {
+                    Ok(cursor) => Message::ViewportMove(cursor),
+                    Err(error) => {
+                        outcome = Err(error);
+                        break;
+                    }
+                }
+            } else if let Some(button) = event.get("press").and_then(Value::as_str) {
+                match button {
+                    "left" => Message::ViewportLeftPress,
+                    "right" => Message::ViewportRightPress,
+                    "middle" => Message::ViewportMiddlePress,
+                    _ => {
+                        outcome = Err(failure("invalid_request", "press is left, right or middle"));
+                        break;
+                    }
+                }
+            } else if let Some(button) = event.get("release").and_then(Value::as_str) {
+                match button {
+                    "left" => Message::ViewportLeftRelease,
+                    "right" => Message::ViewportRightRelease,
+                    "middle" => Message::ViewportMiddleRelease,
+                    _ => {
+                        outcome = Err(failure("invalid_request", "release is left, right or middle"));
+                        break;
+                    }
+                }
+            } else if let Some(lines) = event.get("wheel").and_then(Value::as_f64) {
+                Message::ViewportScroll(iced::mouse::ScrollDelta::Lines {
+                    x: 0.0,
+                    y: lines as f32,
+                })
+            } else {
+                outcome = Err(failure(
+                    "invalid_request",
+                    "Each event is move, move_world, press, release or wheel",
+                ));
+                break;
+            };
+            tasks.push(self.update(message));
+            let tab = &self.tabs[i];
+            let world = tab.last_cursor_world;
+            steps.push(json!({
+                "screen":[tab.last_cursor_screen.x, tab.last_cursor_screen.y],
+                "world":[world.x, world.y, world.z],
+                "dragging_grip":tab.active_grip.is_some(),
+            }));
+        }
+        (self.ctrl_down, self.shift_down) = saved;
+        outcome?;
+        self.set_control_result(json!({"steps":steps}));
+        Ok(Task::batch(tasks))
+    }
+
     pub(super) fn control_set_property(&mut self, req: &Value) -> Result<Task<Message>, Value> {
         self.refresh_properties();
         let field = string(req, "field")?;

src/app/control/mod.rs

@@ -1070,6 +1070,7 @@ impl OpenCADStudio {
             "property" => self.control_set_property(req)?,
             "grip_drag" => self.control_grip_drag(req)?,
             "click" => self.control_click(req)?,
+            "pointer" => self.control_pointer(req)?,
             "set_properties" => self.control_set_record_properties(req)?,
             "action" => self.control_ui_action(req)?,
             #[cfg(not(target_arch = "wasm32"))]

src/mcp.rs

@@ -32,7 +32,7 @@ const MODERN_PROTOCOL_VERSION: &str = "2026-07-28";
 ///
 /// `block_delete` gained its optional `force` parameter.
 #[cfg(test)]
-const TOOL_SCHEMA_DIGEST: &str = "96510df2d38cf3f7feb99e50e9491dda6fc1ee9221dc82020303df918292f580";
+const TOOL_SCHEMA_DIGEST: &str = "4f193abfb8341acd6dc1a86ef7fe85148bc0bb76d1df700bcd289f79291a1d0f";
 const MAX_REQUEST: usize = 1_048_576;
 const MAX_RESPONSE: u64 = 16 * 1024 * 1024;
 const CACHE_TTL_MS: u64 = 3_600_000;
@@ -85,6 +85,7 @@ const EXECUTE_OPS: &[&str] = &[
     "property",
     "grip_drag",
     "click",
+    "pointer",
     "set_properties",
     "action",
     "embed_image",

src/mcp_ops.rs

@@ -304,6 +304,22 @@ pub static CLICK: OpDef = OpDef {
     example: r#"{"op":"click","point":[10,0,0],"modifiers":"ctrl"}"#,
 };
 
+pub static POINTER: OpDef = OpDef {
+    name: "pointer",
+    doc: "Replay mouse events on the drawing canvas in order, as the mouse sends them: move to canvas pixels or to a world point (projected through the camera the cursor edits with), press and release a button, turn the wheel. A grip press-move-release goes through the same cursor path as a hand drag. Each step reports the cursor's world point.",
+    batchable: true,
+    params: &[
+        req(
+            "events",
+            Ty::ArrayOf(&Ty::Raw(r#"{"type":"object","properties":{"move":{"type":"array","items":{"type":"number"},"minItems":2,"maxItems":2},"move_world":{"type":"array","items":{"type":"number"},"minItems":3,"maxItems":3},"press":{"type":"string","enum":["left","right","middle"]},"release":{"type":"string","enum":["left","right","middle"]},"wheel":{"type":"number"}}}"#)),
+            "Events in order; each holds one of move, move_world, press, release or wheel (lines, positive zooms in).",
+        ),
+        opt("modifiers", Ty::Str, "Keys held throughout: \"ctrl\", \"shift\" or \"ctrl+shift\"."),
+    ],
+    rules: &[],
+    example: r#"{"op":"pointer","events":[{"move_world":[10,0,0]},{"press":"left"},{"release":"left"},{"move_world":[12,3,0]},{"press":"left"},{"release":"left"}]}"#,
+};
+
 pub static SET_PROPERTIES: OpDef = OpDef {
     name: "set_properties",
     doc: "Atomically update record properties by RFC 6901 JSON pointer paths.",
@@ -877,6 +893,7 @@ pub static OPS: &[&OpDef] = &[
     &PROPERTY,
     &GRIP_DRAG,
     &CLICK,
+    &POINTER,
     &SET_PROPERTIES,
     &ACTION,
     &EMBED_IMAGE,