feat: expand typed control-driven editor workflows

Author: gajop

docs/design/programmatic-control.md

@@ -9,8 +9,8 @@ A control channel lets a script drive a running SpringBoard the way a user
 drives it — open an editor, set its fields, run a command, place the camera,
 capture the result — without synthesising X11 input.
 
-Status: **implemented for editor fields, registered commands, camera state, and
-ordered capture; domain handles are the next layer.**
+Status: **implemented for editor fields, domain dialogs, registered commands,
+camera state, and ordered capture.**
 
 ## Why
 
@@ -47,7 +47,7 @@ The Python client detects that replacement, waits for the new `instance_id`,
 refreshes the schema, and reconnects subsequent calls; a caller does not need
 to rebuild its editor or camera handles.
 
-## Three surfaces, all first-class
+## First-class surfaces
 
 **Editors** — `ui.open(tab, editor)`, `ui.set(editor, field, value)`,
 `ui.get(editor, field)`. This is the user path: `Editor::set_field_value`
@@ -78,6 +78,22 @@ controller distance/height), `camera.trace_screen_ray(x, y)`,
 reload deliberately preserves engine world/project state; it is not a project
 reset.
 
+**Dialogs** — `dialog.open/get/set/select/accept/cancel` controls typed
+project/file dialogs through their existing action and modal paths. A domain
+workflow can create a project, choose a VFS project, or save/export by name
+without synthesising a click, text entry, or Enter key:
+
+```python
+project = sb.dialog("new_project").open()
+project.set("name", "Example")
+project.set("size_x", 32)
+project.accept()
+
+load = sb.dialog("load_project").open()
+load.select("springboard/projects/Example.sdd")
+load.accept()
+```
+
 ## The Python client
 
 `tools/control/`, a `control` package alongside `e2e`/`smoke`/`lint`, depending
@@ -124,7 +140,7 @@ a step at a time while the rest of it still clicks.
 `native/src/sbc/control/` separates the two halves so neither drifts into the
 other:
 
-- `api/` — the surface, one file per method group (`editors`, `commands`,
+- `api/` — the surface, one file per method group (`editors`, `dialogs`, `commands`,
   `camera`, `capture`, `schema`). Each goes through the seam its user action
   goes through, and knows nothing about sockets.
 - `channel/` — the plumbing: `server` (socket and connections), `discovery`
@@ -136,8 +152,9 @@ other:
 ## When a call is done
 
 A reply is sent once its effect has landed, not once the request was accepted:
-`ui.open` answers when the editor is on screen, `capture` when the image is on
-disk, and `runtime.barrier` after two input-idle native updates. Requests on a
+`ui.open` answers when the editor is on screen, `dialog.open`/`dialog.accept`
+answer when the modal opens/closes, `capture` when the image is on disk, and
+`runtime.barrier` after two input-idle native updates. Requests on a
 connection apply and answer in order, and `capture` is queued at
 `draw_screen_post`, so an image already contains every call before it. No
 scenario needs a sleep to make a screenshot honest.
@@ -148,7 +165,7 @@ Nothing in this channel may fail by doing nothing.
 
 ## Scope
 
-**Built**: `describe`, `ui.open/set/get`, `command.execute`,
+**Built**: `describe`, `ui.open/set/get`, `dialog.open/get/set/select/accept/cancel`, `command.execute`,
 `camera.set/get/trace_screen_ray/zoom`,
 `capture`, `runtime.barrier`, `runtime.reload_native_modules`. `SBC_CONTROL_FILE` names the discovery file and turns the channel on;
 the E2E harness sets it per run and exposes the connection as

justfile

@@ -148,7 +148,7 @@ run config="config/ui-rust.json": build-native
 # invocations store all scenario artifacts and the aggregate suite report under
 # one timestamped suite folder.
 [group('test')]
-test-e2e target *args: build-native
+test-e2e target="all" *args: build-native
     PYTHONPATH="{{tool_pythonpath}}" uv run --locked sbc-e2e run "{{target}}" {{args}}
 
 # `just test-e2e all` shares compatible launch environments by default and

native/src/sbc/actions/action.rs

@@ -23,6 +23,30 @@ pub enum Action {
 }
 
 impl Action {
+    /// The stable name used by the programmatic dialog API for actions that
+    /// collect file/project input.
+    pub fn dialog_name(self) -> Option<&'static str> {
+        match self {
+            Action::NewProject => Some("new_project"),
+            Action::Load => Some("load_project"),
+            Action::Import => Some("import"),
+            Action::SaveAs => Some("save_project_as"),
+            Action::Export => Some("export"),
+            _ => None,
+        }
+    }
+
+    pub fn from_dialog_name(name: &str) -> Option<Self> {
+        match name {
+            "new_project" => Some(Action::NewProject),
+            "load_project" => Some(Action::Load),
+            "import" => Some(Action::Import),
+            "save_project_as" => Some(Action::SaveAs),
+            "export" => Some(Action::Export),
+            _ => None,
+        }
+    }
+
     /// All actions, in a stable order.
     pub const ALL: [Action; 15] = [
         Action::NewProject,

native/src/sbc/actions/dialog.rs

@@ -25,6 +25,8 @@ pub enum ActionResult {
 /// Configuration for the file browser dialog.
 #[derive(Debug, Clone)]
 pub struct FileDialogConfig {
+    /// Stable programmatic name for this action's file dialog.
+    pub control_name: &'static str,
     pub title: String,
     pub root_dir: String,
     /// File extensions to show (e.g. `[".png", ".jpg"]`). Empty = show all.
@@ -40,6 +42,7 @@ pub struct FileDialogConfig {
 impl Default for FileDialogConfig {
     fn default() -> Self {
         FileDialogConfig {
+            control_name: "file",
             title: "File".to_string(),
             root_dir: PROJECTS_DIR.to_string(),
             extensions: Vec::new(),

native/src/sbc/actions/run.rs

@@ -65,6 +65,7 @@ pub fn execute(
 
         Action::Load => {
             let config = FileDialogConfig {
+                control_name: "load_project",
                 title: "Open project".to_string(),
                 root_dir: PROJECTS_DIR.to_string(),
                 dirs_as_items: true,
@@ -86,6 +87,7 @@ pub fn execute(
 
         Action::Import => {
             let config = FileDialogConfig {
+                control_name: "import",
                 title: "Import".to_string(),
                 root_dir: PROJECTS_DIR.to_string(),
                 extensions: [".png", ".jpg", ".bmp", ".tga", ".tif"]
@@ -126,6 +128,7 @@ pub fn execute(
                 return ActionResult::None;
             }
             let config = FileDialogConfig {
+                control_name: "export",
                 title: "Export".to_string(),
                 root_dir: EXPORTS_DIR.to_string(),
                 file_types: [
@@ -221,6 +224,7 @@ pub fn execute_paste(
 
 fn open_save_as() -> ActionResult {
     let config = FileDialogConfig {
+        control_name: "save_project_as",
         title: "Save project as...".to_string(),
         root_dir: PROJECTS_DIR.to_string(),
         show_name_input: true,

native/src/sbc/control/api/camera.rs

@@ -2,6 +2,7 @@
 
 use serde::Deserialize;
 use serde_json::json;
+use spring_native::prelude::NativeInterfaceRef;
 
 use crate::sbc::sbc::SBC;
 
@@ -28,6 +29,11 @@ pub(crate) struct Set {
 #[derive(Deserialize)]
 pub(crate) struct Zoom {
     pub factor: f32,
+    /// Keep this ground point under a screen-space cursor while zooming.
+    /// This is the semantic equivalent of the engine's mouse-wheel zoom; the
+    /// client should pass the screen point, not reproduce camera geometry.
+    #[serde(default)]
+    pub screen: Option<[f32; 2]>,
 }
 
 #[derive(Deserialize)]
@@ -168,7 +174,18 @@ pub(crate) fn zoom(sbc: &mut SBC, params: Zoom) -> Handled {
             "camera.zoom factor must be positive and finite",
         ));
     }
-    let camera = sbc.interface().camera();
+    let interface = sbc.interface();
+    let target = if let Some(screen) = params.screen {
+        if !screen[0].is_finite() || !screen[1].is_finite() {
+            return Err(ControlError::invalid(
+                "camera.zoom screen coordinates must be finite",
+            ));
+        }
+        trace_ground(interface, screen)?
+    } else {
+        None
+    };
+    let camera = interface.camera();
     let mut state = camera
         .get_camera_state(false)
         .map_err(|err| ControlError::failed(format!("get_camera_state: {err:?}")))?;
@@ -187,6 +204,26 @@ pub(crate) fn zoom(sbc: &mut SBC, params: Zoom) -> Handled {
         .map_err(|err| ControlError::failed(format!("set_camera_state: {err:?}")))?
         .then_some(())
         .ok_or_else(|| ControlError::failed("engine rejected camera zoom"))?;
+
+    if let (Some(screen), Some(target)) = (params.screen, target) {
+        // The generic camera.zoom operation scales around the controller
+        // position. The mouse-wheel operation instead keeps the traced ground
+        // point beneath the cursor. Re-trace after scaling and apply the
+        // horizontal ground delta in the controller, where the engine owns the
+        // camera projection and terrain height.
+        if let Some(current) = trace_ground(interface, screen)? {
+            let mut state = camera
+                .get_camera_state(false)
+                .map_err(|err| ControlError::failed(format!("get_camera_state: {err:?}")))?;
+            state.pos.x += target[0] - current[0];
+            state.pos.z += target[2] - current[2];
+            camera
+                .set_camera_state(state, 0.0, 1.0, 1.0)
+                .map_err(|err| ControlError::failed(format!("set_camera_state: {err:?}")))?
+                .then_some(())
+                .ok_or_else(|| ControlError::failed("engine rejected camera focus"))?;
+        }
+    }
     get(sbc)
 }
 
@@ -208,3 +245,14 @@ pub(crate) fn trace(sbc: &mut SBC, params: Trace) -> Handled {
         "position": [position.x, position.y, position.z],
     })))
 }
+
+fn trace_ground(
+    interface: &NativeInterfaceRef,
+    screen: [f32; 2],
+) -> Result<Option<[f32; 3]>, ControlError> {
+    let (hit_type, _, position) = interface
+        .camera()
+        .trace_screen_ray(screen[0], screen[1], true, false, false, true, 0.0)
+        .map_err(|err| ControlError::failed(format!("trace_screen_ray: {err:?}")))?;
+    Ok((hit_type == 3).then_some([position.x, position.y, position.z]))
+}

native/src/sbc/control/api/dialogs.rs

@@ -0,0 +1,103 @@
+//! Typed control of dialogs that collect domain input.
+//!
+//! This is deliberately separate from editor UI control. Domain workflows can
+//! open and submit a project/file dialog without synthesising mouse or key
+//! events; tests that specifically exercise the dialog's visible controls can
+//! continue to use input.
+
+use serde::Deserialize;
+use serde_json::{json, Value};
+
+use crate::sbc::actions::Action;
+use crate::sbc::panels::PanelManager;
+use crate::sbc::sbc::SBC;
+
+use super::super::channel::Effect;
+use super::super::{ControlError, Handled, Reply};
+use super::schema;
+
+#[derive(Deserialize)]
+pub(crate) struct Open {
+    pub dialog: String,
+}
+
+#[derive(Deserialize)]
+pub(crate) struct Field {
+    pub dialog: String,
+    pub field: String,
+}
+
+#[derive(Deserialize)]
+pub(crate) struct Dialog {
+    pub dialog: String,
+}
+
+#[derive(Deserialize)]
+pub(crate) struct Set {
+    pub dialog: String,
+    pub field: String,
+    pub value: Value,
+}
+
+#[derive(Deserialize)]
+pub(crate) struct Select {
+    pub dialog: String,
+    pub path: String,
+}
+
+pub(crate) fn open(sbc: &mut SBC, params: Open) -> Handled {
+    let canonical = sbc.models_mut().with::<PanelManager, _>(|panels, models| {
+        panels.control_open_dialog(&params.dialog, models)
+    })?;
+    Ok(Reply::When(Effect::DialogOpen(canonical)))
+}
+
+pub(crate) fn get(sbc: &mut SBC, params: Field) -> Handled {
+    let (spec, _) = sbc
+        .model::<PanelManager>()
+        .control_dialog_field_value(&params.dialog, &params.field)?;
+    Ok(Reply::now(json!({
+        "value": schema::value_json(&spec.value),
+        "kind": schema::kind_of(&spec.value),
+    })))
+}
+
+pub(crate) fn set(sbc: &mut SBC, params: Set) -> Handled {
+    let (spec, _) = sbc
+        .model::<PanelManager>()
+        .control_dialog_field_value(&params.dialog, &params.field)?;
+    let value = schema::parse_value(&spec.value, &params.value)
+        .map_err(|message| ControlError::invalid(format!("{}: {message}", params.field)))?;
+    let applied = sbc.model::<PanelManager>().control_set_dialog_field(
+        &params.dialog,
+        &params.field,
+        value,
+    )?;
+    Ok(Reply::now(json!({ "value": schema::value_json(&applied) })))
+}
+
+pub(crate) fn select(sbc: &mut SBC, params: Select) -> Handled {
+    sbc.model::<PanelManager>()
+        .control_select_dialog(&params.dialog, &params.path)?;
+    Ok(Reply::now(json!({ "path": params.path })))
+}
+
+pub(crate) fn accept(sbc: &mut SBC, params: Dialog) -> Handled {
+    let canonical = canonical_name(&params.dialog)?;
+    sbc.model::<PanelManager>()
+        .control_accept_dialog(&params.dialog)?;
+    Ok(Reply::When(Effect::DialogClosed(canonical)))
+}
+
+pub(crate) fn cancel(sbc: &mut SBC, params: Dialog) -> Handled {
+    let canonical = canonical_name(&params.dialog)?;
+    sbc.model::<PanelManager>()
+        .control_cancel_dialog(&params.dialog)?;
+    Ok(Reply::When(Effect::DialogClosed(canonical)))
+}
+
+fn canonical_name(name: &str) -> Result<&'static str, ControlError> {
+    Action::from_dialog_name(name)
+        .and_then(Action::dialog_name)
+        .ok_or_else(|| ControlError::unknown(format!("no such dialog: {name}")))
+}

native/src/sbc/control/api/mod.rs

@@ -3,6 +3,7 @@
 pub(crate) mod camera;
 pub(crate) mod capture;
 pub(crate) mod commands;
+pub(crate) mod dialogs;
 pub(crate) mod editors;
 pub(crate) mod runtime;
 pub(crate) mod schema;

native/src/sbc/control/api/schema.rs

@@ -31,6 +31,7 @@ pub(crate) fn describe() -> Value {
 
     json!({
         "tabs": tabs,
+        "dialogs": dialog_descriptors(),
         "commands": registered_class_names(),
     })
 }
@@ -105,3 +106,33 @@ fn field_json(spec: FieldSpec) -> Value {
         "options": spec.options,
     })
 }
+
+fn dialog_descriptors() -> Vec<Value> {
+    vec![
+        json!({
+            "name": "new_project",
+            "fields": [
+                { "name": "name", "kind": "text", "value": "" },
+                { "name": "map", "kind": "text", "value": "SB_Blank_Map" },
+                { "name": "size_x", "kind": "number", "value": 10.0 },
+                { "name": "size_y", "kind": "number", "value": 10.0 },
+            ],
+        }),
+        json!({ "name": "load_project", "fields": [] }),
+        json!({
+            "name": "save_project_as",
+            "fields": [{ "name": "name", "kind": "text", "value": "" }],
+        }),
+        json!({
+            "name": "import",
+            "fields": [{ "name": "file_type", "kind": "text", "value": "" }],
+        }),
+        json!({
+            "name": "export",
+            "fields": [
+                { "name": "name", "kind": "text", "value": "" },
+                { "name": "file_type", "kind": "text", "value": "" },
+            ],
+        }),
+    ]
+}