Add a LeafSystem for dragging objects from meshcat

bindings/generated_docstrings/geometry.h

@@ -3825,6 +3825,20 @@ std∷nullopt is returned if:
 connected.
 - The meshcat session transmitting has an orthographic camera.)""";
         } GetTrackedCameraPose;
+        // Symbol: drake::geometry::Meshcat::GetVirtualSpringKinematics
+        struct /* GetVirtualSpringKinematics */ {
+          // Source: drake/geometry/meshcat.h
+          const char* doc =
+R"""(Returns the current mouse-drag state if a user is presently dragging
+an object in a connected Meshcat browser, or std∷nullopt otherwise.
+
+A downstream system (see multibody∷meshcat∷MeshcatMouseSpring) can
+read this state and convert it into a force applied to a simulated
+body, letting users drag objects with the cursor.
+
+If multiple browsers report drags concurrently, the returned value
+reflects the most recently received message.)""";
+        } GetVirtualSpringKinematics;
         // Symbol: drake::geometry::Meshcat::HasPath
         struct /* HasPath */ {
           // Source: drake/geometry/meshcat.h
@@ -4668,6 +4682,49 @@ You can also use your browser to download this file, by typing
 R"""(Sets a flag to pause/stop recording. When stopped, publish events will
 not add frames to the animation.)""";
         } StopRecording;
+        // Symbol: drake::geometry::Meshcat::VirtualSpringKinematics
+        struct /* VirtualSpringKinematics */ {
+          // Source: drake/geometry/meshcat.h
+          const char* doc =
+R"""(The geometry of a virtual, interactive spring. This spring connects
+the point F (a point fixed on the body B -- named by ``path``) with an
+immovable target point T. Both points are measured and expressed in
+the world frame. This definition is for kinematics only: the spring's
+stiffness, damping, and resultant forces are not defined here.)""";
+          // Symbol: drake::geometry::Meshcat::VirtualSpringKinematics::body_point_in_world
+          struct /* body_point_in_world */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The current position of F affixed to body B. Note: in an interactive
+scenario with active dynamics, this position will change as the body B
+moves.
+
+Note: This is expressed in Drake's z-up world frame (not Meshcat's
+y-up frame).)""";
+          } body_point_in_world;
+          // Symbol: drake::geometry::Meshcat::VirtualSpringKinematics::path
+          struct /* path */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The "/"-delimited Meshcat path of the object being dragged (e.g.,
+"/drake/visualizer/my_model/my_body/my_geometry").
+
+This could be any valid Meshcat path, including for either a body or a
+geometry. Whether it has any effect depends on the caller's treatment.)""";
+          } path;
+          // Symbol: drake::geometry::Meshcat::VirtualSpringKinematics::target_point_in_world
+          struct /* target_point_in_world */ {
+            // Source: drake/geometry/meshcat.h
+            const char* doc =
+R"""(The current position of the fixed target point T. The target point is
+solely determined by the interactive Meshcat application. From Drake's
+perspective it is a point with a prescribed position in the world
+frame.
+
+Note: This is expressed in Drake's z-up world frame (not Meshcat's
+y-up frame).)""";
+          } target_point_in_world;
+        } VirtualSpringKinematics;
         // Symbol: drake::geometry::Meshcat::get_mutable_recording
         struct /* get_mutable_recording */ {
           // Source: drake/geometry/meshcat.h

bindings/generated_docstrings/multibody_meshcat.h

@@ -16,6 +16,7 @@
 // #include "drake/multibody/meshcat/contact_visualizer_params.h"
 // #include "drake/multibody/meshcat/hydroelastic_contact_visualizer.h"
 // #include "drake/multibody/meshcat/joint_sliders.h"
+// #include "drake/multibody/meshcat/meshcat_mouse_spring.h"
 // #include "drake/multibody/meshcat/point_contact_visualizer.h"
 
 // Symbol: pydrake_doc_multibody_meshcat
@@ -378,6 +379,121 @@ Parameter ``q``:
     MultibodyPlant∷num_positions().)""";
           } SetPositions;
         } JointSliders;
+        // Symbol: drake::multibody::meshcat::MeshcatMouseSpring
+        struct /* MeshcatMouseSpring */ {
+          // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+          const char* doc =
+R"""(MeshcatMouseSpring lets a user drag the bodies of a MultibodyPlant
+with the mouse in a Meshcat browser, applying a virtual spring force
+that pulls the grabbed point toward the cursor.
+
+This system reads the drag state from Meshcat (see
+geometry∷Meshcat∷GetVirtualSpringKinematics()) and outputs a
+corresponding ExternallyAppliedSpatialForce on the dragged body.
+Connecting that output to
+MultibodyPlant∷get_applied_spatial_force_input_port() applies the
+force; AddToBuilder() performs that connection along with the input
+connections.
+
+.. pydrake_system::
+
+    name: MeshcatMouseSpring
+    input_ports:
+    - body_poses
+    - body_spatial_velocities
+    output_ports:
+    - applied_spatial_force
+
+The ``body_poses`` and ``body_spatial_velocities`` inputs come from
+the same-named MultibodyPlant output ports.
+
+With ``m`` the dragged body's mass, the applied force (in the world
+frame) is ``m * stiffness * (target - anchor) - m * sqrt(stiffness) *
+v_anchor``, where ``anchor`` is the grabbed point on the body,
+``target`` is the cursor position, and ``v_anchor`` is the world
+velocity of the grabbed point. Scaling by ``m`` makes the
+translational response frequency ``sqrt(stiffness)`` and damping ratio
+independent of the body's mass.
+
+When no drag is in progress the output is empty. Any body with
+geometry published to Meshcat by a geometry∷MeshcatVisualizer can be
+dragged; the world body cannot.)""";
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::AddToBuilder
+          struct /* AddToBuilder */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Adds a MeshcatMouseSpring to ``builder`` and connects it to `plant`'s
+body-pose and body-spatial-velocity output ports and its
+applied-spatial-force input port. Returns a reference to the
+newly-added system.
+
+Precondition:
+    plant is part of builder and is finalized.
+
+Precondition:
+    plant is registered as a geometry source with scene_graph.
+
+Precondition:
+    `plant`'s applied-spatial-force input port is not already
+    connected.)""";
+          } AddToBuilder;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::MeshcatMouseSpring
+          struct /* ctor */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Constructs a MeshcatMouseSpring for the given ``plant``.
+
+Parameter ``meshcat``:
+    The Meshcat instance the user will interact with. The pointer is
+    aliased and must outlive this system.
+
+Parameter ``plant``:
+    The MultibodyPlant whose bodies can be dragged. The pointer is
+    aliased and must outlive this system; the plant must already be
+    finalized and registered as a geometry source with
+    ``scene_graph``.
+
+Parameter ``scene_graph``:
+    The SceneGraph that ``plant`` is registered with. It is used at
+    construction to read the frame names identifying draggable bodies
+    in the Meshcat scene tree; the reference is used only during
+    construction and is not retained.
+
+Parameter ``stiffness``:
+    The mass-normalized spring stiffness, in 1/s²; see the class
+    overview for the force it produces.
+
+Precondition:
+    plant->is_finalized() is true.
+
+Precondition:
+    plant is registered as a geometry source with scene_graph.
+
+Precondition:
+    finite stiffness >= 0.)""";
+          } ctor;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_applied_spatial_force_output_port
+          struct /* get_applied_spatial_force_output_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the output port for the applied spatial forces (a
+``std∷vector<ExternallyAppliedSpatialForce<double>>``).)""";
+          } get_applied_spatial_force_output_port;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_body_poses_input_port
+          struct /* get_body_poses_input_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the input port for the bodies' poses in the world frame (a
+``std∷vector<math∷RigidTransform<double>>``).)""";
+          } get_body_poses_input_port;
+          // Symbol: drake::multibody::meshcat::MeshcatMouseSpring::get_body_spatial_velocities_input_port
+          struct /* get_body_spatial_velocities_input_port */ {
+            // Source: drake/multibody/meshcat/meshcat_mouse_spring.h
+            const char* doc =
+R"""(Returns the input port for the bodies' spatial velocities in the world
+frame (a ``std∷vector<SpatialVelocity<double>>``).)""";
+          } get_body_spatial_velocities_input_port;
+        } MeshcatMouseSpring;
       } meshcat;
     } multibody;
     // Symbol: drake::systems

geometry/meshcat.cc

@@ -1901,6 +1901,14 @@ class Meshcat::Impl {
     return gamepad_;
   }
 
+  std::optional<Meshcat::VirtualSpringKinematics> GetVirtualSpringKinematics()
+      const {
+    DRAKE_DEMAND(IsThread(main_thread_id_));
+
+    std::lock_guard<std::mutex> lock(controls_mutex_);
+    return mouse_drag_;
+  }
+
   // This function is for use by the websocket thread. The Meshcat::StaticHtml()
   // and Meshcat::StaticZip() outer functions call into here using appropriate
   // deferred handling.
@@ -2447,6 +2455,21 @@ class Meshcat::Impl {
       gamepad_.axes = std::move(data.gamepad->axes);
       return;
     }
+    if (data.type == "mouse_drag") {
+      // To protect from ill-formed messages, only a fully-specified message
+      // reports as "dragging".
+      if (data.name.size() > 0 && data.drag_anchor.size() == 3 &&
+          data.drag_target.size() == 3) {
+        mouse_drag_ = Meshcat::VirtualSpringKinematics{
+            .path = std::move(data.name),
+            .body_point_in_world = Eigen::Vector3d(data.drag_anchor.data()),
+            .target_point_in_world = Eigen::Vector3d(data.drag_target.data())};
+      } else {
+        // An empty payload signals the end of a drag (e.g., mouse release).
+        mouse_drag_ = std::nullopt;
+      }
+      return;
+    }
     if (data.type == "camera_pose" && data.camera_pose.size() == 16 &&
         data.is_perspective.has_value()) {
       if (camera_pose_source_ != nullptr && camera_pose_source_ != ws) {
@@ -2541,6 +2564,12 @@ class Meshcat::Impl {
   // The socket for the browser that is sending the camera pose.
   WebSocket* camera_pose_source_{};
   std::optional<math::RigidTransformd> camera_pose_;
+  // The most recently received object-drag state (nullopt when not dragging),
+  // guarded by controls_mutex_. Drag messages are currently accepted from any
+  // browser;
+  // TODO(vincekurtz) add per-socket source tracking and mid-drag disconnect
+  // cleanup alongside the browser-side drag implementation.
+  std::optional<Meshcat::VirtualSpringKinematics> mouse_drag_;
 
   // These variables should only be accessed in the main thread, where "main
   // thread" is the thread in which this class was constructed.
@@ -2968,6 +2997,11 @@ Meshcat::Gamepad Meshcat::GetGamepad() const {
   return impl().GetGamepad();
 }
 
+std::optional<Meshcat::VirtualSpringKinematics>
+Meshcat::GetVirtualSpringKinematics() const {
+  return impl().GetVirtualSpringKinematics();
+}
+
 std::string Meshcat::StaticHtml() const {
   return impl().StaticHtml();
 }

geometry/meshcat.h

@@ -903,6 +903,47 @@ class Meshcat {
   gamepad is working), see https://beej.us/blog/data/javascript-gamepad/. */
   Gamepad GetGamepad() const;
 
+  /** The geometry of a virtual, interactive spring. This spring connects the
+  point F (a point fixed on the body B -- named by `path`) with an immovable
+  target point T. Both points are measured and expressed in the world frame.
+  This definition is for kinematics only: the spring's stiffness, damping, and
+  resultant forces are not defined here. */
+  struct VirtualSpringKinematics {
+    /** The "/"-delimited Meshcat path of the object being dragged (e.g.,
+    "/drake/visualizer/my_model/my_body/my_geometry").
+
+    This could be any valid Meshcat path, including for either a body or a
+    geometry. Whether it has any effect depends on the caller's treatment. */
+    std::string path;
+
+    /** The current position of F affixed to body B. Note: in an
+    interactive scenario with active dynamics, this position will change as the
+    body B moves.
+
+    Note: This is expressed in Drake's z-up world frame (not Meshcat's y-up
+    frame). */
+    Eigen::Vector3d body_point_in_world;
+
+    /** The current position of the fixed target point T. The target point is
+    solely determined by the interactive Meshcat application. From Drake's
+    perspective it is a point with a prescribed position in the world frame.
+
+    Note: This is expressed in Drake's z-up world frame (not Meshcat's y-up
+    frame). */
+    Eigen::Vector3d target_point_in_world;
+  };
+
+  /** Returns the current mouse-drag state if a user is presently dragging an
+  object in a connected Meshcat browser, or std::nullopt otherwise.
+
+  A downstream system (see multibody::meshcat::MeshcatMouseSpring) can read this
+  state and convert it into a force applied to a simulated body, letting users
+  drag objects with the cursor.
+
+  If multiple browsers report drags concurrently, the returned value reflects
+  the most recently received message. */
+  std::optional<VirtualSpringKinematics> GetVirtualSpringKinematics() const;
+
   //@}
 
   /** Returns an HTML string that can be saved to a file for a snapshot of the

geometry/meshcat_types_internal.h

@@ -628,14 +628,28 @@ struct Gamepad {
 //       - If the flattened array doesn't have a valid rotation matrix in its
 //         unflattened upper-left 3x3 sub-matrix, it throws.
 //       - See Drake's meshcat.html for the source of the message.
+//
+//   Object dragging
+//     • Fields
+//       - type = "mouse_drag"
+//       - name = the Meshcat path of the object being dragged.
+//       - drag_anchor = the world-frame attachment point (3 values)
+//       - drag_target = the world-frame cursor target (3 values)
+//     • Semantics
+//       - If drag_anchor and drag_target each have 3 values, the drag state is
+//         updated; an empty payload ends the drag.
+//       - See Meshcat::GetVirtualSpringKinematics().
 struct UserInterfaceEvent {
   std::string type;
   std::string name;
   std::optional<double> value;
   std::optional<internal::Gamepad> gamepad;
   std::vector<double> camera_pose;
   std::optional<bool> is_perspective{};
-  MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective);
+  std::vector<double> drag_anchor;
+  std::vector<double> drag_target;
+  MSGPACK_DEFINE_MAP(type, name, value, gamepad, camera_pose, is_perspective,
+                     drag_anchor, drag_target);
 };
 
 }  // namespace internal

multibody/meshcat/BUILD.bazel

@@ -23,6 +23,7 @@ drake_cc_package_library(
         ":contact_visualizer_params",
         ":hydroelastic_contact_visualizer",
         ":joint_sliders",
+        ":meshcat_mouse_spring",
         ":point_contact_visualizer",
     ],
 )
@@ -111,6 +112,31 @@ drake_cc_googletest(
     ],
 )
 
+drake_cc_library(
+    name = "meshcat_mouse_spring",
+    srcs = ["meshcat_mouse_spring.cc"],
+    hdrs = ["meshcat_mouse_spring.h"],
+    deps = [
+        "//common:essential",
+        "//geometry:meshcat",
+        "//geometry:scene_graph",
+        "//multibody/plant",
+        "//systems/framework:leaf_system",
+    ],
+)
+
+drake_cc_googletest(
+    name = "meshcat_mouse_spring_test",
+    deps = [
+        ":meshcat_mouse_spring",
+        "//common/test_utilities:eigen_matrix_compare",
+        "//common/test_utilities:expect_throws_message",
+        "//geometry:scene_graph",
+        "//geometry/test_utilities:meshcat_environment",
+        "//multibody/plant",
+    ],
+)
+
 drake_cc_library(
     name = "point_contact_visualizer",
     srcs = ["point_contact_visualizer.cc"],