Immersive OSPRay Studio - a plugin to support gesture-based interaction

.gitignore

@@ -8,4 +8,5 @@
 !/plugins
 /plugins/*
 !/plugins/example_plugin
+!/plugins/gesture_plugin
 doc/*.html

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "plugins/gesture_plugin/async-sockets-cpp"]
+	path = plugins/gesture_plugin/async-sockets-cpp
+	url = https://github.com/JungWhoNam/async-sockets-cpp.git

plugins/gesture_plugin/CMakeLists.txt

@@ -0,0 +1,39 @@
+option(BUILD_PLUGIN_GESTURE "Gesture plugin" OFF)
+
+if (BUILD_PLUGIN_GESTURE)  
+  set(pluginName "ospray_studio_plugin_gesture")
+
+  add_library(${pluginName} SHARED
+    plugin_gesture.cpp
+    PanelGesture.cpp
+    tracker/TrackingManager.cpp
+  )
+
+  target_link_libraries(${pluginName} ospray_sg)
+
+  # Only link against imgui if needed (ie, pure file importers don't)
+  target_link_libraries(${pluginName} ospray_ui)
+
+  # There can be other plugins using async_sockets library
+  if (NOT TARGET async_sockets)
+    message(STATUS "Adding async_sockets library...")
+    add_library(async_sockets INTERFACE)
+    target_include_directories(async_sockets 
+    INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/async-sockets-cpp/async-sockets/include>)
+  endif()
+
+  target_link_libraries(${pluginName} async_sockets)
+
+  target_include_directories(${pluginName}
+    PRIVATE ${CMAKE_SOURCE_DIR}
+  )
+
+  install(TARGETS ${pluginName}
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      COMPONENT lib
+    # on Windows put the dlls into bin
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+      COMPONENT lib
+  )
+
+endif()

plugins/gesture_plugin/PanelGesture.cpp

@@ -0,0 +1,107 @@
+#include <ctime>
+
+#include "PanelGesture.h"
+
+#include "app/widgets/GenerateImGuiWidgets.h"
+#include "app/MainWindow.h"
+
+#include "imgui.h"
+
+
+namespace ospray {
+namespace gesture_plugin {
+
+PanelGesture::PanelGesture(std::shared_ptr<StudioContext> _context, std::string _panelName, std::string _configFilePath)
+    : Panel(_panelName.c_str(), _context)
+    , panelName(_panelName)
+    , configFilePath(_configFilePath)
+{
+  trackingManager.reset(new TrackingManager(configFilePath));
+}
+
+void PanelGesture::processRequests() {
+  if (!trackingManager->isRunning())
+    return;
+
+  TrackingState state = trackingManager->pollState();
+  if (state.mode == INTERACTION_FLYING) {
+    MainWindow* pMW = reinterpret_cast<MainWindow*>(context->getMainWindow());
+
+    vec3f pos = pMW->arcballCamera->center();
+    vec3f center = state.leaningDir + pos;
+    pMW->arcballCamera->setCenter(center);
+    context->updateCamera();
+  }
+}
+
+void PanelGesture::buildUI(void *ImGuiCtx)
+{
+  processRequests();
+
+  // Allows plugin to still do other work if the UI isn't shown.
+  if (!isShown())
+    return;
+
+  // Need to set ImGuiContext in *this* address space
+  ImGui::SetCurrentContext((ImGuiContext *)ImGuiCtx);
+  ImGui::OpenPopup(panelName.c_str());
+
+  if (!ImGui::BeginPopupModal(panelName.c_str(), nullptr, ImGuiWindowFlags_None)) return;
+
+  if (trackingManager->isRunning()) {
+    ImGui::Text("%s", "Currently connected to the server...");
+
+    if (ImGui::Button("Disconnect")) {
+      trackingManager->close();
+    }
+  }
+  else {
+    ImGui::Text("%s", "Currently NOT connected to the server...");
+    std::string str = "- Connect to " + trackingManager->ipAddress + ":" + std::to_string(trackingManager->portNumber);
+    ImGui::Text("%s", str.c_str());
+
+    if (ImGui::Button("Connect")) {
+      trackingManager->start();
+    }
+  }
+  ImGui::Separator();
+
+  // Close button
+  if (ImGui::Button("Close")) {
+    setShown(false);
+    ImGui::CloseCurrentPopup();
+  }
+  ImGui::Separator();
+
+  if (ImGui::CollapsingHeader("Configuration", ImGuiTreeNodeFlags_DefaultOpen)) {
+    ImGui::Text("%s", "Offset(s)");
+    ImGui::DragFloat3("Scale", trackingManager->scaleOffset, 0.001, -100, 100, "%.3f");
+    ImGui::DragFloat3("Translate", trackingManager->translationOffset, 0, -100, 100, "%.1f");
+
+    ImGui::Text("%s", "Gestures(s)");
+    ImGui::DragInt("Confidence Level Threshold", &trackingManager->confidenceLevelThreshold, K4ABT_JOINT_CONFIDENCE_LOW, K4ABT_JOINT_CONFIDENCE_NONE, K4ABT_JOINT_CONFIDENCE_LEVELS_COUNT);
+    ImGui::DragFloat("Leaning Angle Threshold", &trackingManager->leaningAngleThreshold, 1, 0, 180);
+    ImGui::DragFloat3("Leaning Dir Scale", trackingManager->leaningDirScaleFactor, 0, -100, 100, "%.1f");
+
+    ImGui::Separator();
+    if (ImGui::Button("Save")) {
+      trackingManager->saveConfig(this->configFilePath);
+    }
+  }
+  ImGui::Separator();
+
+  // Display statuses in a scrolling region
+  if (ImGui::CollapsingHeader("Status", ImGuiTreeNodeFlags_DefaultOpen)) {
+    ImGui::BeginChild("Scrolling", ImVec2(0, 0), false, ImGuiWindowFlags_AlwaysAutoResize);
+    for (std::string status : trackingManager->statuses) {
+      ImGui::Text("%s", status.c_str());
+    }
+    ImGui::EndChild();
+  }
+  ImGui::Separator();
+
+  ImGui::EndPopup();
+}
+
+}  // namespace gesture_plugin
+}  // namespace ospray

plugins/gesture_plugin/PanelGesture.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "app/widgets/Panel.h"
+#include "app/ospStudio.h"
+
+#include "tracker/TrackingManager.h"
+
+namespace ospray {
+namespace gesture_plugin {
+
+struct PanelGesture : public Panel
+{
+  PanelGesture(std::shared_ptr<StudioContext> _context, std::string _panelName, std::string _configFilePath);
+
+  void buildUI(void *ImGuiCtx) override;
+
+  void processRequests();
+
+private:
+  std::string panelName;
+  std::string configFilePath;
+  std::unique_ptr<TrackingManager> trackingManager;
+};
+
+}  // namespace gesture_plugin
+}  // namespace ospray

plugins/gesture_plugin/README.md

@@ -0,0 +1,100 @@
+# Gesture interaction plugin for OSPRay Studio 
+> This project is part of a larger project called [Immersive OSPray Studio](https://github.com/jungwhonam/ImmersiveOSPRay).
+
+## Overview
+<div id="image-table">
+  <table>
+    <td style="padding:6px">
+      <img src="gesture_plugin_demo0.png" width="500"/>
+    </td>
+    <td style="padding:6px">
+      <img src="gesture_plugin_demo1.png" width="500"/>
+    </td>
+  </table>
+</div>
+
+We created a plugin for [OSPRay Studio v1.0.0](https://github.com/RenderKit/ospray-studio/releases/tag/v1.0.0) allowing users to navigate a 3D virtual environment using gestures. Lifting both hands above a belly button triggers a flying mode. Once in the mode, the camera moves into a body-leaning direction.
+
+<div id="image-table">
+  <table>
+    <td style="padding:6px">
+      <img src="gesture_plugin_gui.png" width="350"/>
+    </td>
+    <td style="padding:6px">
+      <img src="gesture_plugin_system.png" width="650"/>
+    </td>
+  </table>
+</div>
+
+This plugin recieves the body tracking data from [Gesture Tracking Server](https://github.com/jungwhonam/GestureTrackingServer). After processing user inputs, such as key-pressed events, OSPRay Studio calls a method in the plugin to update GUIs. At this stage, the plugin retrieves the latest tracking result and use it to update relevant 3D objects, such as modifying camera positions.
+
+## Prerequisites
+Before running `ospStudio` with the plugin, you need to start [Gesture Tracking Server](https://github.com/jungwhonam/GestureTrackingServer).
+
+
+## Setup
+```shell
+# clone this branch
+git clone -b jungwho.nam-feature-plugin-gesture https://github.com/JungWhoNam/ospray_studio.git
+cd ospray_studio
+
+mkdir build
+cd build
+mkdir release
+``` 
+
+
+## CMake configuration and build
+OSPRay Studio needs to be built with `-DBUILD_PLUGINS=ON` and `-DBUILD_PLUGIN_GESTURE=ON` in CMake.
+
+```shell
+cmake -S .. \
+-B release \
+-DCMAKE_BUILD_TYPE=Release \
+-DBUILD_PLUGINS=ON \
+-DBUILD_PLUGIN_GESTURE=ON
+
+cmake --build release
+
+cmake --install release
+```
+
+
+## Run `ospStudio` with the gesture plugin
+
+1. First, start the server by following the steps written in [the server repo](https://github.com/jungwhonam/GestureTrackingServer).
+2. Start `ospStudio` with the plugin.
+```shell
+./release/ospStudio \
+--plugin gesture \
+--plugin:gesture:config gesture_settings.json
+```
+1. Go to `Plugins` > `Gesture Panel` in the menu.
+2. Click "Connect" button.
+
+If connected, you will see "Connected to the server successfully" displayed on the Status sub-panel.
+
+
+## Plugin configuration JSON file
+When running `ospStudio`, you must specify the location of this JSON file using `--plugin:gesture:config` flag. This file contains information about the gesture tracking server and instructions on handling the tracking data.
+
+```json
+{
+    "ipAddress": "127.0.0.1",
+    "portNumber": 8888,
+
+    "scaleOffset": [0.001, -0.001, -0.001],
+    "translationOffset": [0.0, -0.1, 1.19],
+    "confidenceLevelThreshold": 1,
+    "leaningAngleThreshold": 1.0,
+    "leaningDirScaleFactor": [1.0, 1.0, 1.0]
+}
+```
+* `ipAddress` and `portNumber` are used for connecting the gesture tracking server.
+* `scaleOffset` is multiplied to position values of joints to adjust for differences between Kinect and OSPRay Studio coordinate systems.
+* `translationOffset` offsets the sensor's center to calibrate it with displays.
+* `confidenceLevelThreshold` specifies the joint confidence level considered for processing, with different levels detailed in [Microsoft's documentation](https://microsoft.github.io/Azure-Kinect-Body-Tracking/release/1.1.x/namespace_microsoft_1_1_azure_1_1_kinect_1_1_body_tracking_adfff503ebc1491373c89e96887cad226.html#adfff503ebc1491373c89e96887cad226) for different levels.
+* `leaningAngleThreshold` is a threshhold for activating the flying mode when a user's body leans beyond this angle.
+* `leaningDirScaleFactor` sets the speed of camera movement based on body leaning direction.
+
+> Here is [an example JSON file](./gesture_settings.json).

plugins/gesture_plugin/gesture_settings.json

@@ -0,0 +1,21 @@
+{
+    "ipAddress": "127.0.0.1",
+    "portNumber": 8888,
+    "scaleOffset": [
+        -0.0010000000474974513,
+        -0.0010000000474974513,
+        0.0010000000474974513
+    ],
+    "translationOffset": [
+        0.0,
+        0.0,
+        1.190000057220459
+    ],
+    "confidenceLevelThreshold": 1,
+    "leaningAngleThreshold": 1.0,
+    "leaningDirScaleFactor": [
+        1.0,
+        1.0,
+        1.0
+    ]
+}

plugins/gesture_plugin/plugin_gesture.cpp

@@ -0,0 +1,52 @@
+#include <memory>
+
+#include "PanelGesture.h"
+
+#include "app/ospStudio.h"
+#include "app/Plugin.h"
+
+namespace ospray {
+namespace gesture_plugin {
+
+struct PluginGesture : public Plugin
+{
+  PluginGesture() : Plugin("Gesture") {}
+
+  void mainMethod(std::shared_ptr<StudioContext> ctx) override
+  {
+    if (ctx->mode == StudioMode::GUI) {
+      auto &studioCommon = ctx->studioCommon;
+      int ac = studioCommon.plugin_argc;
+      const char **av = studioCommon.plugin_argv;
+
+      std::string optPanelName = "Gesture Panel";
+      std::string configFilePath = "config/tracking_settings.json";
+
+      for (int i=0; i<ac; ++i) {
+        std::string arg = av[i];
+        if (arg == "--plugin:gesture:name") {
+          optPanelName = av[i + 1];
+          ++i;
+        }
+        else if (arg == "--plugin:gesture:config") {
+          configFilePath = av[i + 1];
+          ++i;
+        }
+      }
+
+      panels.emplace_back(new PanelGesture(ctx, optPanelName, configFilePath));
+    }
+    else
+      std::cout << "Plugin functionality unavailable in Batch mode .."
+                << std::endl;
+  }
+};
+
+extern "C" PLUGIN_INTERFACE Plugin *init_plugin_gesture()
+{
+  std::cout << "loaded plugin 'gesture'!" << std::endl;
+  return new PluginGesture();
+}
+
+} // namespace gesture_plugin
+} // namespace ospray