From ba0975286a10625a489ae75b1584ffdac3313346 Mon Sep 17 00:00:00 2001 From: PhilippeFerreiraDeSousa Date: Wed, 24 Jun 2026 16:43:03 -0700 Subject: [PATCH 1/4] Add G2 selectable list support --- .../java/com/mentra/bluetoothsdk/Bridge.kt | 4 +- .../com/mentra/bluetoothsdk/DeviceManager.kt | 43 +- .../java/com/mentra/bluetoothsdk/sgcs/G2.kt | 319 +++++- .../mentra/bluetoothsdk/sgcs/SGCManager.kt | 15 + .../bluetooth-sdk/ios/Source/Bridge.swift | 9 +- .../ios/Source/DeviceManager.swift | 231 +++-- .../bluetooth-sdk/ios/Source/sgcs/G2.swift | 969 +++++++++++------- .../ios/Source/sgcs/SGCManager.swift | 13 + .../src/services/LocalMiniappRuntime.ts | 39 +- mobile/modules/miniapp/src/index.ts | 2 + mobile/modules/miniapp/src/modules/display.ts | 89 ++ mobile/modules/miniapp/src/modules/events.ts | 8 + mobile/modules/miniapp/src/session.test.ts | 49 + 13 files changed, 1299 insertions(+), 491 deletions(-) diff --git a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt index 71e7394a0e..62416d47b4 100644 --- a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +++ b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt @@ -324,7 +324,8 @@ public class Bridge private constructor() { deviceModel: String, gestureName: String, timestamp: Long, - source: Int? = null + source: Int? = null, + extra: Map? = null ) { val body = HashMap() body["type"] = "touch_event" @@ -334,6 +335,7 @@ public class Bridge private constructor() { if (source != null) { body["source"] = source } + extra?.let { body.putAll(it) } sendTypedMessage("touch_event", body) } diff --git a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt index 6339773341..03164f5fe4 100644 --- a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +++ b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt @@ -651,11 +651,7 @@ class DeviceManager { } private fun statesEqual(s1: ViewState, s2: ViewState): Boolean { - val state1 = - "${s1.layoutType}${s1.text}${s1.topText}${s1.bottomText}${s1.title}${s1.data ?: ""}" - val state2 = - "${s2.layoutType}${s2.text}${s2.topText}${s2.bottomText}${s2.title}${s2.data ?: ""}" - return state1 == state2 + return s1 == s2 } private fun Map.getString(key: String, defaultValue: String): String { @@ -680,7 +676,13 @@ class DeviceManager { var bmpHeight: Int? = null, // Optional positioned_text border (used by G2; ignored by others). var borderWidth: Int? = null, - var borderRadius: Int? = null + var borderRadius: Int? = null, + // Optional selectable_list fields (used by G2; ignored by others). + var borderColor: Int? = null, + var paddingLength: Int? = null, + var listItems: List = emptyList(), + var itemWidth: Int? = null, + var showSelectionBorder: Boolean? = null ) // MARK: - End Unique @@ -955,6 +957,22 @@ class DeviceManager { ) } + "selectable_list" -> { + sgc?.sendSelectableList( + currentViewState.listItems, + currentViewState.bmpX ?: 0, + currentViewState.bmpY ?: 0, + currentViewState.bmpWidth ?: 576, + currentViewState.bmpHeight ?: 288, + currentViewState.borderWidth ?: 1, + currentViewState.borderColor ?: 13, + currentViewState.borderRadius ?: 6, + currentViewState.paddingLength ?: 5, + currentViewState.itemWidth ?: 0, + currentViewState.showSelectionBorder ?: true + ) + } + "clear_view" -> sgc?.clearDisplay() else -> Bridge.log("MAN: UNHANDLED LAYOUT_TYPE ${currentViewState.layoutType}") } @@ -1347,6 +1365,12 @@ class DeviceManager { // Optional positioned_text border (G2). val borderWidth = (layout["borderWidth"] as? Number)?.toInt() val borderRadius = (layout["borderRadius"] as? Number)?.toInt() + val borderColor = (layout["borderColor"] as? Number)?.toInt() + val paddingLength = (layout["paddingLength"] as? Number)?.toInt() + val rawListItems = (layout["items"] as? List<*>) ?: (layout["itemName"] as? List<*>) + val listItems = rawListItems?.mapNotNull { it?.toString() } ?: emptyList() + val itemWidth = (layout["itemWidth"] as? Number)?.toInt() + val showSelectionBorder = layout["showSelectionBorder"] as? Boolean var newViewState = ViewState( @@ -1362,7 +1386,12 @@ class DeviceManager { bmpWidth, bmpHeight, borderWidth, - borderRadius + borderRadius, + borderColor, + paddingLength, + listItems, + itemWidth, + showSelectionBorder ) val currentState = viewStates[stateIndex] diff --git a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt index 278d8b8563..251265f7f3 100644 --- a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +++ b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt @@ -331,6 +331,50 @@ private object EvenHubProto { return w.toByteArray() } + fun listItemContainerProperty( + itemCount: Int, + itemWidth: Int = 0, + isItemSelectBorderEn: Boolean = true, + itemName: List + ): ByteArray { + val w = ProtobufWriter() + w.writeInt32Field(1, itemCount) + w.writeInt32Field(2, itemWidth) + w.writeInt32Field(3, if (isItemSelectBorderEn) 1 else 0) + for (item in itemName) w.writeStringField(4, item) + return w.toByteArray() + } + + fun listContainerProperty( + x: Int, + y: Int, + width: Int, + height: Int, + borderWidth: Int = 1, + borderColor: Int = 13, + borderRadius: Int = 6, + paddingLength: Int = 5, + containerID: Int, + containerName: String? = null, + itemContainer: ByteArray, + isEventCapture: Boolean = false + ): ByteArray { + val w = ProtobufWriter() + w.writeInt32Field(1, x) + w.writeInt32Field(2, y) + w.writeInt32Field(3, width) + w.writeInt32Field(4, height) + w.writeInt32Field(5, borderWidth) + w.writeInt32Field(6, borderColor) + w.writeInt32Field(7, borderRadius) + w.writeInt32Field(8, paddingLength) + w.writeInt32Field(9, containerID) + containerName?.let { w.writeStringField(10, it) } + w.writeMessageField(11, itemContainer) + w.writeInt32Field(12, if (isEventCapture) 1 else 0) + return w.toByteArray() + } + fun imageRawDataUpdate( containerID: Int, containerName: String? = null, @@ -355,11 +399,13 @@ private object EvenHubProto { fun createStartupPageContainer( containerTotalNum: Int, + listContainers: List = emptyList(), textContainers: List = emptyList(), imageContainers: List = emptyList() ): ByteArray { val w = ProtobufWriter() w.writeInt32Field(1, containerTotalNum) + for (lc in listContainers) w.writeMessageField(2, lc) for (tc in textContainers) w.writeMessageField(3, tc) for (ic in imageContainers) w.writeMessageField(4, ic) return w.toByteArray() @@ -413,13 +459,14 @@ private object EvenHubProto { } fun createPageMessage( + listContainers: List = emptyList(), textContainers: List = emptyList(), imageContainers: List = emptyList(), magicRandom: Int = 0, appId: Int? = null ): ByteArray { - val total = textContainers.size + imageContainers.size - val createMsg = createStartupPageContainer(total, textContainers, imageContainers) + val total = listContainers.size + textContainers.size + imageContainers.size + val createMsg = createStartupPageContainer(total, listContainers, textContainers, imageContainers) return evenHubMessage( EvenHubCmd.CREATE_STARTUP_PAGE, 3, @@ -430,13 +477,14 @@ private object EvenHubProto { } fun rebuildPageMessage( + listContainers: List = emptyList(), textContainers: List = emptyList(), imageContainers: List = emptyList(), magicRandom: Int = 0, appId: Int? = null ): ByteArray { - val total = textContainers.size + imageContainers.size - val rebuildMsg = createStartupPageContainer(total, textContainers, imageContainers) + val total = listContainers.size + textContainers.size + imageContainers.size + val rebuildMsg = createStartupPageContainer(total, listContainers, textContainers, imageContainers) return evenHubMessage( EvenHubCmd.REBUILD_PAGE, 7, @@ -1458,16 +1506,62 @@ class G2 : SGCManager() { this.paddingLength == paddingLength } + /** A tracked native selectable list container on the current page. */ + private data class ListContainer( + val id: Int, + val x: Int, + val y: Int, + val width: Int, + val height: Int, + val borderWidth: Int, + val borderColor: Int, + val borderRadius: Int, + val paddingLength: Int, + val itemWidth: Int, + val showSelectionBorder: Boolean, + val items: List + ) { + val name: String + get() = "list-$id" + + fun matches( + x: Int, + y: Int, + width: Int, + height: Int, + borderWidth: Int, + borderColor: Int, + borderRadius: Int, + paddingLength: Int, + itemWidth: Int, + showSelectionBorder: Boolean, + items: List + ): Boolean = + this.x == x && + this.y == y && + this.width == width && + this.height == height && + this.borderWidth == borderWidth && + this.borderColor == borderColor && + this.borderRadius == borderRadius && + this.paddingLength == paddingLength && + this.itemWidth == itemWidth && + this.showSelectionBorder == showSelectionBorder && + this.items == items + } + /** * Live list of image containers on the page, ordered oldest→newest (for LRU eviction). The page * may hold at most 4 image containers (IDs from the pool below). */ private val imageContainers: MutableList = mutableListOf() private val textContainers: MutableList = mutableListOf() + private val listContainers: MutableList = mutableListOf() /** Fixed pool of container IDs the page protocol expects. */ private val imageContainerIDPool: List = listOf(10, 11, 12, 13) private val textContainerIDPool: List = listOf(1, 2, 3, 4, 5, 6) + private val listContainerIDPool: List = listOf(7, 8) /** Default container seeded into every fresh page: 200x100 centered at 188,44. */ private val defaultImgX = 188 @@ -1994,6 +2088,121 @@ class G2 : SGCManager() { } } + override fun sendSelectableList( + items: List, + x: Int, + y: Int, + width: Int, + height: Int, + borderWidth: Int, + borderColor: Int, + borderRadius: Int, + paddingLength: Int, + itemWidth: Int, + showSelectionBorder: Boolean + ) { + displayScope.launch { + sendSelectableList2( + items, + x, + y, + width, + height, + borderWidth, + borderColor, + borderRadius, + paddingLength, + itemWidth, + showSelectionBorder + ) + } + } + + private suspend fun sendSelectableList2( + items: List, + x: Int, + y: Int, + width: Int, + height: Int, + borderWidth: Int, + borderColor: Int, + borderRadius: Int, + paddingLength: Int, + itemWidth: Int, + showSelectionBorder: Boolean + ) { + val useNativeDashboard = + DeviceStore.get("bluetooth", "use_native_dashboard") as? Boolean ?: false + if (useNativeDashboard && dashboardShowing > 0) { + return + } + + val normalizedItems = + if (items.isEmpty()) { + listOf(" ") + } else { + items.take(20).map { item -> + val clipped = item.take(64) + if (clipped.isEmpty()) " " else clipped + } + } + val rx = x + val ry = y + val rw = width + val rh = height + val rBorderWidth = borderWidth + val rBorderColor = borderColor + val rBorderRadius = borderRadius + val rPaddingLength = paddingLength + val rItemWidth = itemWidth + + val existingIndex = + listContainers.indexOfFirst { + it.matches( + rx, + ry, + rw, + rh, + rBorderWidth, + rBorderColor, + rBorderRadius, + rPaddingLength, + rItemWidth, + showSelectionBorder, + normalizedItems + ) + } + if (existingIndex >= 0 && textContainers.isEmpty() && imageContainers.isEmpty() && pageCreated) { + return + } + + textContainers.clear() + imageContainers.clear() + listContainers.clear() + val id = listContainerIDPool.first() + listContainers.add( + ListContainer( + id = id, + x = rx, + y = ry, + width = rw, + height = rh, + borderWidth = rBorderWidth, + borderColor = rBorderColor, + borderRadius = rBorderRadius, + paddingLength = rPaddingLength, + itemWidth = rItemWidth, + showSelectionBorder = showSelectionBorder, + items = normalizedItems + ) + ) + + if (pageCreated) { + rebuildPage() + } + rebuildState() + } + private suspend fun sendText2( text: String, x: Int? = null, @@ -2014,6 +2223,11 @@ class G2 : SGCManager() { return } + val hadListContainers = listContainers.isNotEmpty() + if (hadListContainers) { + listContainers.clear() + } + val rx = x ?: defaultTextX val ry = y ?: defaultTextY val rw = width ?: defaultTextWidth @@ -2055,6 +2269,11 @@ class G2 : SGCManager() { Bridge.log("G2: sendText() - page down, buffering latest content for container ${container.id} (rebuild deferred to reconcile)") return } + if (hadListContainers) { + Bridge.log("G2: sendText() - replacing list page with text container ${container.id}") + rebuildPage() + return + } Bridge.log("G2: sendText() - reusing container ${container.id} for rect $rx,$ry ${rw}x$rh") return } @@ -2107,7 +2326,14 @@ class G2 : SGCManager() { imageContainers[i].bmpData = ByteArray(0) imageContainers[i].dirty = false } + val hadListContainers = listContainers.isNotEmpty() + if (hadListContainers) { + listContainers.clear() + } signalDisplayDirty() + if (hadListContainers && pageCreated) { + displayScope.launch { rebuildPage() } + } } /** @@ -2153,6 +2379,11 @@ class G2 : SGCManager() { return false } + val hadListContainers = listContainers.isNotEmpty() + if (hadListContainers) { + listContainers.clear() + } + // Pure state mutation: update the target container's bytes and mark it dirty. The reconcile // loop is the sole sender, so two displayBitmap calls can never overlap a sendImageData and // clobber the single-slot image ACK — no lock needed. Reuse an existing container if the rect @@ -2167,7 +2398,7 @@ class G2 : SGCManager() { Bridge.log("G2: displayBitmap() - reusing container ${container.id} for rect $rx,$ry ${rw}x$rh") // A brand-new page needs its structure built before the loop can push pixels; the dirty // flag stays set so the reconcile loop sends the image once the page exists. - if (!pageCreated) { + if (!pageCreated || hadListContainers) { displayScope.launch { rebuildPage() } } return true @@ -2344,7 +2575,10 @@ class G2 : SGCManager() { // that would let a clear burst churn the page back up pointlessly. val hasPendingText = textContainers.any { it.pendingSends > 0 && it.content.isNotBlank() } val hasPendingImage = imageContainers.any { it.dirty && it.bmpData.isNotEmpty() } - if ((hasPendingText || hasPendingImage) && !(useNativeDashboard && dashboardShowing > 0)) { + val hasPendingList = listContainers.isNotEmpty() + if ((hasPendingText || hasPendingImage || hasPendingList) && + !(useNativeDashboard && dashboardShowing > 0) + ) { Bridge.log("G2: reconcileDisplay() - page down with pending content, rebuilding once") rebuildState() } @@ -2645,6 +2879,35 @@ class G2 : SGCManager() { // ---------- Private Display Helpers ---------- private fun createPageWithContainers() { + val listContainerProps: List = ArrayList(listContainers.size).apply { + for (i in listContainers.indices) { + val c = listContainers[i] + val itemContainer = + EvenHubProto.listItemContainerProperty( + itemCount = c.items.size, + itemWidth = c.itemWidth, + isItemSelectBorderEn = c.showSelectionBorder, + itemName = c.items + ) + add( + EvenHubProto.listContainerProperty( + x = c.x, + y = c.y, + width = c.width, + height = c.height, + borderWidth = c.borderWidth, + borderColor = c.borderColor, + borderRadius = c.borderRadius, + paddingLength = c.paddingLength, + containerID = c.id, + containerName = c.name, + itemContainer = itemContainer, + isEventCapture = i == 0 + ) + ) + } + } + // build the page's text containers from the live tracked list. val textContainerProps: List = ArrayList(textContainers.size).apply { for (i in textContainers.indices) { @@ -2661,7 +2924,7 @@ class G2 : SGCManager() { paddingLength = c.paddingLength, containerID = c.id, containerName = c.name, - isEventCapture = i == 0,// the first container is the event capture container + isEventCapture = listContainers.isEmpty() && i == 0, content = c.content ) ) @@ -2702,6 +2965,7 @@ class G2 : SGCManager() { Bridge.log("G2: using createPageMessage (first time)") msg = EvenHubProto.createPageMessage( + listContainers = listContainerProps, textContainers = textContainerProps, imageContainers = imageContainerProps, magicRandom = sendManager.nextMagicRandom(), @@ -2711,6 +2975,7 @@ class G2 : SGCManager() { Bridge.log("G2: using rebuildPageMessage") msg = EvenHubProto.rebuildPageMessage( + listContainers = listContainerProps, textContainers = textContainerProps, imageContainers = imageContainerProps, magicRandom = sendManager.nextMagicRandom(), @@ -4116,12 +4381,11 @@ class G2 : SGCManager() { } if (eventType == OsEventType.DOUBLE_CLICK) { - // trigger dashboard: val isHeadUp = DeviceStore.get("glasses", "headUp") as? Boolean ?: false val useNativeDashboard = DeviceStore.get("bluetooth", "use_native_dashboard") as? Boolean ?: false if (useNativeDashboard) { - showDashboard() + Bridge.log("G2: native dashboard double-click shortcut disabled") } else { // toggle head up: DeviceStore.apply("glasses", "headUp", !isHeadUp) @@ -4162,7 +4426,42 @@ class G2 : SGCManager() { return } - // ListEvent (field 1) - interaction with list container (not currently handled) + // ListEvent (field 1) - interaction with list container + (fields[1] as? ByteArray)?.let { listData -> + val listReader = ProtobufReader(listData) + val listFields = listReader.parseFields() + val eventTypeRaw = listFields[5] as? Int + val eventType = + if (eventTypeRaw != null) OsEventType.fromInt(eventTypeRaw) else OsEventType.CLICK + + if (eventType == null) { + Bridge.log("G2: unknown list event type: $listFields") + return@let + } + + val gestureName = mapEventTypeToGesture(eventType) + if (gestureName == null) { + Bridge.log("G2: no gesture mapping for $eventType $listFields") + return@let + } + + val extra = mutableMapOf() + (listFields[1] as? Int)?.let { extra["containerId"] = it } + (listFields[2] as? ByteArray)?.let { + val containerName = String(it, Charsets.UTF_8) + if (containerName.isNotBlank()) { + extra["containerName"] = containerName + } + } + (listFields[3] as? ByteArray)?.let { + extra["selectedItemName"] = String(it, Charsets.UTF_8) + } + (listFields[4] as? Int)?.let { extra["selectedItemIndex"] = it } + + Bridge.sendTouchEvent(DeviceTypes.G2, gestureName, timestamp, extra = extra) + Bridge.log("G2: ListEvent → $gestureName $extra") + return + } } private fun mapEventTypeToGesture(eventType: OsEventType): String? { diff --git a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt index 7a4943bf73..5349eacf8a 100644 --- a/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +++ b/mobile/modules/bluetooth-sdk/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt @@ -91,6 +91,21 @@ abstract class SGCManager { borderWidth: Int = 0, borderRadius: Int = 0 ) {} + open fun sendSelectableList( + items: List, + x: Int = 0, + y: Int = 0, + width: Int = 576, + height: Int = 288, + borderWidth: Int = 1, + borderColor: Int = 13, + borderRadius: Int = 6, + paddingLength: Int = 5, + itemWidth: Int = 0, + showSelectionBorder: Boolean = true + ) { + sendTextWall(items.joinToString("\n")) + } abstract fun showDashboard() abstract fun setDashboardPosition(height: Int, depth: Int) diff --git a/mobile/modules/bluetooth-sdk/ios/Source/Bridge.swift b/mobile/modules/bluetooth-sdk/ios/Source/Bridge.swift index 986d70e321..b8f426f3a5 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/Bridge.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/Bridge.swift @@ -238,7 +238,13 @@ class Bridge { Bridge.sendTypedMessage("button_press", body: body) } - static func sendTouchEvent(deviceModel: String, gestureName: String, timestamp: Int64, source: Int32? = nil) { + static func sendTouchEvent( + deviceModel: String, + gestureName: String, + timestamp: Int64, + source: Int32? = nil, + extra: [String: Any] = [:] + ) { var body: [String: Any] = [ "type": "touch_event", "deviceModel": deviceModel, @@ -248,6 +254,7 @@ class Bridge { if let source { body["source"] = source } + body.merge(extra) { _, new in new } Bridge.sendTypedMessage("touch_event", body: body) } diff --git a/mobile/modules/bluetooth-sdk/ios/Source/DeviceManager.swift b/mobile/modules/bluetooth-sdk/ios/Source/DeviceManager.swift index 29557a5f0b..b68d347a4d 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/DeviceManager.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/DeviceManager.swift @@ -11,7 +11,7 @@ import CoreBluetooth import Foundation import UIKit #if SWIFT_PACKAGE -import MentraBluetoothSDKCoreObjC + import MentraBluetoothSDKCoreObjC #endif struct ViewState { @@ -30,6 +30,12 @@ struct ViewState { // Optional positioned_text border (used by G2; ignored by others) var borderWidth: Int32? = nil var borderRadius: Int32? = nil + // Optional selectable_list fields (used by G2; ignored by others) + var borderColor: Int32? = nil + var paddingLength: Int32? = nil + var listItems: [String] = [] + var itemWidth: Int32? = nil + var showSelectionBorder: Bool? = nil } @MainActor @@ -301,9 +307,9 @@ struct ViewState { private var lastLc3Event: Date? private var micReinitTimer: Timer? - /// STT: + // STT: #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT - private var transcriber: SherpaOnnxTranscriber? + private var transcriber: SherpaOnnxTranscriber? #endif var viewStates: [ViewState] = [ @@ -334,20 +340,20 @@ struct ViewState { // Initialize SherpaOnnx Transcriber #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT - if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, - let window = windowScene.windows.first, - let rootViewController = window.rootViewController - { - transcriber = SherpaOnnxTranscriber(context: rootViewController) - } else { - Bridge.log("Failed to create SherpaOnnxTranscriber - no root view controller found") - } + if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, + let window = windowScene.windows.first, + let rootViewController = window.rootViewController + { + transcriber = SherpaOnnxTranscriber(context: rootViewController) + } else { + Bridge.log("Failed to create SherpaOnnxTranscriber - no root view controller found") + } - // Initialize the transcriber - if let transcriber = transcriber { - transcriber.initialize() - Bridge.log("SherpaOnnxTranscriber fully initialized") - } + // Initialize the transcriber + if let transcriber = transcriber { + transcriber.initialize() + Bridge.log("SherpaOnnxTranscriber fully initialized") + } #endif // Initialize persistent LC3 converter for unified audio encoding @@ -358,8 +364,10 @@ struct ViewState { guard let self = self else { return } self.micReinitTimer = Timer.scheduledTimer( withTimeInterval: 10.0, repeats: true - ) { [weak self] _ in - self?.checkAndReinitGlassesMic() + ) { _ in + Task { @MainActor [weak self] in + self?.checkAndReinitGlassesMic() + } } } } @@ -420,11 +428,11 @@ struct ViewState { handleSendingPcm(pcmData) // Send PCM to local transcriber. -#if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT - if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive { - transcriber?.acceptAudio(pcm16le: pcmData) - } -#endif + #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT + if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive { + transcriber?.acceptAudio(pcm16le: pcmData) + } + #endif } func updateMicState() { @@ -556,47 +564,29 @@ struct ViewState { let delay = 0.25 // Frame delay in seconds let totalCycles = 2 // Number of animation cycles - // Variables to track animation state - var frameIndex = 0 - var cycles = 0 - - // Create a dispatch queue for the animation - let animationQueue = DispatchQueue.global(qos: .userInteractive) - - /// Function to display the current animation frame - func displayFrame() { - // Check if we've completed all cycles - if cycles >= totalCycles { - // End animation with final message - Task { await sgc?.sendTextWall(" /// MentraOS Connected \\\\\\") } - animationQueue.asyncAfter(deadline: .now() + 1.0) { - self.sgc?.clearDisplay() - } - return - } + Task { @MainActor [weak self] in + guard let self = self else { return } - // Display current animation frame - let frameText = - " \(arrowFrames[frameIndex]) MentraOS Booting \(arrowFrames[frameIndex])" - Task { await sgc?.sendTextWall(frameText) } + try? await Task.sleep(nanoseconds: 350_000_000) - // Move to next frame - frameIndex = (frameIndex + 1) % arrowFrames.count + var frameIndex = 0 + var cycles = 0 + while cycles < totalCycles { + let arrow = arrowFrames[frameIndex] + let frameText = " \(arrow) MentraOS Booting \(arrow)" + await self.sgc?.sendTextWall(frameText) - // Count completed cycles - if frameIndex == 0 { - cycles += 1 - } + frameIndex = (frameIndex + 1) % arrowFrames.count + if frameIndex == 0 { + cycles += 1 + } - // Schedule next frame - animationQueue.asyncAfter(deadline: .now() + delay) { - displayFrame() + try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) } - } - // Start the animation after a short initial delay - animationQueue.asyncAfter(deadline: .now() + 0.35) { - displayFrame() + await self.sgc?.sendTextWall(" /// MentraOS Connected \\\\\\") + try? await Task.sleep(nanoseconds: 1_000_000_000) + self.sgc?.clearDisplay() } } @@ -626,21 +616,21 @@ struct ViewState { } else if wearable.contains(DeviceTypes.FRAME) { // sgc = FrameManager() } -#if !SWIFT_PACKAGE || MENTRA_FEATURE_NEX - if sgc == nil && wearable.contains(DeviceTypes.NEX) { - sgc = MentraNexSGC.getInstance() - } -#endif -#if !SWIFT_PACKAGE || MENTRA_FEATURE_VUZIX - if sgc == nil { - if wearable.contains(DeviceTypes.MACH1) { - sgc = Mach1() - } else if wearable.contains(DeviceTypes.Z100) { - sgc = Mach1() // Z100 uses same hardware/SDK as Mach1 - sgc?.type = DeviceTypes.Z100 // Override type to Z100 + #if !SWIFT_PACKAGE || MENTRA_FEATURE_NEX + if sgc == nil && wearable.contains(DeviceTypes.NEX) { + sgc = MentraNexSGC.getInstance() } - } -#endif + #endif + #if !SWIFT_PACKAGE || MENTRA_FEATURE_VUZIX + if sgc == nil { + if wearable.contains(DeviceTypes.MACH1) { + sgc = Mach1() + } else if wearable.contains(DeviceTypes.Z100) { + sgc = Mach1() // Z100 uses same hardware/SDK as Mach1 + sgc?.type = DeviceTypes.Z100 // Override type to Z100 + } + } + #endif // update device model: DeviceStore.shared.apply("glasses", "deviceModel", sgc?.type ?? "") } @@ -730,6 +720,20 @@ struct ViewState { borderWidth: currentViewState.borderWidth ?? 0, borderRadius: currentViewState.borderRadius ?? 0 ) + case "selectable_list": + await sgc?.sendSelectableList( + currentViewState.listItems, + x: currentViewState.bmpX ?? 0, + y: currentViewState.bmpY ?? 0, + width: currentViewState.bmpWidth ?? 576, + height: currentViewState.bmpHeight ?? 288, + borderWidth: currentViewState.borderWidth ?? 1, + borderColor: currentViewState.borderColor ?? 13, + borderRadius: currentViewState.borderRadius ?? 6, + paddingLength: currentViewState.paddingLength ?? 5, + itemWidth: currentViewState.itemWidth ?? 0, + showSelectionBorder: currentViewState.showSelectionBorder ?? true + ) case "clear_view": sgc?.clearDisplay() default: @@ -783,6 +787,29 @@ struct ViewState { return result } + private func viewStateKey(_ state: ViewState) -> String { + var parts: [String] = [] + parts.reserveCapacity(18) + parts.append(state.layoutType) + parts.append(state.text) + parts.append(state.topText) + parts.append(state.bottomText) + parts.append(state.title) + parts.append(state.data ?? "") + parts.append(state.bmpX.map(String.init) ?? "") + parts.append(state.bmpY.map(String.init) ?? "") + parts.append(state.bmpWidth.map(String.init) ?? "") + parts.append(state.bmpHeight.map(String.init) ?? "") + parts.append(state.borderWidth.map(String.init) ?? "") + parts.append(state.borderRadius.map(String.init) ?? "") + parts.append(state.borderColor.map(String.init) ?? "") + parts.append(state.paddingLength.map(String.init) ?? "") + parts.append(state.listItems.joined(separator: "\n")) + parts.append(state.itemWidth.map(String.init) ?? "") + parts.append(state.showSelectionBorder.map(String.init) ?? "") + return parts.joined(separator: "|") + } + private func checkAndReinitGlassesMic() { // if the glasses mic is marked as enabled (and the glasses are connected), but our last known lc3 event is from > 5 seconds ago, reinitialize the mic: let glassesMicEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false @@ -848,7 +875,7 @@ struct ViewState { let deviceName = session.availableInputs?.first(where: { $0.portName.localizedCaseInsensitiveContains(audioDevicePattern) })?.portName - Bridge.log("MAN: ✅ Successfully detected newly paired device '\(deviceName)'") + Bridge.log("MAN: ✅ Successfully detected newly paired device '\(deviceName ?? "unknown")'") glassesBluetoothClassicConnected = true } else { glassesBluetoothClassicConnected = false @@ -885,10 +912,10 @@ struct ViewState { func restartTranscriber() { #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT - Bridge.log("MAN: Restarting SherpaOnnxTranscriber via command") - transcriber?.restart() + Bridge.log("MAN: Restarting SherpaOnnxTranscriber via command") + transcriber?.restart() #else - Bridge.log("MAN: Local STT is not included in this SwiftPM build") + Bridge.log("MAN: Local STT is not included in this SwiftPM build") #endif } @@ -907,7 +934,7 @@ struct ViewState { let connectionKey = "\(sgc.type):\(deviceName)" syncSystemTimeOnceForConnection(sgc, connectionKey: connectionKey) - + // re-apply display height/depth after reconnection DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [weak self] in // Re-read the current sgc rather than capturing the connect-time instance: the user may @@ -949,7 +976,6 @@ struct ViewState { Bridge.saveSetting("default_wearable", defaultWearable) Bridge.saveSetting("device_name", deviceName) Bridge.saveSetting("device_address", deviceAddress) - } private func syncSystemTimeOnceForConnection(_ sgc: SGCManager, connectionKey: String) { @@ -1057,6 +1083,17 @@ struct ViewState { let bmpHeight = (layout["height"] as? NSNumber).map { $0.int32Value } let borderWidth = (layout["borderWidth"] as? NSNumber).map { $0.int32Value } let borderRadius = (layout["borderRadius"] as? NSNumber).map { $0.int32Value } + let borderColor = (layout["borderColor"] as? NSNumber).map { $0.int32Value } + let paddingLength = (layout["paddingLength"] as? NSNumber).map { $0.int32Value } + let rawListItems = (layout["items"] as? [Any]) ?? (layout["itemName"] as? [Any]) ?? [] + let listItems = rawListItems.map { item in + if let stringItem = item as? String { + return stringItem + } + return "\(item)" + } + let itemWidth = (layout["itemWidth"] as? NSNumber).map { $0.int32Value } + let showSelectionBorder = layout["showSelectionBorder"] as? Bool text = parsePlaceholders(text) topText = parsePlaceholders(topText) @@ -1067,7 +1104,9 @@ struct ViewState { topText: topText, bottomText: bottomText, title: title, layoutType: layoutType, text: text, data: data, animationData: nil, bmpX: bmpX, bmpY: bmpY, bmpWidth: bmpWidth, bmpHeight: bmpHeight, - borderWidth: borderWidth, borderRadius: borderRadius + borderWidth: borderWidth, borderRadius: borderRadius, + borderColor: borderColor, paddingLength: paddingLength, listItems: listItems, + itemWidth: itemWidth, showSelectionBorder: showSelectionBorder ) if layoutType == "bitmap_animation" { @@ -1111,10 +1150,8 @@ struct ViewState { let cS = viewStates[stateIndex] let nS = newViewState - let currentState = - cS.layoutType + cS.text + cS.topText + cS.bottomText + cS.title + (cS.data ?? "") - let newState = - nS.layoutType + nS.text + nS.topText + nS.bottomText + nS.title + (nS.data ?? "") + let currentState = viewStateKey(cS) + let newState = viewStateKey(nS) if currentState == newState { // Core.log("MAN: View state is the same, skipping update") @@ -1381,9 +1418,23 @@ struct ViewState { ispDigitalGain: request.ispDigitalGain, ispAnalogGain: request.ispAnalogGain ) - Bridge.log( - "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(routed.requestId) appId=\(routed.appId) webhookUrl=\(routed.webhookUrl ?? "nil") size=\(routed.size.rawValue) compress=\(routed.compress?.rawValue ?? "none") save=\(routed.save) sound=\(routed.sound) exposureTimeNs=\(manualExposureNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") aeDivisor=\(routed.aeExposureDivisor.map { String($0) } ?? "nil") isoCap=\(routed.isoCap.map { String($0) } ?? "nil") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")" - ) + let sgcDescription = sgc.map { String(describing: type(of: $0)) } ?? "null" + let photoLogParts = [ + "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto", + "requestId=\(routed.requestId)", + "appId=\(routed.appId)", + "webhookUrl=\(routed.webhookUrl ?? "nil")", + "size=\(routed.size.rawValue)", + "compress=\(routed.compress?.rawValue ?? "none")", + "save=\(routed.save)", + "sound=\(routed.sound)", + "exposureTimeNs=\(manualExposureNs.map { String($0) } ?? "nil")", + "iso=\(manualIso.map { String($0) } ?? "auto")", + "aeDivisor=\(routed.aeExposureDivisor.map { String($0) } ?? "nil")", + "isoCap=\(routed.isoCap.map { String($0) } ?? "nil")", + "sgc=\(sgcDescription)", + ] + Bridge.log(photoLogParts.joined(separator: " ")) guard let sgc else { Bridge.log( "MAN: PHOTO PIPELINE — sgc is null (glasses not connected); dropping requestId=\(routed.requestId)" @@ -1572,10 +1623,10 @@ struct ViewState { func cleanup() { // Clean up transcriber resources -#if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT - transcriber?.shutdown() - transcriber = nil -#endif + #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT + transcriber?.shutdown() + transcriber = nil + #endif // Clean up LC3 converter lc3Converter = nil diff --git a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/G2.swift b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/G2.swift index 52269ed349..f5e2462eb6 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/G2.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/G2.swift @@ -13,18 +13,18 @@ import UIKit // MARK: - Data Little-Endian Helpers (for BMP construction) -extension Data { - fileprivate mutating func appendLittleEndian(_ value: UInt16) { +private extension Data { + mutating func appendLittleEndian(_ value: UInt16) { var v = value.littleEndian Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) } } - fileprivate mutating func appendLittleEndian(_ value: UInt32) { + mutating func appendLittleEndian(_ value: UInt32) { var v = value.littleEndian Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) } } - fileprivate mutating func appendLittleEndian(_ value: Int32) { + mutating func appendLittleEndian(_ value: Int32) { var v = value.littleEndian Swift.withUnsafeBytes(of: &v) { append(contentsOf: $0) } } @@ -51,47 +51,47 @@ private enum G2BLE { /// Service IDs from service_id_def.proto private enum ServiceID: UInt8 { - case dashboard = 1 // 0x01 - UI_BACKGROUND_DASHBOARD_APP_ID - case menu = 3 // 0x03 - UI_FOREGROUND_MEUN_ID (typo is intentional — matches Even's proto) - case notification = 4 // 0x04 - UI_FOREGROUND_NOTIFICATION_ID - case evenAI = 7 // 0x07 - UI_FOREGROUND_EVEN_AI_ID - case navigation = 8 // 0x08 - UI_BACKGROUND_NAVIGATION_ID (compass/heading lives here) - case g2Setting = 9 // 0x09 - UI_SETTING_APP_ID - case gestureCtrl = 13 // 0x0D - gesture_ctrl lifecycle signals - case onboarding = 16 // 0x10 - UI_ONBOARDING_APP_ID - case deviceSettings = 128 // 0x80 - UX_DEVICE_SETTINGS_APP_ID - case evenHubCtrl = 129 // 0x81 - EvenHub CTRL channel (init/registration) - case evenHub = 224 // 0xE0 - UI_BACKGROUND_EVENHUB_APP_ID + case dashboard = 1 // 0x01 - UI_BACKGROUND_DASHBOARD_APP_ID + case menu = 3 // 0x03 - UI_FOREGROUND_MEUN_ID (typo is intentional — matches Even's proto) + case notification = 4 // 0x04 - UI_FOREGROUND_NOTIFICATION_ID + case evenAI = 7 // 0x07 - UI_FOREGROUND_EVEN_AI_ID + case navigation = 8 // 0x08 - UI_BACKGROUND_NAVIGATION_ID (compass/heading lives here) + case g2Setting = 9 // 0x09 - UI_SETTING_APP_ID + case gestureCtrl = 13 // 0x0D - gesture_ctrl lifecycle signals + case onboarding = 16 // 0x10 - UI_ONBOARDING_APP_ID + case deviceSettings = 128 // 0x80 - UX_DEVICE_SETTINGS_APP_ID + case evenHubCtrl = 129 // 0x81 - EvenHub CTRL channel (init/registration) + case evenHub = 224 // 0xE0 - UI_BACKGROUND_EVENHUB_APP_ID } /// EvenHub command IDs from EvenHub.proto private enum EvenHubCmd: Int32 { - case createStartupPage = 0 // APP_REQUEST_CREATE_STARTUP_PAGE_PACKET - case updateImageRawData = 3 // APP_UPDATE_IMAGE_RAW_DATA_PACKET - case updateTextData = 5 // APP_UPDATE_TEXT_DATA_PACKET - case rebuildPage = 7 // APP_REQUEST_REBUILD_PAGE_PACKET - case shutdownPage = 9 // APP_REQUEST_SHUTDOWN_PAGE_PACKET - case heartbeat = 12 // APP_REQUEST_HEARTBEAT_PACKET - case audioControl = 15 // APP_REQUEST_AUDIO_CTR_PACKET - case imuControl = 19 // APP_REQUEST_IMU_CTR_PACKET (confirmed via on-device brute-force) + case createStartupPage = 0 // APP_REQUEST_CREATE_STARTUP_PAGE_PACKET + case updateImageRawData = 3 // APP_UPDATE_IMAGE_RAW_DATA_PACKET + case updateTextData = 5 // APP_UPDATE_TEXT_DATA_PACKET + case rebuildPage = 7 // APP_REQUEST_REBUILD_PAGE_PACKET + case shutdownPage = 9 // APP_REQUEST_SHUTDOWN_PAGE_PACKET + case heartbeat = 12 // APP_REQUEST_HEARTBEAT_PACKET + case audioControl = 15 // APP_REQUEST_AUDIO_CTR_PACKET + case imuControl = 19 // APP_REQUEST_IMU_CTR_PACKET (confirmed via on-device brute-force) } /// Navigation_Cmd_list from navigation.proto (service 0x08) private enum NavigationCmd: Int32 { - case appSendHeartbeat = 0 // APP_SEND_HEARTBEAT_CMD - case appRequestStartUp = 5 // APP_REQUEST_START_UP — begin navigation/compass session - case appSendBasicInfo = 7 // APP_SEND_BASIC_INFO - case appRequestExit = 12 // APP_REQUEST_EXIT - case osNotifyExit = 13 // OS_NOTIFY_EXIT - case osNotifyReviewChanged = 14 // OS_NOTIFY_REVIEW_CHANGED - case osNotifyCompassChanged = 15 // OS_NOTIFY_COMPASS_CHANGED — heading update - case osNotifyCompassCalibrateStart = 16 // OS_NOTIFY_COMPASS_CALIBRATE_STRAT (sic) - case osNotifyCompassCalibrateComplete = 17 // OS_NOTIFY_COMPASS_CALIBRATE_COMPLETE + case appSendHeartbeat = 0 // APP_SEND_HEARTBEAT_CMD + case appRequestStartUp = 5 // APP_REQUEST_START_UP — begin navigation/compass session + case appSendBasicInfo = 7 // APP_SEND_BASIC_INFO + case appRequestExit = 12 // APP_REQUEST_EXIT + case osNotifyExit = 13 // OS_NOTIFY_EXIT + case osNotifyReviewChanged = 14 // OS_NOTIFY_REVIEW_CHANGED + case osNotifyCompassChanged = 15 // OS_NOTIFY_COMPASS_CHANGED — heading update + case osNotifyCompassCalibrateStart = 16 // OS_NOTIFY_COMPASS_CALIBRATE_STRAT (sic) + case osNotifyCompassCalibrateComplete = 17 // OS_NOTIFY_COMPASS_CALIBRATE_COMPLETE } /// EvenHub response command IDs (from glasses → phone) private enum EvenHubResponseCmd: Int32 { - case osNotifyEventToApp = 2 // OS_NOITY_EVENT_TO_APP_PACKET - touch/gesture events + case osNotifyEventToApp = 2 // OS_NOITY_EVENT_TO_APP_PACKET - touch/gesture events } /// OsEventTypeList from EvenHub.proto @@ -104,16 +104,16 @@ private enum OsEventType: Int32 { case foregroundExit = 5 case abnormalExit = 6 case systemExit = 7 - case imuDataReport = 8 // IMU_DATA_REPORT — Sys_ItemEvent carries imuData + case imuDataReport = 8 // IMU_DATA_REPORT — Sys_ItemEvent carries imuData } /// g2_settingCommandId from g2_setting.proto private enum G2SettingCommandId: Int32 { case none = 0 - case deviceReceiveInfo = 1 // Send settings TO glasses - case deviceReceiveRequest = 2 // Request info FROM glasses - case deviceSendToApp = 3 // Glasses sends info TO app - case deviceRespondToApp = 4 // Glasses responds to app + case deviceReceiveInfo = 1 // Send settings TO glasses + case deviceReceiveRequest = 2 // Request info FROM glasses + case deviceSendToApp = 3 // Glasses sends info TO app + case deviceRespondToApp = 4 // Glasses responds to app } /// DevCfgCommandId from dev_config_protocol.proto @@ -157,7 +157,7 @@ private struct ProtobufWriter { } mutating func writeInt32Field(_ fieldNumber: Int, _ value: Int32) { - let tag = UInt64(fieldNumber << 3) | 0 // wire type 0 = varint + let tag = UInt64(fieldNumber << 3) | 0 // wire type 0 = varint writeVarint(tag) // protobuf int32 uses varint encoding; negative values use 10 bytes if value >= 0 { @@ -168,13 +168,13 @@ private struct ProtobufWriter { } mutating func writeInt64Field(_ fieldNumber: Int, _ value: Int64) { - let tag = UInt64(fieldNumber << 3) | 0 // wire type 0 = varint + let tag = UInt64(fieldNumber << 3) | 0 // wire type 0 = varint writeVarint(tag) writeVarint(UInt64(bitPattern: value)) } mutating func writeStringField(_ fieldNumber: Int, _ value: String) { - let tag = UInt64(fieldNumber << 3) | 2 // wire type 2 = length-delimited + let tag = UInt64(fieldNumber << 3) | 2 // wire type 2 = length-delimited writeVarint(tag) let utf8 = Array(value.utf8) writeVarint(UInt64(utf8.count)) @@ -182,7 +182,7 @@ private struct ProtobufWriter { } mutating func writeBytesField(_ fieldNumber: Int, _ value: Data) { - let tag = UInt64(fieldNumber << 3) | 2 // wire type 2 = length-delimited + let tag = UInt64(fieldNumber << 3) | 2 // wire type 2 = length-delimited writeVarint(tag) writeVarint(UInt64(value.count)) data.append(value) @@ -244,7 +244,7 @@ private struct ProtobufReader { guard let len = readVarint() else { return nil } let length = Int(len) guard offset + length <= data.count else { return nil } - let result = data[(data.startIndex + offset)..<(data.startIndex + offset + length)] + let result = data[(data.startIndex + offset) ..< (data.startIndex + offset + length)] offset += length return Data(result) } @@ -257,10 +257,10 @@ private struct ProtobufReader { /// Skip a field value based on wire type mutating func skipField(wireType: Int) { switch wireType { - case 0: _ = readVarint() // varint - case 1: offset += 8 // 64-bit - case 2: _ = readBytes() // length-delimited - case 5: offset += 4 // 32-bit + case 0: _ = readVarint() // varint + case 1: offset += 8 // 64-bit + case 2: _ = readBytes() // length-delimited + case 5: offset += 4 // 32-bit default: break } } @@ -272,9 +272,9 @@ private struct ProtobufReader { while hasMore { guard let (fieldNum, wireType) = readTag() else { break } switch wireType { - case 0: // varint + case 0: // varint if let v = readVarint() { fields[fieldNum] = Int32(truncatingIfNeeded: v) } - case 2: // length-delimited (submessage or bytes or string) + case 2: // length-delimited (submessage or bytes or string) if let d = readBytes() { fields[fieldNum] = d } default: skipField(wireType: wireType) @@ -296,21 +296,21 @@ private enum EvenHubProto { content: String? = nil ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, x) // XPosition - w.writeInt32Field(2, y) // YPosition - w.writeInt32Field(3, width) // Width - w.writeInt32Field(4, height) // Height - w.writeInt32Field(5, borderWidth) // BorderWidth - w.writeInt32Field(6, borderColor) // BorderColor - w.writeInt32Field(7, borderRadius) // BorderRdaius (sic - typo in proto) - w.writeInt32Field(8, paddingLength) // PaddingLength - w.writeInt32Field(9, containerID) // ContainerID + w.writeInt32Field(1, x) // XPosition + w.writeInt32Field(2, y) // YPosition + w.writeInt32Field(3, width) // Width + w.writeInt32Field(4, height) // Height + w.writeInt32Field(5, borderWidth) // BorderWidth + w.writeInt32Field(6, borderColor) // BorderColor + w.writeInt32Field(7, borderRadius) // BorderRdaius (sic - typo in proto) + w.writeInt32Field(8, paddingLength) // PaddingLength + w.writeInt32Field(9, containerID) // ContainerID if let name = containerName { - w.writeStringField(10, name) // ContainerName + w.writeStringField(10, name) // ContainerName } - w.writeInt32Field(11, isEventCapture ? 1 : 0) // IsEventCapture + w.writeInt32Field(11, isEventCapture ? 1 : 0) // IsEventCapture if let content = content { - w.writeStringField(12, content) // Content + w.writeStringField(12, content) // Content } return w.data } @@ -321,17 +321,57 @@ private enum EvenHubProto { containerID: Int32, containerName: String? = nil ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, x) // XPosition - w.writeInt32Field(2, y) // YPosition - w.writeInt32Field(3, width) // Width - w.writeInt32Field(4, height) // Height - w.writeInt32Field(5, containerID) // ContainerID + w.writeInt32Field(1, x) // XPosition + w.writeInt32Field(2, y) // YPosition + w.writeInt32Field(3, width) // Width + w.writeInt32Field(4, height) // Height + w.writeInt32Field(5, containerID) // ContainerID if let name = containerName { - w.writeStringField(6, name) // ContainerName + w.writeStringField(6, name) // ContainerName } return w.data } + /// Build a ListItemContainerProperty message + static func listItemContainerProperty( + itemCount: Int32, itemWidth: Int32 = 0, isItemSelectBorderEn: Bool = true, + itemName: [String] + ) -> Data { + var w = ProtobufWriter() + w.writeInt32Field(1, itemCount) + w.writeInt32Field(2, itemWidth) + w.writeInt32Field(3, isItemSelectBorderEn ? 1 : 0) + for item in itemName { + w.writeStringField(4, item) + } + return w.data + } + + /// Build a ListContainerProperty message + static func listContainerProperty( + x: Int32, y: Int32, width: Int32, height: Int32, + borderWidth: Int32 = 1, borderColor: Int32 = 13, borderRadius: Int32 = 6, + paddingLength: Int32 = 5, containerID: Int32, containerName: String? = nil, + itemContainer: Data, isEventCapture: Bool = false + ) -> Data { + var w = ProtobufWriter() + w.writeInt32Field(1, x) + w.writeInt32Field(2, y) + w.writeInt32Field(3, width) + w.writeInt32Field(4, height) + w.writeInt32Field(5, borderWidth) + w.writeInt32Field(6, borderColor) + w.writeInt32Field(7, borderRadius) + w.writeInt32Field(8, paddingLength) + w.writeInt32Field(9, containerID) + if let name = containerName { + w.writeStringField(10, name) + } + w.writeMessageField(11, itemContainer) + w.writeInt32Field(12, isEventCapture ? 1 : 0) + return w.data + } + /// Build an ImageRawDataUpdate message static func imageRawDataUpdate( containerID: Int32, containerName: String? = nil, @@ -339,33 +379,36 @@ private enum EvenHubProto { mapFragmentIndex: Int32, mapFragmentPacketSize: Int32, mapRawData: Data ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, containerID) // ContainerID + w.writeInt32Field(1, containerID) // ContainerID if let name = containerName { - w.writeStringField(2, name) // ContainerName - } - w.writeInt32Field(3, mapSessionId) // MapSessionId - w.writeInt32Field(4, mapTotalSize) // MapTotalSize - w.writeInt32Field(5, compressMode) // CompressMode - w.writeInt32Field(6, mapFragmentIndex) // MapFragmentIndex - w.writeInt32Field(7, mapFragmentPacketSize) // MapFragmentPacketSize - w.writeBytesField(8, mapRawData) // MapRawData + w.writeStringField(2, name) // ContainerName + } + w.writeInt32Field(3, mapSessionId) // MapSessionId + w.writeInt32Field(4, mapTotalSize) // MapTotalSize + w.writeInt32Field(5, compressMode) // CompressMode + w.writeInt32Field(6, mapFragmentIndex) // MapFragmentIndex + w.writeInt32Field(7, mapFragmentPacketSize) // MapFragmentPacketSize + w.writeBytesField(8, mapRawData) // MapRawData return w.data } /// Build a CreateStartUpPageContainer message static func createStartupPageContainer( containerTotalNum: Int32, + listContainers: [Data] = [], textContainers: [Data] = [], imageContainers: [Data] = [] ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, containerTotalNum) // ContainerTotalNum - // field 2 = repeated ListContainerProperty ListObject (not used here) + w.writeInt32Field(1, containerTotalNum) // ContainerTotalNum + for lc in listContainers { + w.writeMessageField(2, lc) // field 2 = repeated ListObject + } for tc in textContainers { - w.writeMessageField(3, tc) // field 3 = repeated TextObject + w.writeMessageField(3, tc) // field 3 = repeated TextObject } for ic in imageContainers { - w.writeMessageField(4, ic) // field 4 = repeated ImageObject + w.writeMessageField(4, ic) // field 4 = repeated ImageObject } return w.data } @@ -376,17 +419,17 @@ private enum EvenHubProto { contentLength: Int32, content: String ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, containerID) // ContainerID - w.writeInt32Field(3, contentOffset) // ContentOffset - w.writeInt32Field(4, contentLength) // ContentLength - w.writeStringField(5, content) // Content + w.writeInt32Field(1, containerID) // ContainerID + w.writeInt32Field(3, contentOffset) // ContentOffset + w.writeInt32Field(4, contentLength) // ContentLength + w.writeStringField(5, content) // Content return w.data } /// Build a ShutDownContaniner message (sic - typo in proto) static func shutdownContainer(exitMode: Int32 = 0) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, exitMode) // exitMode + w.writeInt32Field(1, exitMode) // exitMode return w.data } @@ -394,7 +437,7 @@ private enum EvenHubProto { static func heartbeatPacket(cnt: Int32 = 0) -> Data { var w = ProtobufWriter() if cnt != 0 { - w.writeInt32Field(1, cnt) // Cnt + w.writeInt32Field(1, cnt) // Cnt } return w.data } @@ -402,7 +445,7 @@ private enum EvenHubProto { /// Build an AudioCtrCmd message static func audioCtrCmd(enable: Bool) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, enable ? 1 : 0) // AudoFuncEn + w.writeInt32Field(1, enable ? 1 : 0) // AudoFuncEn return w.data } @@ -413,23 +456,25 @@ private enum EvenHubProto { appId: Int32? = nil ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, cmd.rawValue) // Cmd (field 1, enum) - w.writeInt32Field(2, magicRandom) // MagicRandom (field 2) - w.writeMessageField(subFieldNumber, subMessage) // the actual command payload + w.writeInt32Field(1, cmd.rawValue) // Cmd (field 1, enum) + w.writeInt32Field(2, magicRandom) // MagicRandom (field 2) + w.writeMessageField(subFieldNumber, subMessage) // the actual command payload if let appId = appId { - w.writeInt32Field(5, appId) // Associate page with a menu item appId + w.writeInt32Field(5, appId) // Associate page with a menu item appId } return w.data } /// Convenience builders for full evenhub messages static func createPageMessage( + listContainers: [Data] = [], textContainers: [Data] = [], imageContainers: [Data] = [], magicRandom: Int32 = 0, appId _: Int32? = nil ) -> Data { - let total = Int32(textContainers.count + imageContainers.count) + let total = Int32(listContainers.count + textContainers.count + imageContainers.count) let createMsg = createStartupPageContainer( containerTotalNum: total, + listContainers: listContainers, textContainers: textContainers, imageContainers: imageContainers ) @@ -441,14 +486,16 @@ private enum EvenHubProto { // RebuildPageContainer: same structure as CreateStartUpPageContainer, but cmd=7, field 7 static func rebuildPageMessage( + listContainers: [Data] = [], textContainers: [Data] = [], imageContainers: [Data] = [], magicRandom: Int32 = 0, appId: Int32? = nil ) -> Data { - let total = Int32(textContainers.count + imageContainers.count) + let total = Int32(listContainers.count + textContainers.count + imageContainers.count) let rebuildMsg = createStartupPageContainer( containerTotalNum: total, + listContainers: listContainers, textContainers: textContainers, imageContainers: imageContainers ) @@ -504,17 +551,18 @@ private enum EvenHubProto { } // MARK: - IMU control - // - // Wire format recovered by on-device brute-force (sample magnitude ≈ 1.0 g confirms - // the decode). Shapes from even_hub_sdk@0.0.10; numeric proto tags confirmed live: - // EvenHub_Cmd_List IMU command = 19 - // evenhub_main_msg_ctx ImuCtrlCmd slot = field 20 - // ImuCtrlCmd { field 1 = IMU_ReportEn (bool), field 2 = reportFrq (pacing 100…1000) } - // Report path: cmd=2 (osNotifyEventToApp) → SendDeviceEvent.field13 → - // Sys_ItemEvent { field 1 = eventType = 8 (IMU_DATA_REPORT), - // field 3 = imuData = IMU_Report_Data } - // IMU_Report_Data { field 1 = x, 2 = y, 3 = z } — each a 32-bit float (NOT double), - // gravity-normalized (|v| ≈ 1 at rest). + + /// + /// Wire format recovered by on-device brute-force (sample magnitude ≈ 1.0 g confirms + /// the decode). Shapes from even_hub_sdk@0.0.10; numeric proto tags confirmed live: + /// EvenHub_Cmd_List IMU command = 19 + /// evenhub_main_msg_ctx ImuCtrlCmd slot = field 20 + /// ImuCtrlCmd { field 1 = IMU_ReportEn (bool), field 2 = reportFrq (pacing 100…1000) } + /// Report path: cmd=2 (osNotifyEventToApp) → SendDeviceEvent.field13 → + /// Sys_ItemEvent { field 1 = eventType = 8 (IMU_DATA_REPORT), + /// field 3 = imuData = IMU_Report_Data } + /// IMU_Report_Data { field 1 = x, 2 = y, 3 = z } — each a 32-bit float (NOT double), + /// gravity-normalized (|v| ≈ 1 at rest). static let imuCtrlSubField = 20 /// ImuReportPace pacing codes (protocol values, NOT literal Hz). Step 100, 100…1000. @@ -525,9 +573,9 @@ private enum EvenHubProto { /// Build an ImuCtrlCmd sub-message. static func imuCtrlCmd(enable: Bool, reportFrq: Int32) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, enable ? 1 : 0) // IMU_ReportEn + w.writeInt32Field(1, enable ? 1 : 0) // IMU_ReportEn if enable { - w.writeInt32Field(2, reportFrq) // reportFrq (pacing code 100…1000) + w.writeInt32Field(2, reportFrq) // reportFrq (pacing code 100…1000) } return w.data } @@ -539,9 +587,9 @@ private enum EvenHubProto { ) -> Data { let imuMsg = imuCtrlCmd(enable: enable, reportFrq: reportFrq) var w = ProtobufWriter() - w.writeInt32Field(1, EvenHubCmd.imuControl.rawValue) // Cmd - w.writeInt32Field(2, magicRandom) // MagicRandom - w.writeMessageField(imuCtrlSubField, imuMsg) // ImuCtrlCmd slot (field 20) + w.writeInt32Field(1, EvenHubCmd.imuControl.rawValue) // Cmd + w.writeInt32Field(2, magicRandom) // MagicRandom + w.writeMessageField(imuCtrlSubField, imuMsg) // ImuCtrlCmd slot (field 20) return w.data } } @@ -556,17 +604,17 @@ private enum DevSettingsProto { // field 2 = magicRandom (int32) // field 3 = authMgr (AuthMgr message) var w = ProtobufWriter() - w.writeInt32Field(1, DevCfgCommandId.authentication.rawValue) // commandId - w.writeInt32Field(2, magicRandom) // magicRandom + w.writeInt32Field(1, DevCfgCommandId.authentication.rawValue) // commandId + w.writeInt32Field(2, magicRandom) // magicRandom // AuthMgr sub-message: // field 1 = secAuth (bool) // field 2 = phoneType (enum eDevice: PHONE_IOS=3, PHONE_ANDROID=4) var authW = ProtobufWriter() - authW.writeBoolField(1, true) // secAuth - authW.writeInt32Field(2, 3) // phoneType = PHONE_IOS (eDevice.PHONE_IOS=3) + authW.writeBoolField(1, true) // secAuth + authW.writeInt32Field(2, 3) // phoneType = PHONE_IOS (eDevice.PHONE_IOS=3) - w.writeMessageField(3, authW.data) // authMgr + w.writeMessageField(3, authW.data) // authMgr return w.data } @@ -578,8 +626,8 @@ private enum DevSettingsProto { // PipeRoleChange: field 1 = asCmdRole (enum GlassesLR.RIGHT=1) var roleW = ProtobufWriter() - roleW.writeInt32Field(1, 1) // RIGHT - w.writeMessageField(4, roleW.data) // roleChange (field 4 in DevCfgDataPackage) + roleW.writeInt32Field(1, 1) // RIGHT + w.writeMessageField(4, roleW.data) // roleChange (field 4 in DevCfgDataPackage) return w.data } @@ -596,7 +644,7 @@ private enum DevSettingsProto { let nowSec = Int64(Date().timeIntervalSince1970) let tzSec = Int64(TimeZone.current.secondsFromGMT()) tsW.writeInt32Field(1, Int32(truncatingIfNeeded: nowSec + tzSec)) - w.writeMessageField(128, tsW.data) // timeSync (field 128 in DevCfgDataPackage) + w.writeMessageField(128, tsW.data) // timeSync (field 128 in DevCfgDataPackage) return w.data } @@ -636,8 +684,8 @@ private enum DevSettingsProto { // BaseConnHeartBeat: empty message var hbW = ProtobufWriter() - _ = hbW // empty - w.writeMessageField(13, hbW.data) // baseHeartBeat (field 13) + _ = hbW // empty + w.writeMessageField(13, hbW.data) // baseHeartBeat (field 13) return w.data } @@ -648,18 +696,18 @@ private enum DevSettingsProto { magicRandom: Int32, connect: Bool, ringMac: Data, ringName: String = "" ) -> Data { var w = ProtobufWriter() - w.writeInt32Field(1, DevCfgCommandId.ringConnectInfo.rawValue) // commandId = RING_CONNECT_INFO (6) + w.writeInt32Field(1, DevCfgCommandId.ringConnectInfo.rawValue) // commandId = RING_CONNECT_INFO (6) w.writeInt32Field(2, magicRandom) // RingInfo sub-message (field 5 in DevCfgDataPackage) var ringW = ProtobufWriter() - ringW.writeBoolField(1, connect) // connectRing - ringW.writeBytesField(2, ringMac) // ringMac (6 bytes) + ringW.writeBoolField(1, connect) // connectRing + ringW.writeBytesField(2, ringMac) // ringMac (6 bytes) if !ringName.isEmpty { - ringW.writeBytesField(3, Data(ringName.utf8)) // ringName + ringW.writeBytesField(3, Data(ringName.utf8)) // ringName } - w.writeMessageField(5, ringW.data) // ringInfo (field 5) + w.writeMessageField(5, ringW.data) // ringInfo (field 5) return w.data } } @@ -671,18 +719,18 @@ private enum G2SettingProto { static func setBrightness(magicRandom: Int32, level: Int32, autoAdjust: Bool) -> Data { // DeviceReceive_Brightness var brightnessW = ProtobufWriter() - brightnessW.writeInt32Field(1, autoAdjust ? 1 : 0) // autoAdjust - brightnessW.writeInt32Field(2, level) // brightnessLevel + brightnessW.writeInt32Field(1, autoAdjust ? 1 : 0) // autoAdjust + brightnessW.writeInt32Field(2, level) // brightnessLevel // DeviceReceiveInfoFromAPP var infoW = ProtobufWriter() - infoW.writeMessageField(1, brightnessW.data) // deviceReceiveBrightness (field 1) + infoW.writeMessageField(1, brightnessW.data) // deviceReceiveBrightness (field 1) // G2SettingPackage var w = ProtobufWriter() - w.writeInt32Field(1, G2SettingCommandId.deviceReceiveInfo.rawValue) // commandId + w.writeInt32Field(1, G2SettingCommandId.deviceReceiveInfo.rawValue) // commandId w.writeInt32Field(2, magicRandom) - w.writeMessageField(3, infoW.data) // deviceReceiveInfoFromApp (field 3) + w.writeMessageField(3, infoW.data) // deviceReceiveInfoFromApp (field 3) return w.data } @@ -691,13 +739,13 @@ private enum G2SettingProto { // DeviceReceiveRequestFromAPP - empty message triggers glasses to respond with all fields var reqW = ProtobufWriter() // Request brightness info type - reqW.writeInt32Field(1, 1) // settingInfoType = APP_REQUIRE_BASIC_SETTING + reqW.writeInt32Field(1, 1) // settingInfoType = APP_REQUIRE_BASIC_SETTING // G2SettingPackage var w = ProtobufWriter() - w.writeInt32Field(1, G2SettingCommandId.deviceReceiveRequest.rawValue) // commandId + w.writeInt32Field(1, G2SettingCommandId.deviceReceiveRequest.rawValue) // commandId w.writeInt32Field(2, magicRandom) - w.writeMessageField(4, reqW.data) // deviceReceiveRequestFromApp (field 4) + w.writeMessageField(4, reqW.data) // deviceReceiveRequestFromApp (field 4) return w.data } @@ -705,17 +753,17 @@ private enum G2SettingProto { static func setHeadUpSwitch(magicRandom: Int32, enabled: Bool) -> Data { // DeviceReceive_Head_UP_Setting var headUpW = ProtobufWriter() - headUpW.writeInt32Field(1, enabled ? 1 : 0) // headUpSwitch + headUpW.writeInt32Field(1, enabled ? 1 : 0) // headUpSwitch // DeviceReceiveInfoFromAPP var infoW = ProtobufWriter() - infoW.writeMessageField(4, headUpW.data) // deviceReceiveHeadUpSetting (field 4) + infoW.writeMessageField(4, headUpW.data) // deviceReceiveHeadUpSetting (field 4) // G2SettingPackage var w = ProtobufWriter() w.writeInt32Field(1, G2SettingCommandId.deviceReceiveInfo.rawValue) w.writeInt32Field(2, magicRandom) - w.writeMessageField(3, infoW.data) // deviceReceiveInfoFromApp (field 3) + w.writeMessageField(3, infoW.data) // deviceReceiveInfoFromApp (field 3) return w.data } @@ -723,11 +771,11 @@ private enum G2SettingProto { static func setHeadUpAngle(magicRandom: Int32, angle: Int32) -> Data { // DeviceReceive_Head_UP_Setting var headUpW = ProtobufWriter() - headUpW.writeInt32Field(2, angle) // headUpAngle (field 2) + headUpW.writeInt32Field(2, angle) // headUpAngle (field 2) // DeviceReceiveInfoFromAPP var infoW = ProtobufWriter() - infoW.writeMessageField(4, headUpW.data) // deviceReceiveHeadUpSetting (field 4) + infoW.writeMessageField(4, headUpW.data) // deviceReceiveHeadUpSetting (field 4) // G2SettingPackage var w = ProtobufWriter() @@ -741,11 +789,11 @@ private enum G2SettingProto { static func setScreenHeight(magicRandom: Int32, level: Int32) -> Data { // DeviceReceive_Y_Coordinate var yW = ProtobufWriter() - yW.writeInt32Field(1, level) // yCoordinateLevel + yW.writeInt32Field(1, level) // yCoordinateLevel // DeviceReceiveInfoFromAPP var infoW = ProtobufWriter() - infoW.writeMessageField(2, yW.data) // deviceReceiveYCoordinate (field 2) + infoW.writeMessageField(2, yW.data) // deviceReceiveYCoordinate (field 2) // G2SettingPackage var w = ProtobufWriter() @@ -759,11 +807,11 @@ private enum G2SettingProto { static func setScreenDepth(magicRandom: Int32, level: Int32) -> Data { // DeviceReceive_X_Coordinate var xW = ProtobufWriter() - xW.writeInt32Field(1, level) // xCoordinateLevel + xW.writeInt32Field(1, level) // xCoordinateLevel // DeviceReceiveInfoFromAPP var infoW = ProtobufWriter() - infoW.writeMessageField(3, xW.data) // deviceReceiveXCoordinate (field 3) + infoW.writeMessageField(3, xW.data) // deviceReceiveXCoordinate (field 3) // G2SettingPackage var w = ProtobufWriter() @@ -781,13 +829,13 @@ private enum OnboardingProto { static func skipOnboarding(magicRandom: Int32) -> Data { // OnboardingConfig: processId = FINISH (4) var configW = ProtobufWriter() - configW.writeInt32Field(1, 4) // processId = FINISH + configW.writeInt32Field(1, 4) // processId = FINISH // OnboardingDataPackage var w = ProtobufWriter() - w.writeInt32Field(1, 1) // commandId = CONFIG + w.writeInt32Field(1, 1) // commandId = CONFIG w.writeInt32Field(2, magicRandom) - w.writeMessageField(3, configW.data) // config (field 3) + w.writeMessageField(3, configW.data) // config (field 3) return w.data } } @@ -807,15 +855,15 @@ private enum EvenAIProto { // EvenAIConfig var configW = ProtobufWriter() if enabled { - configW.writeInt32Field(1, 1) // voiceSwitch (omitted when off, matching the app) + configW.writeInt32Field(1, 1) // voiceSwitch (omitted when off, matching the app) } - configW.writeInt32Field(2, 32) // streamSpeed (always sent, app uses 32) + configW.writeInt32Field(2, 32) // streamSpeed (always sent, app uses 32) // EvenAIDataPackage var w = ProtobufWriter() - w.writeInt32Field(1, 10) // commandId = CONFIG + w.writeInt32Field(1, 10) // commandId = CONFIG w.writeInt32Field(2, magicRandom) - w.writeMessageField(13, configW.data) // config (field 13) + w.writeMessageField(13, configW.data) // config (field 13) return w.data } @@ -825,13 +873,13 @@ private enum EvenAIProto { /// into the glasses' AI session so the following SKILL packet has context. static func aiAsk(magicRandom: Int32, text: String, streamEnable: Int32 = 0) -> Data { var askW = ProtobufWriter() - askW.writeInt32Field(2, streamEnable) // streamEnable - askW.writeBytesField(4, Data(text.utf8)) // text + askW.writeInt32Field(2, streamEnable) // streamEnable + askW.writeBytesField(4, Data(text.utf8)) // text var w = ProtobufWriter() - w.writeInt32Field(1, 3) // commandId = ASK + w.writeInt32Field(1, 3) // commandId = ASK w.writeInt32Field(2, magicRandom) - w.writeMessageField(5, askW.data) // askInfo (field 5) + w.writeMessageField(5, askW.data) // askInfo (field 5) return w.data } @@ -841,12 +889,12 @@ private enum EvenAIProto { /// status: 1 WAKE_UP, 2 ENTER, 3 EXIT static func aiCtrl(magicRandom: Int32, status: Int32) -> Data { var ctrlW = ProtobufWriter() - ctrlW.writeInt32Field(1, status) // status + ctrlW.writeInt32Field(1, status) // status var w = ProtobufWriter() - w.writeInt32Field(1, 1) // commandId = CTRL + w.writeInt32Field(1, 1) // commandId = CTRL w.writeInt32Field(2, magicRandom) - w.writeMessageField(3, ctrlW.data) // ctrl (field 3) + w.writeMessageField(3, ctrlW.data) // ctrl (field 3) return w.data } @@ -861,17 +909,17 @@ private enum EvenAIProto { ) -> Data { // EvenAISkillInfo var skillW = ProtobufWriter() - skillW.writeInt32Field(1, streamEnable) // streamEnable - skillW.writeInt32Field(2, skillId) // skillId - skillW.writeInt32Field(3, skillParam) // skillParam — for NOTIFICATION skill this is a NotificationType enum - skillW.writeBytesField(4, Data(text.utf8)) // text (utterance / payload) - skillW.writeInt32Field(6, fTextEnd) // fTextEnd — 1 signals "this is the final/complete packet" + skillW.writeInt32Field(1, streamEnable) // streamEnable + skillW.writeInt32Field(2, skillId) // skillId + skillW.writeInt32Field(3, skillParam) // skillParam — for NOTIFICATION skill this is a NotificationType enum + skillW.writeBytesField(4, Data(text.utf8)) // text (utterance / payload) + skillW.writeInt32Field(6, fTextEnd) // fTextEnd — 1 signals "this is the final/complete packet" // EvenAIDataPackage var w = ProtobufWriter() - w.writeInt32Field(1, 6) // commandId = SKILL + w.writeInt32Field(1, 6) // commandId = SKILL w.writeInt32Field(2, magicRandom) - w.writeMessageField(8, skillW.data) // skillInfo (field 8) + w.writeMessageField(8, skillW.data) // skillInfo (field 8) return w.data } } @@ -886,13 +934,13 @@ private enum NotificationProto { /// (Returned errorCode=8 NOT_SUPPORT in testing — Service 4 doesn't accept this outbound.) static func iosNotification(magicRandom: Int32, appID: String, displayName: String) -> Data { var iosW = ProtobufWriter() - iosW.writeBytesField(1, Data(appID.utf8)) // appID - iosW.writeBytesField(2, Data(displayName.utf8)) // displayName + iosW.writeBytesField(1, Data(appID.utf8)) // appID + iosW.writeBytesField(2, Data(displayName.utf8)) // displayName var w = ProtobufWriter() - w.writeInt32Field(1, 2) // commandId = NOTIFICATION_IOS + w.writeInt32Field(1, 2) // commandId = NOTIFICATION_IOS w.writeInt32Field(2, magicRandom) - w.writeMessageField(4, iosW.data) // IOS (field 4) + w.writeMessageField(4, iosW.data) // IOS (field 4) return w.data } @@ -909,15 +957,15 @@ private enum NotificationProto { avoidDisturbEnable: Int32 = 0 ) -> Data { var ctrlW = ProtobufWriter() - ctrlW.writeInt32Field(1, notifEnable) // notifEnable - ctrlW.writeInt32Field(2, autoDispEnable) // autoDispEnable - ctrlW.writeInt32Field(3, dispTime) // dispTime (seconds) - ctrlW.writeInt32Field(5, avoidDisturbEnable) // avoidDisturbEnable + ctrlW.writeInt32Field(1, notifEnable) // notifEnable + ctrlW.writeInt32Field(2, autoDispEnable) // autoDispEnable + ctrlW.writeInt32Field(3, dispTime) // dispTime (seconds) + ctrlW.writeInt32Field(5, avoidDisturbEnable) // avoidDisturbEnable var w = ProtobufWriter() - w.writeInt32Field(1, 1) // commandId = NOTIFICATION_CTRL + w.writeInt32Field(1, 1) // commandId = NOTIFICATION_CTRL w.writeInt32Field(2, magicRandom) - w.writeMessageField(3, ctrlW.data) // ctrl (field 3) + w.writeMessageField(3, ctrlW.data) // ctrl (field 3) return w.data } } @@ -935,7 +983,7 @@ private enum MenuProto { /// G2 firmware requires minimum 5, maximum 10 menu items static let MIN_MENU_SIZE = 5 static let MAX_MENU_SIZE = 10 - static let MAX_NAME_LENGTH = 15 // 17 char limit minus 2 for running indicator prefix + static let MAX_NAME_LENGTH = 15 // 17 char limit minus 2 for running indicator prefix /// Placeholder appIds for padding slots (in valid Even range, unique per slot) static let PLACEHOLDER_APP_IDS: [Int32] = [10535, 10536, 10537, 10538, 10539] @@ -962,7 +1010,7 @@ private enum MenuProto { // Wire items carry either a built-in (itemType=0, no name) or third-party (itemType=1, with name) struct WireItem { - let displayName: String? // nil for built-ins + let displayName: String? // nil for built-ins let appId: Int32 let isBuiltIn: Bool } @@ -979,8 +1027,8 @@ private enum MenuProto { let truncated = item.name.count > MAX_NAME_LENGTH - ? String(item.name.prefix(MAX_NAME_LENGTH)) - : item.name + ? String(item.name.prefix(MAX_NAME_LENGTH)) + : item.name let prefix = item.running ? "● " : "" wireItems.append( WireItem(displayName: prefix + truncated, appId: appId, isBuiltIn: false) @@ -989,7 +1037,7 @@ private enum MenuProto { // Pad to MIN_MENU_SIZE with placeholder third-party items while wireItems.count < MIN_MENU_SIZE { - let idx = wireItems.count - 1 // -1 because built-in occupies slot 0 + let idx = wireItems.count - 1 // -1 because built-in occupies slot 0 wireItems.append( WireItem( displayName: " ---", @@ -1001,27 +1049,27 @@ private enum MenuProto { // MenuInfoSend var menuW = ProtobufWriter() - menuW.writeInt32Field(1, Int32(wireItems.count)) // itemTotalNum + menuW.writeInt32Field(1, Int32(wireItems.count)) // itemTotalNum for item in wireItems { var itemW = ProtobufWriter() if item.isBuiltIn { - itemW.writeInt32Field(1, 0) // itemType = 0 (built-in) - itemW.writeInt32Field(4, item.appId) // itemAppId = SID + itemW.writeInt32Field(1, 0) // itemType = 0 (built-in) + itemW.writeInt32Field(4, item.appId) // itemAppId = SID } else { - itemW.writeInt32Field(1, 1) // itemType = 1 (third-party) - itemW.writeInt32Field(2, 1) // iconNum = 1 - itemW.writeStringField(3, item.displayName ?? "") // itemName - itemW.writeInt32Field(4, item.appId) // itemAppId + itemW.writeInt32Field(1, 1) // itemType = 1 (third-party) + itemW.writeInt32Field(2, 1) // iconNum = 1 + itemW.writeStringField(3, item.displayName ?? "") // itemName + itemW.writeInt32Field(4, item.appId) // itemAppId } - menuW.writeMessageField(2, itemW.data) // repeated item (field 2) + menuW.writeMessageField(2, itemW.data) // repeated item (field 2) } // meun_main_msg_ctx var w = ProtobufWriter() - w.writeInt32Field(1, 0) // Cmd = APP_SEND_MENU_INFO (0) - w.writeInt32Field(2, magicRandom) // MagicRandom - w.writeMessageField(3, menuW.data) // sendData (field 3) + w.writeInt32Field(1, 0) // Cmd = APP_SEND_MENU_INFO (0) + w.writeInt32Field(2, magicRandom) // MagicRandom + w.writeMessageField(3, menuW.data) // sendData (field 3) return (w.data, appIdMap) } } @@ -1034,7 +1082,7 @@ private enum DashboardProto { /// eDashboardCommandId values from dashboard.proto enum CommandId: Int32 { case dashboardRespond = 1 - case dashboardReceive = 2 // phone → glasses widget/config push + case dashboardReceive = 2 // phone → glasses widget/config push case appRespond = 3 case appReceive = 4 } @@ -1179,7 +1227,7 @@ private struct EvenBLETransport { var offset = 0 while offset < payload.count { let end = min(offset + maxPayload, payload.count) - chunks.append(payload[offset..> 8) & 0xFF)) // CRC high + packet.append(UInt8(crc & 0xFF)) // CRC low + packet.append(UInt8((crc >> 8) & 0xFF)) // CRC high } packets.append(packet) @@ -1260,10 +1308,9 @@ private class G2SendManager { // MARK: - G2 Receive Manager (multi-part reassembly) private class G2ReceiveManager { - private var partials: [String: (Data, UInt8)] = [:] // key -> (accumulated payload, lastSerialNum) + private var partials: [String: (Data, UInt8)] = [:] // key -> (accumulated payload, lastSerialNum) - func handlePacket(_ rawData: Data, sourceKey: String = "") -> (serviceId: UInt8, payload: Data)? - { + func handlePacket(_ rawData: Data, sourceKey: String = "") -> (serviceId: UInt8, payload: Data)? { guard rawData.count >= 8 else { return nil } guard rawData[0] == G2BLE.HEADER_BYTE else { return nil } @@ -1282,7 +1329,7 @@ private class G2ReceiveManager { let isLast = (serialNum == totalPackets) let hasCrc = isLast let payloadEnd = 8 + payloadLen - (hasCrc ? 2 : 0) - let payload = rawData[8..? private let intervalSeconds: TimeInterval private var attempts = 0 - private let maxAttempts: Int // -1 for unlimited + private let maxAttempts: Int // -1 for unlimited init(intervalSeconds: TimeInterval = 30, maxAttempts: Int = -1) { self.intervalSeconds = intervalSeconds @@ -1455,11 +1502,11 @@ class G2: NSObject, SGCManager { private var pairingTimeoutTimer: DispatchWorkItem? private var useEvenDashboard = true private var dashboardShowing = 0 - // The 08011A00 gesture_ctrl event is ambiguous: the firmware sends it BOTH when the dashboard - // opens (it shuts our page down to take the screen) and when it closes (returns to us). When - // showDashboard() runs we set this latch; the next 08011A00 is the OPEN confirm — consume it - // WITHOUT recovering (else we rebuild our page and snatch the screen back from the dashboard). - // The following 08011A00 is the real CLOSE → recover. + /// The 08011A00 gesture_ctrl event is ambiguous: the firmware sends it BOTH when the dashboard + /// opens (it shuts our page down to take the screen) and when it closes (returns to us). When + /// showDashboard() runs we set this latch; the next 08011A00 is the OPEN confirm — consume it + /// WITHOUT recovering (else we rebuild our page and snatch the screen back from the dashboard). + /// The following 08011A00 is the real CLOSE → recover. private var dashboardOpening = false // Recovery throttle: the firmware spams systemExit + dashboard-close ~1×/sec on its own. // Coalesce so recovery can't storm — one rebuild in flight, one per RECOVERY_DEBOUNCE_MS. @@ -1518,7 +1565,7 @@ class G2: NSObject, SGCManager { private let sendManager = G2SendManager() private let receiveManager = G2ReceiveManager() private var foregroundObserver: NSObjectProtocol? - private var startupPageCreated: Bool = false // createStartUpPageContainer can only be called once + private var startupPageCreated: Bool = false // createStartUpPageContainer can only be called once private var pageCreated: Bool = false // Live hardware truth: is the firmware mic actually streaming. DISTINCT from the // glasses/micEnabled DeviceStore flag, which is *intent* (does the user want the mic on). @@ -1543,7 +1590,7 @@ class G2: NSObject, SGCManager { /// Wakes the reconcile loop the instant a container is marked dirty, instead of waiting out the /// idle tick. `signalDisplayDirty()` (and the ticker) yield into this; the loop drains it. private var displayDirtySignal: AsyncStream.Continuation? - private let IMG_ACK_TIMEOUT_NS: UInt64 = 2_000_000_000 // 1000ms timeout (matches Dart host) + private let IMG_ACK_TIMEOUT_NS: UInt64 = 2_000_000_000 // 1000ms timeout (matches Dart host) private let IMG_MAX_ATTEMPTS = 3 private var heartbeatTask: Task? private var heartbeatCounter: Int = 0 @@ -1574,6 +1621,7 @@ class G2: NSObject, SGCManager { var name: String { "img-\(id)" } + var bmpData: Data /// Set true when `bmpData` changes and the new pixels haven't been pushed to the glasses yet. /// The reconcile loop (see `displayReconcileTask`) is the sole sender; it clears this once the @@ -1615,13 +1663,45 @@ class G2: NSObject, SGCManager { } } + private struct ListContainer: Equatable { + let id: Int32 + let x: Int32 + let y: Int32 + let width: Int32 + let height: Int32 + let borderWidth: Int32 + let borderColor: Int32 + let borderRadius: Int32 + let paddingLength: Int32 + let itemWidth: Int32 + let showSelectionBorder: Bool + let items: [String] + var name: String { + "list-\(id)" + } + + func matches( + x: Int32, y: Int32, width: Int32, height: Int32, borderWidth: Int32, + borderColor: Int32, borderRadius: Int32, paddingLength: Int32, itemWidth: Int32, + showSelectionBorder: Bool, items: [String] + ) -> Bool { + self.x == x && self.y == y && self.width == width && self.height == height + && self.borderWidth == borderWidth && self.borderColor == borderColor + && self.borderRadius == borderRadius && self.paddingLength == paddingLength + && self.itemWidth == itemWidth && self.showSelectionBorder == showSelectionBorder + && self.items == items + } + } + /// Live list of image containers on the page, ordered oldest→newest (for LRU eviction). /// The page may hold at most 4 image containers (IDs from the pool below). private var imageContainers: [ImgContainer] = [] private var textContainers: [TextContainer] = [] + private var listContainers: [ListContainer] = [] /// Fixed pool of container IDs the page protocol expects. private let imageContainerIDPool: [Int32] = [10, 11, 12, 13] private let textContainerIDPool: [Int32] = [1, 2, 3, 4, 5, 6] + private let listContainerIDPool: [Int32] = [7, 8] private static let defaultImgContainer = ( x: Int32(188), y: Int32(44), width: Int32(200), height: Int32(100) ) @@ -1813,44 +1893,44 @@ class G2: NSObject, SGCManager { // Small delay then auth right + pipe role change + time sync try? await Task.sleep(nanoseconds: 200_000_000) - let authR = DevSettingsProto.authCmd(magicRandom: self.sendManager.nextMagicRandom()) - self.sendDevSettingsCommand(authR, left: false, right: true) + let authR = DevSettingsProto.authCmd(magicRandom: sendManager.nextMagicRandom()) + sendDevSettingsCommand(authR, left: false, right: true) try? await Task.sleep(nanoseconds: 200_000_000) let roleChange = DevSettingsProto.pipeRoleChange( - magicRandom: self.sendManager.nextMagicRandom() + magicRandom: sendManager.nextMagicRandom() ) - self.sendDevSettingsCommand(roleChange, left: false, right: true) + sendDevSettingsCommand(roleChange, left: false, right: true) try? await Task.sleep(nanoseconds: 200_000_000) let timeSync = DevSettingsProto.timeSync( - magicRandom: self.sendManager.nextMagicRandom() + magicRandom: sendManager.nextMagicRandom() ) - self.sendDevSettingsCommand(timeSync) + sendDevSettingsCommand(timeSync) // Skip onboarding on connect try? await Task.sleep(nanoseconds: 200_000_000) let onboarding = OnboardingProto.skipOnboarding( - magicRandom: self.sendManager.nextMagicRandom() + magicRandom: sendManager.nextMagicRandom() ) - self.sendOnboardingCommand(onboarding) + sendOnboardingCommand(onboarding) Bridge.log("G2: Sent onboarding skip (FINISH)") // 1. gesture_ctrl init (field1=0, field2=magicRandom) var gestureInitW = ProtobufWriter() gestureInitW.writeInt32Field(1, 0) - gestureInitW.writeInt32Field(2, self.sendManager.nextMagicRandom()) - self.sendGestureCtrlCommand(gestureInitW.data) + gestureInitW.writeInt32Field(2, sendManager.nextMagicRandom()) + sendGestureCtrlCommand(gestureInitW.data) // 2. ui_setting_app (0x0C) — query (cmd=2, field4={settingInfoType=1, autoBrightnessLevel=0}) var uiSettW = ProtobufWriter() - uiSettW.writeInt32Field(1, 2) // cmd = DeviceReceiveRequest - uiSettW.writeInt32Field(2, self.sendManager.nextMagicRandom()) - uiSettW.writeMessageField(4, Data([0x08, 0x01, 0x10, 0x00])) // {1:1, 2:0} - self.sendToGlasses( - self.sendManager.buildPackets( + uiSettW.writeInt32Field(1, 2) // cmd = DeviceReceiveRequest + uiSettW.writeInt32Field(2, sendManager.nextMagicRandom()) + uiSettW.writeMessageField(4, Data([0x08, 0x01, 0x10, 0x00])) // {1:1, 2:0} + sendToGlasses( + sendManager.buildPackets( serviceId: 0x0C, payload: uiSettW.data, reserveFlag: true ) ) @@ -1859,30 +1939,30 @@ class G2: NSObject, SGCManager { // halfDayFormat: 1 = 12h, 0 = 24h // temperatureUnit: 1 = Celsius (metric), 2 = Fahrenheit (imperial) var dashDisplayW = ProtobufWriter() - dashDisplayW.writeInt32Field(1, 4) // displayMode - dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount - dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder - dashDisplayW.writeInt32Field(4, 4) // widgetDisplayCount + dashDisplayW.writeInt32Field(1, 4) // displayMode + dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount + dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder + dashDisplayW.writeInt32Field(4, 4) // widgetDisplayCount // WidgetType: 1=News, 2=Stock, 3=Schedule, 4=Quicklist, 5=Health - dashDisplayW.writeMessageField(5, Data([3, 1, 2, 4, 5])) // widgetDisplayOrder: Schedule, News, Stock, Quicklist - dashDisplayW.writeInt32Field(6, self.dashboardHalfDayFormat()) // halfDayFormat - dashDisplayW.writeInt32Field(7, self.dashboardTemperatureUnit()) // temperatureUnit + dashDisplayW.writeMessageField(5, Data([3, 1, 2, 4, 5])) // widgetDisplayOrder: Schedule, News, Stock, Quicklist + dashDisplayW.writeInt32Field(6, dashboardHalfDayFormat()) // halfDayFormat + dashDisplayW.writeInt32Field(7, dashboardTemperatureUnit()) // temperatureUnit var dashRecvW = ProtobufWriter() dashRecvW.writeMessageField(2, dashDisplayW.data) var dashPkgW = ProtobufWriter() - dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive - dashPkgW.writeInt32Field(2, self.sendManager.nextMagicRandom()) + dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive + dashPkgW.writeInt32Field(2, sendManager.nextMagicRandom()) dashPkgW.writeMessageField(4, dashRecvW.data) - self.sendDashboardCommand(dashPkgW.data) + sendDashboardCommand(dashPkgW.data) // Disable "Hey Even" wakeword on connect let heyEvenOff = EvenAIProto.setHeyEven( - magicRandom: self.sendManager.nextMagicRandom(), + magicRandom: sendManager.nextMagicRandom(), enabled: false ) - self.sendEvenAICommand(heyEvenOff) + sendEvenAICommand(heyEvenOff) Bridge.log("G2: Disabled Hey Even wakeword") // 7. Dashboard REQUEST_NEWS_INFO (cmd=5, field7={1:1}) @@ -1916,15 +1996,15 @@ class G2: NSObject, SGCManager { Bridge.log("G2: Sent full Even-compatible init sequence") // Start heartbeats after auth - self.startHeartbeats() + startHeartbeats() Task { await self.reconnectionManager.stop() } Bridge.log("G2: Auth sequence complete, glasses ready") // Set device_name so DeviceManager can save it for reconnection - if let peripheralName = self.rightPeripheral?.name - ?? self.leftPeripheral?.name, - let serialNumber = self.deviceNameToSerialNumber[peripheralName] + if let peripheralName = rightPeripheral?.name + ?? leftPeripheral?.name, + let serialNumber = deviceNameToSerialNumber[peripheralName] { DeviceStore.shared.apply("bluetooth", "device_name", serialNumber) Bridge.log("G2: Set device_name to \(serialNumber)") @@ -1932,29 +2012,29 @@ class G2: NSObject, SGCManager { // Set bluetooth name and device model for Device Info page let btName = - self.rightPeripheral?.name - ?? self.leftPeripheral?.name ?? "" + rightPeripheral?.name + ?? leftPeripheral?.name ?? "" DeviceStore.shared.apply("glasses", "bluetoothName", btName) DeviceStore.shared.apply("glasses", "deviceModel", DeviceTypes.G2) - self.setFullyConnected() + setFullyConnected() // connnect a controller if we have one: - self.connectController() + connectController() // Query version + battery info from glasses - self.requestDeviceInfo() + requestDeviceInfo() // send dashboard menu if we have stored items - self.sendMenuApps() + sendMenuApps() // order the calendar (Schedule) widget first on the dashboard - self.setCalendarWidgetFirst() + setCalendarWidgetFirst() // send calendar events let calendarEvents = DeviceStore.shared.get("bluetooth", "calendar_events") as? [[String: Any]] ?? [] - self.sendCalendarEvents(calendarEvents) + sendCalendarEvents(calendarEvents) } // MARK: - Heartbeats @@ -2075,8 +2155,8 @@ class G2: NSObject, SGCManager { ) } - // Protocol witness for SGCManager.sendText — G2 renders a simple string as a - // default-positioned text wall. The positioned variant is `sendTextAt`. + /// Protocol witness for SGCManager.sendText — G2 renders a simple string as a + /// default-positioned text wall. The positioned variant is `sendTextAt`. func sendText(_ text: String) async { await sendTextWall(text) } @@ -2091,6 +2171,57 @@ class G2: NSObject, SGCManager { ) } + func sendSelectableList( + _ items: [String], x: Int32, y: Int32, width: Int32, height: Int32, + borderWidth: Int32, borderColor: Int32, borderRadius: Int32, paddingLength: Int32, + itemWidth: Int32, showSelectionBorder: Bool + ) async { + let useNativeDashboard = + DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false + if useNativeDashboard && dashboardShowing > 0 { + return + } + + let normalizedItems: [String] + if items.isEmpty { + normalizedItems = [" "] + } else { + normalizedItems = items.prefix(20).map { item in + let clipped = String(item.prefix(64)) + return clipped.isEmpty ? " " : clipped + } + } + + if listContainers.firstIndex(where: { + $0.matches( + x: x, y: y, width: width, height: height, borderWidth: borderWidth, + borderColor: borderColor, borderRadius: borderRadius, + paddingLength: paddingLength, itemWidth: itemWidth, + showSelectionBorder: showSelectionBorder, items: normalizedItems + ) + }) != nil, textContainers.isEmpty, imageContainers.isEmpty, pageCreated { + return + } + + textContainers.removeAll() + imageContainers.removeAll() + listContainers.removeAll() + let id = listContainerIDPool[0] + listContainers.append( + ListContainer( + id: id, x: x, y: y, width: width, height: height, borderWidth: borderWidth, + borderColor: borderColor, borderRadius: borderRadius, + paddingLength: paddingLength, itemWidth: itemWidth, + showSelectionBorder: showSelectionBorder, items: normalizedItems + ) + ) + + if pageCreated { + await rebuildPage() + } + await rebuildState() + } + func sendTextAt( _ text: String, x: Int32? = nil, y: Int32? = nil, width: Int32? = nil, height: Int32? = nil, borderWidth: Int32? = nil, borderColor: Int32? = nil, borderRadius: Int32? = nil, @@ -2105,6 +2236,11 @@ class G2: NSObject, SGCManager { return } + let hadListContainers = !listContainers.isEmpty + if hadListContainers { + listContainers.removeAll() + } + let rx = x ?? G2.defaultTextContainer.x let ry = y ?? G2.defaultTextContainer.y let rw = width ?? G2.defaultTextContainer.width @@ -2121,7 +2257,8 @@ class G2: NSObject, SGCManager { if let i = textContainers.firstIndex(where: { $0.matches( x: rx, y: ry, width: rw, height: rh, borderWidth: borderWidth, - borderColor: borderColor, borderRadius: borderRadius, paddingLength: paddingLength) + borderColor: borderColor, borderRadius: borderRadius, paddingLength: paddingLength + ) }) { textContainers[i].content = content textContainers[i].pendingSends = 1 + EVEN_HUB_RESEND_COUNT @@ -2138,6 +2275,13 @@ class G2: NSObject, SGCManager { ) return } + if hadListContainers { + Bridge.log( + "G2: sendText() - replacing list page with text container \(container.id)" + ) + await rebuildPage() + return + } Bridge.log( "G2: sendText() - reusing container \(container.id) for rect \(rx),\(ry) \(rw)x\(rh)" ) @@ -2146,7 +2290,8 @@ class G2: NSObject, SGCManager { let container = addTextContainer( x: rx, y: ry, width: rw, height: rh, content: content, borderWidth: borderWidth, - borderColor: borderColor, borderRadius: borderRadius, paddingLength: paddingLength) + borderColor: borderColor, borderRadius: borderRadius, paddingLength: paddingLength + ) Bridge.log( "G2: sendText() - added text container \(container.id) for rect \(rx),\(ry) \(rw)x\(rh), rebuilding page" ) @@ -2180,7 +2325,16 @@ class G2: NSObject, SGCManager { imageContainers[i].bmpData = Data() imageContainers[i].dirty = false } + let hadListContainers = !listContainers.isEmpty + if hadListContainers { + listContainers.removeAll() + } signalDisplayDirty() + if hadListContainers && pageCreated { + Task { + await rebuildPage() + } + } } /// Send a bitmap to an image container as fragmented updateImageRawData packets. @@ -2205,7 +2359,7 @@ class G2: NSObject, SGCManager { // "G2: sendImageData(\(containerName)) - \(fragmentCount) fragments, \(bmpData.count) bytes" // ) - for attempt in 1...IMG_MAX_ATTEMPTS { + for _ in 1 ... IMG_MAX_ATTEMPTS { // One session id per WHOLE image transfer (per attempt). The glasses key their // reassembly buffer on MapSessionId, so every fragment of this image must reuse the // same session id with an incrementing MapFragmentIndex; the per-fragment ACK is @@ -2222,7 +2376,7 @@ class G2: NSObject, SGCManager { // } while offset < bmpData.count { let end = min(offset + fragmentSize, bmpData.count) - let fragment = bmpData[offset.. 0 && $0.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false } let hasPendingImage = imageContainers.contains { $0.dirty && !$0.bmpData.isEmpty } - if (hasPendingText || hasPendingImage) && !(useNativeDashboard && dashboardShowing > 0) { + let hasPendingList = !listContainers.isEmpty + if (hasPendingText || hasPendingImage || hasPendingList) + && !(useNativeDashboard && dashboardShowing > 0) + { Bridge.log("G2: reconcileDisplay() - page down with pending content, rebuilding once") await rebuildState() } @@ -2406,7 +2568,7 @@ class G2: NSObject, SGCManager { // keeps being re-dirtied mid-send can't spin this pass forever (next tick picks it up). var guardCount = 0 while pageCreated, guardCount < imageContainerIDPool.count, - let i = imageContainers.firstIndex(where: { $0.dirty }) + let i = imageContainers.firstIndex(where: { $0.dirty }) { guardCount += 1 let container = imageContainers[i] @@ -2422,7 +2584,7 @@ class G2: NSObject, SGCManager { ) // Re-find by id: the array may have shifted (eviction) during the await. if let j = imageContainers.firstIndex(where: { $0.id == container.id }), - imageContainers[j].bmpData == sentBytes + imageContainers[j].bmpData == sentBytes { imageContainers[j].dirty = false } @@ -2443,7 +2605,8 @@ class G2: NSObject, SGCManager { let usedIDs = Set(imageContainers.map { $0.id }) let id = imageContainerIDPool.first { !usedIDs.contains($0) } ?? imageContainerIDPool[0] let container = ImgContainer( - id: id, x: x, y: y, width: width, height: height, bmpData: bmpData) + id: id, x: x, y: y, width: width, height: height, bmpData: bmpData + ) imageContainers.append(container) return container } @@ -2463,7 +2626,8 @@ class G2: NSObject, SGCManager { let container = TextContainer( id: id, x: x, y: y, width: width, height: height, content: content, borderWidth: borderWidth, borderColor: borderColor, borderRadius: borderRadius, - paddingLength: paddingLength) + paddingLength: paddingLength + ) textContainers.append(container) return container } @@ -2473,18 +2637,18 @@ class G2: NSObject, SGCManager { let msg = EvenHubProto.shutdownMessage() sendEvenHubCommand(msg) pageCreated = false - try? await Task.sleep(nanoseconds: 300_000_000)// 300ms to settle + try? await Task.sleep(nanoseconds: 300_000_000) // 300ms to settle // we will automatically rebuild state when we detect the glasses shutdown: // await rebuildState() } - // re-creates the containers and re-sends all images to the glasses: + /// re-creates the containers and re-sends all images to the glasses: private func rebuildState() async { Bridge.log("G2: rebuildState()") // recreate the containers (sets pageCreated = true; embeds text content directly): createPageWithContainers() - try? await Task.sleep(nanoseconds: 300_000_000) // 300ms to settle + try? await Task.sleep(nanoseconds: 300_000_000) // 300ms to settle // Mark every image container dirty and let the reconcile loop re-send them, one at a time. // Doing the sends here directly is what used to race a concurrent displayBitmap and clobber // imgAckBox; routing through the dirty flag keeps a single sender (see displayReconcileTask). @@ -2497,7 +2661,7 @@ class G2: NSObject, SGCManager { } signalDisplayDirty() - try? await Task.sleep(nanoseconds: 300_000_000) // 300ms to settle + try? await Task.sleep(nanoseconds: 300_000_000) // 300ms to settle restartMicIfAlreadyEnabled() } @@ -2546,7 +2710,7 @@ class G2: NSObject, SGCManager { return nil } - let srcPaddedRowSize = ((srcWidth + 1) / 2 + 3) & ~3 // 4-bit rows padded to 4 bytes + let srcPaddedRowSize = ((srcWidth + 1) / 2 + 3) & ~3 // 4-bit rows padded to 4 bytes let pixelDataOffset = headerSize let dstWidth = srcWidth * 2 @@ -2579,7 +2743,7 @@ class G2: NSObject, SGCManager { dst.appendLittleEndian(UInt32(0)) // --- Color Table (same 16-entry grayscale) --- - for i in 0..<16 { + for i in 0 ..< 16 { let val = UInt8(i * 17) dst.append(contentsOf: [val, val, val, 0]) } @@ -2587,12 +2751,12 @@ class G2: NSObject, SGCManager { // --- Pixel Data (nearest-neighbor 2x upscale) --- // BMP is bottom-up, so row 0 = bottom of image // Each dst row maps to srcRow = dstRow / 2 - for dstRow in 0.. Data? { // 4-bit: 2 pixels per byte, rows padded to 4-byte boundary - let bytesPerRow4bit = (width + 1) / 2 // ceil(width / 2) - let paddedRowSize = (bytesPerRow4bit + 3) & ~3 // pad to 4-byte boundary + let bytesPerRow4bit = (width + 1) / 2 // ceil(width / 2) + let paddedRowSize = (bytesPerRow4bit + 3) & ~3 // pad to 4-byte boundary let pixelDataSize = paddedRowSize * height // BMP file header (14 bytes) + DIB header (40 bytes) + color table (16 * 4 = 64 bytes) @@ -2700,46 +2864,46 @@ class G2: NSObject, SGCManager { var bmp = Data(capacity: fileSize) // --- BMP File Header (14 bytes) --- - bmp.append(contentsOf: [0x42, 0x4D]) // "BM" signature - bmp.appendLittleEndian(UInt32(fileSize)) // File size - bmp.appendLittleEndian(UInt16(0)) // Reserved1 - bmp.appendLittleEndian(UInt16(0)) // Reserved2 - bmp.appendLittleEndian(UInt32(headerSize)) // Pixel data offset + bmp.append(contentsOf: [0x42, 0x4D]) // "BM" signature + bmp.appendLittleEndian(UInt32(fileSize)) // File size + bmp.appendLittleEndian(UInt16(0)) // Reserved1 + bmp.appendLittleEndian(UInt16(0)) // Reserved2 + bmp.appendLittleEndian(UInt32(headerSize)) // Pixel data offset // --- DIB Header (BITMAPINFOHEADER, 40 bytes) --- - bmp.appendLittleEndian(UInt32(40)) // DIB header size - bmp.appendLittleEndian(Int32(width)) // Width - bmp.appendLittleEndian(Int32(height)) // Height (positive = bottom-up) - bmp.appendLittleEndian(UInt16(1)) // Color planes - bmp.appendLittleEndian(UInt16(4)) // Bits per pixel (4-bit) - bmp.appendLittleEndian(UInt32(0)) // Compression (none) - bmp.appendLittleEndian(UInt32(pixelDataSize)) // Image size - bmp.appendLittleEndian(Int32(2835)) // X pixels/meter (~72 DPI) - bmp.appendLittleEndian(Int32(2835)) // Y pixels/meter - bmp.appendLittleEndian(UInt32(16)) // Colors used - bmp.appendLittleEndian(UInt32(0)) // Important colors (0 = all) + bmp.appendLittleEndian(UInt32(40)) // DIB header size + bmp.appendLittleEndian(Int32(width)) // Width + bmp.appendLittleEndian(Int32(height)) // Height (positive = bottom-up) + bmp.appendLittleEndian(UInt16(1)) // Color planes + bmp.appendLittleEndian(UInt16(4)) // Bits per pixel (4-bit) + bmp.appendLittleEndian(UInt32(0)) // Compression (none) + bmp.appendLittleEndian(UInt32(pixelDataSize)) // Image size + bmp.appendLittleEndian(Int32(2835)) // X pixels/meter (~72 DPI) + bmp.appendLittleEndian(Int32(2835)) // Y pixels/meter + bmp.appendLittleEndian(UInt32(16)) // Colors used + bmp.appendLittleEndian(UInt32(0)) // Important colors (0 = all) // --- Color Table (16 entries, 4 bytes each: B, G, R, 0) --- - for i in 0..<16 { - let val = UInt8(i * 17) // 0, 17, 34, ... 255 (evenly spaced grayscale) - bmp.append(contentsOf: [val, val, val, 0]) // B, G, R, Reserved + for i in 0 ..< 16 { + let val = UInt8(i * 17) // 0, 17, 34, ... 255 (evenly spaced grayscale) + bmp.append(contentsOf: [val, val, val, 0]) // B, G, R, Reserved } // --- Pixel Data (bottom-up rows, 4-bit packed) --- let rowBytes = [UInt8](repeating: 0, count: paddedRowSize) - for row in 0..> 4 // divide by 16 + let index4 = gray8 >> 4 // divide by 16 let bytePos = col / 2 if col % 2 == 0 { @@ -2771,7 +2935,7 @@ class G2: NSObject, SGCManager { let msg = EvenHubProto.shutdownMessage() sendEvenHubCommand(msg) pageCreated = false - evenHubMicActive = false // dashboard takes EvenHub focus; firmware kills the mic + evenHubMicActive = false // dashboard takes EvenHub focus; firmware kills the mic currentBitmapBase64 = "" DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in guard let self = self else { return } @@ -2793,20 +2957,20 @@ class G2: NSObject, SGCManager { func sendDashboardDisplaySettings() { var dashDisplayW = ProtobufWriter() - dashDisplayW.writeInt32Field(1, 4) // displayMode - dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount - dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder - dashDisplayW.writeInt32Field(4, 4) // widgetDisplayCount + dashDisplayW.writeInt32Field(1, 4) // displayMode + dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount + dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder + dashDisplayW.writeInt32Field(4, 4) // widgetDisplayCount // WidgetType: 1=News, 2=Stock, 3=Schedule, 4=Quicklist, 5=Health dashDisplayW.writeMessageField(5, Data([3, 1, 2, 4, 5])) - dashDisplayW.writeInt32Field(6, dashboardHalfDayFormat()) // halfDayFormat - dashDisplayW.writeInt32Field(7, dashboardTemperatureUnit()) // temperatureUnit + dashDisplayW.writeInt32Field(6, dashboardHalfDayFormat()) // halfDayFormat + dashDisplayW.writeInt32Field(7, dashboardTemperatureUnit()) // temperatureUnit var dashRecvW = ProtobufWriter() dashRecvW.writeMessageField(2, dashDisplayW.data) var dashPkgW = ProtobufWriter() - dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive + dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive dashPkgW.writeInt32Field(2, sendManager.nextMagicRandom()) dashPkgW.writeMessageField(4, dashRecvW.data) sendDashboardCommand(dashPkgW.data) @@ -2873,8 +3037,8 @@ class G2: NSObject, SGCManager { let total = Int32(events.count) for (i, ev) in events.enumerated() { guard let title = ev["title"] as? String, - let time = ev["time"] as? String, - let endTs = ev["endDate"] as? Double + let time = ev["time"] as? String, + let endTs = ev["endDate"] as? Double else { continue } let location = ev["location"] as? String sendCalendarEvent( @@ -2928,6 +3092,25 @@ class G2: NSObject, SGCManager { // MARK: - Private Display Helpers private func createPageWithContainers() { + var listContainerProps: [Data] = [] + for (index, c) in listContainers.enumerated() { + let itemContainer = EvenHubProto.listItemContainerProperty( + itemCount: Int32(c.items.count), + itemWidth: c.itemWidth, + isItemSelectBorderEn: c.showSelectionBorder, + itemName: c.items + ) + listContainerProps.append( + EvenHubProto.listContainerProperty( + x: c.x, y: c.y, width: c.width, height: c.height, + borderWidth: c.borderWidth, borderColor: c.borderColor, + borderRadius: c.borderRadius, paddingLength: c.paddingLength, + containerID: c.id, containerName: c.name, itemContainer: itemContainer, + isEventCapture: index == 0 + ) + ) + } + // build the page's text containers from the live tracked list. // iterate by index not using map: var textContainerProps: [Data] = [] @@ -2938,9 +3121,10 @@ class G2: NSObject, SGCManager { borderWidth: c.borderWidth, borderColor: c.borderColor, borderRadius: c.borderRadius, paddingLength: c.paddingLength, containerID: c.id, - containerName: c.name, isEventCapture: index == 0, // the first container is the event capture container + containerName: c.name, isEventCapture: listContainers.isEmpty && index == 0, content: c.content - )) + ) + ) } // iterate all image containers, remove any entrys with duplicate id's, and ensure the ids in the imageContainerIDPool is up-to-date: @@ -2970,6 +3154,7 @@ class G2: NSObject, SGCManager { if !pageCreated { Bridge.log("G2: createPageWithContainers() - using createPageMessage (first time)") msg = EvenHubProto.createPageMessage( + listContainers: listContainerProps, textContainers: textContainerProps, imageContainers: imageContainerProps, magicRandom: sendManager.nextMagicRandom(), @@ -2978,6 +3163,7 @@ class G2: NSObject, SGCManager { } else { Bridge.log("G2: createPageWithContainers() - using rebuildPageMessage") msg = EvenHubProto.rebuildPageMessage( + listContainers: listContainerProps, textContainers: textContainerProps, imageContainers: imageContainerProps, magicRandom: sendManager.nextMagicRandom(), @@ -3248,7 +3434,7 @@ class G2: NSObject, SGCManager { let enterPayload = EvenAIProto.aiCtrl( magicRandom: sendManager.nextMagicRandom(), - status: 2 // EVEN_AI_ENTER + status: 2 // EVEN_AI_ENTER ) sendEvenAICommand(enterPayload) @@ -3262,7 +3448,7 @@ class G2: NSObject, SGCManager { try? await Task.sleep(nanoseconds: 400_000_000) triggerSkill( - 3, skillParam: 1, // NOTIFICATION, show + 3, skillParam: 1, // NOTIFICATION, show text: " ", streamEnable: 1, fTextEnd: 1 ) @@ -3295,8 +3481,8 @@ class G2: NSObject, SGCManager { /// the wearer should look around until `…{status:"complete"}`. func startCompass() { var w = ProtobufWriter() - w.writeInt32Field(1, NavigationCmd.appRequestStartUp.rawValue) // cmd - w.writeInt32Field(2, sendManager.nextMagicRandom()) // magicRandom + w.writeInt32Field(1, NavigationCmd.appRequestStartUp.rawValue) // cmd + w.writeInt32Field(2, sendManager.nextMagicRandom()) // magicRandom sendNavigationCommand(w.data) } @@ -3386,19 +3572,19 @@ class G2: NSObject, SGCManager { let widgetOrder: [UInt8] = [3, 1, 2, 4, 5] var dashDisplayW = ProtobufWriter() - dashDisplayW.writeInt32Field(1, 4) // displayMode - dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount - dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder - dashDisplayW.writeInt32Field(4, Int32(widgetOrder.count)) // widgetDisplayCount - dashDisplayW.writeMessageField(5, Data(widgetOrder)) // widgetDisplayOrder: Schedule first - dashDisplayW.writeInt32Field(6, dashboardHalfDayFormat()) // halfDayFormat - dashDisplayW.writeInt32Field(7, dashboardTemperatureUnit()) // temperatureUnit + dashDisplayW.writeInt32Field(1, 4) // displayMode + dashDisplayW.writeInt32Field(2, 3) // statusDisplayCount + dashDisplayW.writeMessageField(3, Data([1, 2, 3])) // statusDisplayOrder + dashDisplayW.writeInt32Field(4, Int32(widgetOrder.count)) // widgetDisplayCount + dashDisplayW.writeMessageField(5, Data(widgetOrder)) // widgetDisplayOrder: Schedule first + dashDisplayW.writeInt32Field(6, dashboardHalfDayFormat()) // halfDayFormat + dashDisplayW.writeInt32Field(7, dashboardTemperatureUnit()) // temperatureUnit var dashRecvW = ProtobufWriter() dashRecvW.writeMessageField(2, dashDisplayW.data) var dashPkgW = ProtobufWriter() - dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive + dashPkgW.writeInt32Field(1, 2) // Dashboard_Receive dashPkgW.writeInt32Field(2, sendManager.nextMagicRandom()) dashPkgW.writeMessageField(4, dashRecvW.data) sendDashboardCommand(dashPkgW.data) @@ -3409,7 +3595,7 @@ class G2: NSObject, SGCManager { func setDashboardMenu(_ items: [[String: Any]]) { let menuItems = items.compactMap { dict -> MenuProto.MenuItem? in guard let name = dict["name"] as? String, - let packageName = dict["packageName"] as? String + let packageName = dict["packageName"] as? String else { return nil } let running = dict["running"] as? Bool ?? false return MenuProto.MenuItem(packageName: packageName, name: name, running: running) @@ -3486,7 +3672,7 @@ class G2: NSObject, SGCManager { func sendWifiCredentials(_: String, _: String) {} func forgetWifiNetwork(_: String) {} func sendHotspotState(_: Bool) {} - func sendOtaStart(otaVersionUrl: String?) {} + func sendOtaStart(otaVersionUrl _: String?) {} func sendOtaQueryStatus() {} // MARK: - SGCManager: User Context @@ -3567,7 +3753,7 @@ class G2: NSObject, SGCManager { centralManager!.scanForPeripherals( withServices: nil, options: [ - CBCentralManagerScanOptionAllowDuplicatesKey: false + CBCentralManagerScanOptionAllowDuplicatesKey: false, ] ) return true @@ -3585,7 +3771,7 @@ class G2: NSObject, SGCManager { } guard let leftUUID = leftGlassUUID(forSN: DEVICE_SEARCH_ID), - let rightUUID = rightGlassUUID(forSN: DEVICE_SEARCH_ID) + let rightUUID = rightGlassUUID(forSN: DEVICE_SEARCH_ID) else { return false } let knownLeft = centralManager?.retrievePeripherals(withIdentifiers: [leftUUID]) @@ -3626,8 +3812,8 @@ class G2: NSObject, SGCManager { // a different service than our primary one, and retrieveConnectedPeripherals only // returns peripherals whose services match. let serviceUUIDs: [CBUUID] = [ - G2BLE.SERVICE_UUID, // EvenHub: 00002760-...-0000 - CBUUID(string: "6E400001-B5A3-F393-E0A9-E50E24DCCA9E"), // Nordic UART + G2BLE.SERVICE_UUID, // EvenHub: 00002760-...-0000 + CBUUID(string: "6E400001-B5A3-F393-E0A9-E50E24DCCA9E"), // Nordic UART ] var devices: [CBPeripheral] = [] for svc in serviceUUIDs { @@ -3655,8 +3841,8 @@ class G2: NSObject, SGCManager { // Extract XX (the numeric ID between G2_ and _L_/_R_) let pattern = "G2_(\\d+)_" guard let regex = try? NSRegularExpression(pattern: pattern), - let match = regex.firstMatch(in: name, range: NSRange(name.startIndex..., in: name)), - let range = Range(match.range(at: 1), in: name) + let match = regex.firstMatch(in: name, range: NSRange(name.startIndex..., in: name)), + let range = Range(match.range(at: 1), in: name) else { return nil } @@ -3709,7 +3895,7 @@ class G2: NSObject, SGCManager { if cmd == 10, let configData = fields[13] as? Data { var cReader = ProtobufReader(configData) let cFields = cReader.parseFields() - let voiceSwitch = cFields[1] as? Int32 ?? 0 // omitted = 0 = OFF + let voiceSwitch = cFields[1] as? Int32 ?? 0 // omitted = 0 = OFF Bridge.log( "G2: EvenAI CONFIG echo — voiceSwitch=\(voiceSwitch) (\(voiceSwitch == 1 ? "ON" : "OFF")) config=\(cFields)" ) @@ -3744,7 +3930,8 @@ class G2: NSObject, SGCManager { body: [ "heading": Int(heading), "timestamp": Int64(Date().timeIntervalSince1970 * 1000), - ]) + ] + ) case NavigationCmd.osNotifyCompassCalibrateStart.rawValue: Bridge.log("G2: compass calibration started — wearer should look around") @@ -3764,7 +3951,6 @@ class G2: NSObject, SGCManager { var reader = ProtobufReader(payload) let fields = reader.parseFields() - let payloadStr = "\(payload.map { String(format: "%02X", $0) }.joined())" if payloadStr.contains("080C7A02100C") { // heartbeat response @@ -3780,7 +3966,6 @@ class G2: NSObject, SGCManager { return } - if cmdValue == EvenHubResponseCmd.osNotifyEventToApp.rawValue { // Touch/gesture event from glasses guard let devEventData = fields[13] as? Data else { return } @@ -3821,7 +4006,6 @@ class G2: NSObject, SGCManager { } } } else { - // NOTE: the per-fragment image ACK is correlated inline on the BLE callback queue in // correlateImageAck() (called from didUpdateValueFor before this is dispatched to the // main actor) so it is never delayed behind a saturated main actor. Nothing to do here. @@ -3854,7 +4038,7 @@ class G2: NSObject, SGCManager { "G2: WARN: Glasses shutdown our EvenHub page — resetting page state" ) pageCreated = false - evenHubMicActive = false // mic dies with the page + evenHubMicActive = false // mic dies with the page } } // if let errorCode = resFields[8] as? Int32 { @@ -3872,7 +4056,7 @@ class G2: NSObject, SGCManager { if cmdValue == 9 || cmdValue == 10 { Bridge.log("G2: ERROR: Glasses shutdown our EvenHub page — resetting page state") pageCreated = false - evenHubMicActive = false // mic dies with the page + evenHubMicActive = false // mic dies with the page } } } @@ -3916,8 +4100,8 @@ class G2: NSObject, SGCManager { let wireType = Int(tag & 0x07) guard wireType == 5, data.distance(from: i, to: data.endIndex) >= 4 else { break } var bits: UInt32 = 0 - for b in 0..<4 { - bits |= UInt32(data[data.index(i, offsetBy: b)]) << (8 * b) // little-endian + for b in 0 ..< 4 { + bits |= UInt32(data[data.index(i, offsetBy: b)]) << (8 * b) // little-endian } i = data.index(i, offsetBy: 4) let value = Float(bitPattern: bits) @@ -3955,8 +4139,8 @@ class G2: NSObject, SGCManager { // (IMU_Report_Data { x, y, z } as 32-bit floats). Handle and return before // the gesture-mapping path. if (sysFields[1] as? Int32) == OsEventType.imuDataReport.rawValue, - let imuData = sysFields[3] as? Data, - let imu = parseImuReportData(imuData) + let imuData = sysFields[3] as? Data, + let imu = parseImuReportData(imuData) { Bridge.log("G2: IMU data report: \(imu.x), \(imu.y), \(imu.z)") Bridge.sendAccelEvent(x: imu.x, y: imu.y, z: imu.z, timestamp: timestamp) @@ -3999,13 +4183,12 @@ class G2: NSObject, SGCManager { } if eventType == .doubleClick { - // trigger dashboard: let isHeadUp = DeviceStore.shared.get("glasses", "headUp") as? Bool ?? false let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false if useNativeDashboard { - showDashboard() + Bridge.log("G2: native dashboard double-click shortcut disabled") } else { // toggle head up: DeviceStore.shared.apply("glasses", "headUp", !isHeadUp) @@ -4040,7 +4223,7 @@ class G2: NSObject, SGCManager { // micEnabled (user intent) — recovery reads it to re-arm; clobbering it strands the mic. if eventType == .systemExit || eventType == .abnormalExit { pageCreated = false - evenHubMicActive = false // firmware killed the mic with the page + evenHubMicActive = false // firmware killed the mic with the page } return } @@ -4050,7 +4233,7 @@ class G2: NSObject, SGCManager { var textReader = ProtobufReader(textData) let textFields = textReader.parseFields() if let eventTypeRaw = textFields[3] as? Int32, - let eventType = OsEventType(rawValue: eventTypeRaw) + let eventType = OsEventType(rawValue: eventTypeRaw) { guard let gestureName = mapEventTypeToGesture(eventType) else { Bridge.log("G2: no gesture mapping for \(eventType) \(textFields)") @@ -4065,21 +4248,48 @@ class G2: NSObject, SGCManager { } // ListEvent (field 1) - interaction with list container - // if let listData = fields[1] as? Data { - // var listReader = ProtobufReader(listData) - // let listFields = listReader.parseFields() - // if let eventTypeRaw = listFields[5] as? Int32, - // let eventType = OsEventType(rawValue: eventTypeRaw) - // { - // let gestureName = mapEventTypeToGesture(eventType) - // if let gestureName = gestureName { - // Bridge.sendTouchEvent( - // deviceModel: DeviceTypes.G2, gestureName: gestureName, timestamp: timestamp - // ) - // Bridge.log("G2: ListEvent → \(gestureName)") - // } - // } - // } + if let listData = fields[1] as? Data { + var listReader = ProtobufReader(listData) + let listFields = listReader.parseFields() + let eventType: OsEventType + if let eventTypeRaw = listFields[5] as? Int32, + let parsedEventType = OsEventType(rawValue: eventTypeRaw) + { + eventType = parsedEventType + } else { + eventType = .click + } + + guard let gestureName = mapEventTypeToGesture(eventType) else { + Bridge.log("G2: no gesture mapping for \(eventType) \(listFields)") + return + } + + var extra: [String: Any] = [:] + if let containerId = listFields[1] as? Int32 { + extra["containerId"] = containerId + } + if let containerNameData = listFields[2] as? Data, + let containerName = String(data: containerNameData, encoding: .utf8), + !containerName.isEmpty + { + extra["containerName"] = containerName + } + if let selectedNameData = listFields[3] as? Data, + let selectedItemName = String(data: selectedNameData, encoding: .utf8) + { + extra["selectedItemName"] = selectedItemName + } + if let selectedItemIndex = listFields[4] as? Int32 { + extra["selectedItemIndex"] = selectedItemIndex + } + + Bridge.sendTouchEvent( + deviceModel: DeviceTypes.G2, gestureName: gestureName, timestamp: timestamp, + extra: extra + ) + Bridge.log("G2: ListEvent → \(gestureName) \(extra)") + } } private func mapEventTypeToGesture(_ eventType: OsEventType) -> String? { @@ -4092,7 +4302,7 @@ class G2: NSObject, SGCManager { case .foregroundExit: return "foreground_exit" case .systemExit: return "system_exit" case .imuDataReport: return nil - case .abnormalExit: return nil // don't report abnormal exits as gestures + case .abnormalExit: return nil // don't report abnormal exits as gestures } } @@ -4115,7 +4325,7 @@ class G2: NSObject, SGCManager { // if the data is just a heartbeat, ignore it: if let cmdValue = fields[1] as? Int32, - cmdValue == DevCfgCommandId.baseConnHeartBeat.rawValue + cmdValue == DevCfgCommandId.baseConnHeartBeat.rawValue { return } @@ -4139,7 +4349,7 @@ class G2: NSObject, SGCManager { // Bridge.log("G2: Ring connection status: connStat=\(connStat)") // Bridge.log("G2: RingConnectInfo: \(fields)") - if let ringData = fields[5] as? Data { // field 5 = ringInfo + if let ringData = fields[5] as? Data { // field 5 = ringInfo var ringReader = ProtobufReader(ringData) let ringFields = ringReader.parseFields() @@ -4166,10 +4376,10 @@ class G2: NSObject, SGCManager { // DeviceStore.shared.apply("glasses", "controllerSearching", true) // } - if let ringData = fields[5] as? Data { // field 5 = ringInfo + if let ringData = fields[5] as? Data { // field 5 = ringInfo var ringReader = ProtobufReader(ringData) let ringFields = ringReader.parseFields() - let connStatus = ringFields[4] as? Int32 ?? -1 // field 4 = connStatus + let connStatus = ringFields[4] as? Int32 ?? -1 // field 4 = connStatus // Bridge.log( // "G2: Ring connection status: connStatus?=\(connStatus))" // ) @@ -4273,13 +4483,13 @@ class G2: NSObject, SGCManager { // Software versions if let leftVer = fields[5] as? Data, - let leftVersion = String(data: leftVer, encoding: .utf8) + let leftVersion = String(data: leftVer, encoding: .utf8) { // Bridge.log("G2: Left firmware: \(leftVersion)") DeviceStore.shared.apply("glasses", "leftFirmwareVersion", leftVersion) } if let rightVer = fields[6] as? Data, - let rightVersion = String(data: rightVer, encoding: .utf8) + let rightVersion = String(data: rightVer, encoding: .utf8) { // Bridge.log("G2: Right firmware: \(rightVersion)") DeviceStore.shared.apply("glasses", "rightFirmwareVersion", rightVersion) @@ -4317,13 +4527,13 @@ class G2: NSObject, SGCManager { // AppRespondToDashboard: field1=packageId, field2=flag (0=success) if cmd == 3 { var appRespW = ProtobufWriter() - appRespW.writeInt32Field(1, packageId) // packageId - appRespW.writeInt32Field(2, 0) // flag = APP_RECEIVED_SUCCESS + appRespW.writeInt32Field(1, packageId) // packageId + appRespW.writeInt32Field(2, 0) // flag = APP_RECEIVED_SUCCESS var pkgW = ProtobufWriter() - pkgW.writeInt32Field(1, 4) // commandId = APP_RECEIVE + pkgW.writeInt32Field(1, 4) // commandId = APP_RECEIVE pkgW.writeInt32Field(2, magicRandom) - pkgW.writeMessageField(5, appRespW.data) // field5 = appRespond + pkgW.writeMessageField(5, appRespW.data) // field5 = appRespond sendDashboardCommand(pkgW.data) } } @@ -4361,7 +4571,7 @@ class G2: NSObject, SGCManager { if data == Data([0x08, 0x01, 0x1A, 0x00]) { Bridge.log("G2: dashboard toggle - dashboardShowing=\(dashboardShowing) opening=\(dashboardOpening)") if dashboardOpening { - dashboardOpening = false // open confirmed; dashboard now owns the screen + dashboardOpening = false // open confirmed; dashboard now owns the screen return } dashboardShowing = 0 @@ -4418,7 +4628,7 @@ func extractSN(from data: Data) -> String? { // where the SN string starts. // Skip "ER" prefix (2 bytes), read 14 bytes of SN - let snData = data[2..<16] + let snData = data[2 ..< 16] return String(data: snData, encoding: .ascii)? .replacingOccurrences( of: "[\\x00-\\x1F\\x7F]", with: "", options: .regularExpression @@ -4430,7 +4640,7 @@ func extractSN(from data: Data) -> String? { /// Returns "AA:BB:CC:DD:EE:FF" (big-endian, colon-separated). func extractMac(from data: Data) -> String? { guard data.count >= 22 else { return nil } - let macLE = data[16..<22] + let macLE = data[16 ..< 22] return macLE.reversed().map { String(format: "%02X", $0) }.joined(separator: ":") } @@ -4454,13 +4664,13 @@ extension G2: CBCentralManagerDelegate { ) { guard let name = peripheral.name ?? advertisementData[CBAdvertisementDataLocalNameKey] - as? String + as? String else { return } // G2 glasses have "Even" prefix and "G2" in name, with _L_ or _R_ for side guard name.contains("G2") else { return } guard let mfgData = advertisementData[CBAdvertisementDataManufacturerDataKey] as? Data, - mfgData.count >= 16 + mfgData.count >= 16 else { return } DispatchQueue.main.async { [weak self] in @@ -4743,22 +4953,23 @@ extension G2: CBPeripheralDelegate { // Strip the 2-byte CRC trailer on the (last == only) packet. let payloadEnd = 8 + payloadLen - 2 guard payloadEnd >= 8, payloadEnd <= rawData.count else { return } - let payload = rawData.subdata(in: (rawData.startIndex + 8)..<(rawData.startIndex + payloadEnd)) + let payload = rawData.subdata(in: (rawData.startIndex + 8) ..< (rawData.startIndex + payloadEnd)) var reader = ProtobufReader(payload) let fields = reader.parseFields() - guard let resData = fields[6] as? Data else { return } // field 6 = ImgResCmd + guard let resData = fields[6] as? Data else { return } // field 6 = ImgResCmd var resReader = ProtobufReader(resData) let resFields = resReader.parseFields() guard let errorCode = resFields[8] as? Int32, - let ackSession = resFields[3] as? Int32 + let ackSession = resFields[3] as? Int32 else { return } let ackFragment = (resFields[6] as? Int32) ?? 0 Bridge.log( "G2: img_res: session=\(ackSession) fragment=\(ackFragment) errorCode=\(errorCode) success=\(errorCode == 4)" ) completeImageAck( - session: Int(ackSession), fragmentIndex: ackFragment, success: errorCode == 4) + session: Int(ackSession), fragmentIndex: ackFragment, success: errorCode == 4 + ) } nonisolated func peripheral( diff --git a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/SGCManager.swift b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/SGCManager.swift index 6774bdf64d..54ec254ebe 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/sgcs/SGCManager.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/sgcs/SGCManager.swift @@ -62,6 +62,11 @@ protocol SGCManager { _ text: String, x: Int32, y: Int32, width: Int32, height: Int32, borderWidth: Int32, borderRadius: Int32 ) async + func sendSelectableList( + _ items: [String], x: Int32, y: Int32, width: Int32, height: Int32, + borderWidth: Int32, borderColor: Int32, borderRadius: Int32, paddingLength: Int32, + itemWidth: Int32, showSelectionBorder: Bool + ) async func showDashboard() func setDashboardPosition(_ height: Int, _ depth: Int) /// Default implementation sends both via [setDashboardPosition]; Nex overrides to one protobuf. @@ -156,6 +161,14 @@ extension SGCManager { borderWidth _: Int32, borderRadius _: Int32 ) async {} + func sendSelectableList( + _ items: [String], x _: Int32, y _: Int32, width _: Int32, height _: Int32, + borderWidth _: Int32, borderColor _: Int32, borderRadius _: Int32, paddingLength _: Int32, + itemWidth _: Int32, showSelectionBorder _: Bool + ) async { + await sendTextWall(items.joined(separator: "\n")) + } + // MARK: - Video recording (default: ignore custom settings, use saved defaults) func startVideoRecording( diff --git a/mobile/modules/island/src/services/LocalMiniappRuntime.ts b/mobile/modules/island/src/services/LocalMiniappRuntime.ts index 849e01fe19..7d95e37694 100644 --- a/mobile/modules/island/src/services/LocalMiniappRuntime.ts +++ b/mobile/modules/island/src/services/LocalMiniappRuntime.ts @@ -203,6 +203,37 @@ function normalizeCameraRoiPosition(value: unknown): CameraRoiPosition { return "center" } +function normalizeTouchEventData(data: unknown): unknown { + if (!data || typeof data !== "object") return data + const record = data as Record + if (typeof record.kind === "string" && record.kind.length > 0) return data + + const gestureName = typeof record.gestureName === "string" ? record.gestureName : "" + const kind = normalizeTouchGestureKind(gestureName) + if (!kind) return data + return {...record, kind} +} + +function normalizeTouchGestureKind(gestureName: string): string | null { + switch (gestureName) { + case "click": + case "tap": + case "single_tap": + return "click" + case "double_click": + case "double_tap": + return "double_click" + case "scroll_top": + case "swipe_up": + return "scroll_top" + case "scroll_bottom": + case "swipe_down": + return "scroll_bottom" + default: + return gestureName || null + } +} + const ALL_CANONICAL_PERMISSIONS = ["location", "microphone", "camera", "notifications", "calendar"] as const /** @@ -2697,6 +2728,8 @@ class LocalMiniappRuntime { public forwardEvent(streamType: string, data: unknown): void { // Translate cloud event names to miniapp protocol stream types const normalizedStream = this.normalizeStreamType(streamType) + const eventData = + normalizedStream === MiniappStreamType.TOUCH_EVENT ? normalizeTouchEventData(data) : data // Collect all subscribers: exact match, plus wildcard matches for streams // that carry a language tag. A miniapp subscribed to "transcription:auto" @@ -2747,7 +2780,7 @@ class LocalMiniappRuntime { // The bare `touch_event` stream above still catches `onTouch(handler)`. let perGestureStream: string | null = null if (normalizedStream === MiniappStreamType.TOUCH_EVENT) { - const kind = (data as {kind?: string} | null)?.kind + const kind = (eventData as {kind?: string} | null)?.kind if (typeof kind === "string" && kind.length > 0) { perGestureStream = `${MiniappStreamType.TOUCH_EVENT}:${kind}` } @@ -2774,7 +2807,7 @@ class LocalMiniappRuntime { this.sendToMiniapp(packageName, { type: MiniappResponseType.EVENT, streamType: normalizedStream, - data, + data: eventData, }) } @@ -2787,7 +2820,7 @@ class LocalMiniappRuntime { this.sendToMiniapp(packageName, { type: MiniappResponseType.EVENT, streamType: perGestureStream, - data, + data: eventData, }) } } diff --git a/mobile/modules/miniapp/src/index.ts b/mobile/modules/miniapp/src/index.ts index cbcf84a889..024771126e 100644 --- a/mobile/modules/miniapp/src/index.ts +++ b/mobile/modules/miniapp/src/index.ts @@ -64,6 +64,8 @@ export type { Layout, LayoutType, ReferenceCard, + SelectableList, + SelectableListOptions, TextWall, ViewType, } from "./modules/display" diff --git a/mobile/modules/miniapp/src/modules/display.ts b/mobile/modules/miniapp/src/modules/display.ts index c33aa8d757..36dc6040cc 100644 --- a/mobile/modules/miniapp/src/modules/display.ts +++ b/mobile/modules/miniapp/src/modules/display.ts @@ -27,6 +27,7 @@ export type LayoutType = | "dashboard_card" | "bitmap_view" | "positioned_text" + | "selectable_list" | "clear_view" export type DisplayBreakMode = "character" | "character-no-hyphen" | "word" | "strict-word" @@ -88,6 +89,32 @@ export interface PositionedText { borderRadius?: number } +export interface SelectableList { + layoutType: "selectable_list" + /** Plain text rows rendered by the glasses' native list widget. G2 supports up to 20 rows. */ + items: string[] + /** Top-left x of the list container on the 576×288 canvas. Omit for full-screen placement. */ + x?: number + /** Top-left y of the list container on the 576×288 canvas. */ + y?: number + /** Container width. */ + width?: number + /** Container height. */ + height?: number + /** Border stroke width (px). 0 = no border. */ + borderWidth?: number + /** Border gray level. G2 list containers use 0-15. */ + borderColor?: number + /** Border corner radius (px). */ + borderRadius?: number + /** Uniform inner padding (px). */ + paddingLength?: number + /** Row width. 0 lets firmware auto-fill. */ + itemWidth?: number + /** Whether the firmware draws a border around the current row. */ + showSelectionBorder?: boolean +} + export interface ClearView { layoutType: "clear_view" } @@ -99,6 +126,7 @@ export type Layout = | DashboardCard | BitmapView | PositionedText + | SelectableList | ClearView export interface DisplayOptions { @@ -133,6 +161,29 @@ export interface TextAtOptions extends DisplayOptions { borderRadius?: number } +export interface SelectableListOptions extends DisplayOptions { + /** Top-left x of the list container on the 576×288 canvas. */ + x?: number + /** Top-left y of the list container on the 576×288 canvas. */ + y?: number + /** Container width. */ + width?: number + /** Container height. */ + height?: number + /** Border stroke width (px). 0 = no border. */ + borderWidth?: number + /** Border gray level. G2 list containers use 0-15. */ + borderColor?: number + /** Border corner radius (px). */ + borderRadius?: number + /** Uniform inner padding (px). */ + paddingLength?: number + /** Row width. 0 lets firmware auto-fill. */ + itemWidth?: number + /** Whether the firmware draws a border around the current row. */ + showSelectionBorder?: boolean +} + export class DisplayManager { constructor(private readonly session: MiniappSession) {} @@ -199,6 +250,44 @@ export class DisplayManager { ) } + /** + * Show a native selectable list. On G2, the glasses firmware owns scroll + * movement and selection highlight; miniapps receive selection clicks through + * `session.input.onTouch(...)` with optional selected item fields. + */ + showSelectableList(items: string[], options: SelectableListOptions = {}): void { + const { + x, + y, + width, + height, + borderWidth, + borderColor, + borderRadius, + paddingLength, + itemWidth, + showSelectionBorder, + ...display + } = options + this.send( + { + layoutType: "selectable_list", + items, + x, + y, + width, + height, + borderWidth, + borderColor, + borderRadius, + paddingLength, + itemWidth, + showSelectionBorder, + }, + display, + ) + } + /** Clear the specified view. */ clear(view: ViewType = "main"): void { this.send({layoutType: "clear_view"}, {view}) diff --git a/mobile/modules/miniapp/src/modules/events.ts b/mobile/modules/miniapp/src/modules/events.ts index 06e539ebb8..ae75849461 100644 --- a/mobile/modules/miniapp/src/modules/events.ts +++ b/mobile/modules/miniapp/src/modules/events.ts @@ -149,6 +149,14 @@ export interface VadData { export interface TouchData { kind: "click" | "double_click" | "scroll_top" | "scroll_bottom" | string + gestureName?: string + deviceModel?: string + timestamp?: number + source?: number + containerId?: number + containerName?: string + selectedItemIndex?: number + selectedItemName?: string } export interface AudioChunkData { diff --git a/mobile/modules/miniapp/src/session.test.ts b/mobile/modules/miniapp/src/session.test.ts index 8fd1e06c60..0f8f8c0dd5 100644 --- a/mobile/modules/miniapp/src/session.test.ts +++ b/mobile/modules/miniapp/src/session.test.ts @@ -112,6 +112,55 @@ describe("MiniappSession queue-before-ACK", () => { session.display.showTextWall("hello post-ack") expect(transport.sent.length).toBe(before + 1) }) + + test("showSelectableList sends native list layout options", async () => { + const transport = new FakeTransport() + const session = new MiniappSession({transport}) + const connectPromise = session.connect() + transport.deliverFromPhone({ + type: MiniappResponseType.CONNECT_ACK, + userId: "u", + packageName: "com.test.list", + capabilities: null, + }) + await connectPromise + + session.display.showSelectableList(["One", "Two"], { + x: 12, + y: 24, + width: 320, + height: 160, + borderWidth: 2, + borderColor: 11, + borderRadius: 8, + paddingLength: 6, + itemWidth: 300, + showSelectionBorder: false, + durationMs: 5000, + }) + + const outbound = parseEnvelope(transport.sent[transport.sent.length - 1]!) + expect(outbound).not.toBeNull() + expect(outbound!.payload).toEqual({ + type: MiniappRequestType.DISPLAY, + view: "main", + durationMs: 5000, + layout: { + layoutType: "selectable_list", + items: ["One", "Two"], + x: 12, + y: 24, + width: 320, + height: 160, + borderWidth: 2, + borderColor: 11, + borderRadius: 8, + paddingLength: 6, + itemWidth: 300, + showSelectionBorder: false, + }, + }) + }) }) describe("MiniappSession auto-PONG", () => { From 5cf68284930494e05b562575614c6a0e8ef318e6 Mon Sep 17 00:00:00 2001 From: PhilippeFerreiraDeSousa Date: Thu, 25 Jun 2026 09:53:27 -0700 Subject: [PATCH 2/4] Add Dev HUD miniapp --- bun.lock | 21 + miniapps/dev-hud/build.ts | 47 ++ miniapps/dev-hud/icon.png | Bin 0 -> 4342 bytes miniapps/dev-hud/miniapp.json | 24 + miniapps/dev-hud/package.json | 27 + miniapps/dev-hud/scripts/dev.ts | 71 ++ miniapps/dev-hud/sidecar/index.ts | 969 +++++++++++++++++++++++ miniapps/dev-hud/src/background/index.ts | 806 +++++++++++++++++++ miniapps/dev-hud/src/shared/channels.ts | 22 + miniapps/dev-hud/src/shared/types.ts | 157 ++++ miniapps/dev-hud/src/ui/App.tsx | 698 ++++++++++++++++ miniapps/dev-hud/src/ui/env.d.ts | 1 + miniapps/dev-hud/src/ui/index.css | 40 + miniapps/dev-hud/src/ui/index.html | 13 + miniapps/dev-hud/src/ui/main.tsx | 17 + miniapps/dev-hud/tsconfig.json | 15 + 16 files changed, 2928 insertions(+) create mode 100644 miniapps/dev-hud/build.ts create mode 100644 miniapps/dev-hud/icon.png create mode 100644 miniapps/dev-hud/miniapp.json create mode 100644 miniapps/dev-hud/package.json create mode 100644 miniapps/dev-hud/scripts/dev.ts create mode 100644 miniapps/dev-hud/sidecar/index.ts create mode 100644 miniapps/dev-hud/src/background/index.ts create mode 100644 miniapps/dev-hud/src/shared/channels.ts create mode 100644 miniapps/dev-hud/src/shared/types.ts create mode 100644 miniapps/dev-hud/src/ui/App.tsx create mode 100644 miniapps/dev-hud/src/ui/env.d.ts create mode 100644 miniapps/dev-hud/src/ui/index.css create mode 100644 miniapps/dev-hud/src/ui/index.html create mode 100644 miniapps/dev-hud/src/ui/main.tsx create mode 100644 miniapps/dev-hud/tsconfig.json diff --git a/bun.lock b/bun.lock index 6b0fca29fc..60f62641e1 100644 --- a/bun.lock +++ b/bun.lock @@ -108,6 +108,23 @@ "bun-plugin-tailwind": "^0.1.2", }, }, + "miniapps/dev-hud": { + "name": "dev-hud", + "dependencies": { + "@mentra/miniapp": "workspace:*", + "lucide-react": "^0.511.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "tailwindcss": "^4.0.8", + }, + "devDependencies": { + "@mentra/miniapp-cli": "workspace:*", + "@types/bun": "^1.3.14", + "@types/react": "^19.1.6", + "@types/react-dom": "^19.1.6", + "bun-plugin-tailwind": "^0.1.2", + }, + }, "miniapps/everything": { "name": "local-everything", "dependencies": { @@ -731,6 +748,8 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/d3-voronoi": ["@types/d3-voronoi@1.1.12", "", {}, "sha512-DauBl25PKZZ0WVJr42a6CNvI6efsdzofl9sajqZr2Gf5Gu733WkDdUGiPkUHXiUvYGzNNlFQde2wdZdfQPG+yw=="], "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, ""], @@ -959,6 +978,8 @@ "detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + "dev-hud": ["dev-hud@workspace:miniapps/dev-hud"], + "devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, ""], "doctrine": ["doctrine@2.1.0", "", { "dependencies": { "esutils": "^2.0.2" } }, ""], diff --git a/miniapps/dev-hud/build.ts b/miniapps/dev-hud/build.ts new file mode 100644 index 0000000000..8b38bef72d --- /dev/null +++ b/miniapps/dev-hud/build.ts @@ -0,0 +1,47 @@ +import {copyFile, rm} from "fs/promises" +import {reactSingletonPlugin} from "@mentra/miniapp-cli/build-helpers" + +const distDir = "./dist" + +await rm(distDir, {recursive: true, force: true}) + +const define: Record = {} +for (const [key, value] of Object.entries(process.env)) { + if (key.startsWith("MENTRA_PUBLIC_") && typeof value === "string") { + define[`process.env.${key}`] = JSON.stringify(value) + } +} + +const backgroundResult = await Bun.build({ + entrypoints: ["./src/background/index.ts"], + outdir: `${distDir}/background`, + target: "browser", + format: "iife", + minify: false, + define, +}) +if (!backgroundResult.success) { + console.error("Background build failed:") + for (const log of backgroundResult.logs) console.error(log) + process.exit(1) +} + +const tailwind = (await import("bun-plugin-tailwind")).default +const uiResult = await Bun.build({ + entrypoints: ["./src/ui/index.html"], + outdir: `${distDir}/ui`, + target: "browser", + plugins: [tailwind, reactSingletonPlugin(import.meta.url)], + minify: true, + define, +}) +if (!uiResult.success) { + console.error("UI build failed:") + for (const log of uiResult.logs) console.error(log) + process.exit(1) +} + +await copyFile("./miniapp.json", `${distDir}/miniapp.json`) +await copyFile("./icon.png", `${distDir}/icon.png`) + +console.log("built dev-hud -> dist/background/index.js + dist/ui + dist/icon.png + dist/miniapp.json") diff --git a/miniapps/dev-hud/icon.png b/miniapps/dev-hud/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..f7f84fb7b582e921f989eca6cf58477951b45b24 GIT binary patch literal 4342 zcmeI0>06Um7Qjzp*kqCAk)$;UNC5}I1e8_;0Tl&=f?^es&Apm52B!jLtevKwQnZPf-IKM6N0QzP#9}=8CL&HH*e2DKVxb~3V zk1v$%D?eQcP@bzl6{Q7We8S&nYaoYm1McVBrzcEDJww z-Dl=H=Gyc%VG+B1{Z882F~iM6*ZaD^5Ok4WHt(Q2WHqdHenheGYc@4qw<L2SIqpo#7yz8*;6CzEGB5s!|bD4aBQWvyCbL!qs zod{QwpQAn5Qdwk9-THb~aT=L>cE3TExsp6>0`fD_Oq6_a6xd5Ol3!BbLeB^zT%!ML zFfiBU)HMrP#b<&2TYhjNL(YGhiT1SV!rH+ncjIxvf3b>lfNiRg-AU#)#t}>Fm_gvf zVOn7F{S2JHHwc*i((G$f>lq?88$&Tq5A3eeGls^gH{@ELc2M9UV(*Azyfz^Tb}!Rj ztiSqIr0ar2M2u~;gRbS1xpiOEOc1%bybc5<3xgBwA9@Ya3^vI#e>QA6@a>{UOP=LRDb3*5tqi&k2q<|(s`B}9OYm0=4rb$-88G(|wuPAj zbC&ezc=pJQlDV97#=tb=m9n-F_2VknBEgd0SkYr4Yrh^8SQDn7qQHZK8GCt-WcYG# zvZiQwbTX_W9CAq`!`DQ7PJKdxbRuSb7}AA=^oR{uX{3`)fE&$RTFy&027yo&&E%EV zbk?nu$4pS*wVt$J(J(Hv$T)YLAIJmPOVnP>X$tJB9QmE<-d9@MR4cd5XwoW?F4?X; z(EQ?wMXsZ2BvC!h5#7n_oJ#F}bfx{=>WIYQx9#oA<+i?-oG9ASYXkl4Pol8 z$Gv>U(^WnE&WOqE4pznPD(cMuC@T+{Dp5~}6_?T`KU!E<{(0_!?Stb%WBljyj7M^XR zows<7KPoZ@hP1h?ywK{VnxTr4YGY|yL(}1bmK#M1xvlsz=Uik5@+xEu-IjFQv|Dzj z#FwhPB=--+@ zpioZ@L@eSo;=(`448eVq>W3Z4cVG1{uK)hlirYzds}+yW^0&1fV9*1h41u<7pT{xt z9GyM6X=JvHp3&HoWb8zr(>6aAXmg9wVumWMoDMIRFi3)h<5P=HG--X+9bcD<&i2ZO@Rn!g{BtpEMFRBU<2r~A#+<7 zKCAO+-ymlB*+W6zaQ&39zR8jGM)(f2^y)Xi_we|vJ7apmiHM(zrtskD@JOkBZO`Db z>!&rym1ZmM#(H%3=b(@&$(urKosW#60l|e}ca>*Wnn43TA&)Cf!H4KMLb@KDPp-Ov zFt#UilQb|!R-DDnT2Afr27#w$%!LHcTGI^rB;6M*=nbp_2ND#W_h^qudhwNrjjcEE z8{p2;5N;g>K6_25`G+yAZzS*$Z4bp8lB*Wgjt}(be5e$L=|Nb1V{amZAR!LWaA2dG zHiBcLm_?gMf_WPI!gS$99X2kvIj?92S0wt;R#5CKozw6`G?O(HQqPjuBAQ1CZ4o9= zW=mL0I}Kc?;RC)%V?V^LrUzN;0?iLKj4+Vx&k@bulj?*!CGMNcsW<&WV5BjA9tGqY zvQFR>5z$ARsR++5q6eoS>=qC@Bu9Yz?H14Wyik7GaemoioU$pkRqRZ|)vqF+31Csf zQK=h|i6@9n6YPQZxt;(18+iE>g8sX;W0+~-N(uz7AO9v6TalP``vVe~A(Am!uMnfl zh&3GXDP2u)M3vy-B)8=O_X8q=-=S|OiSF090N8!`A~v=u zLr$YLSRHAq#U?*u4_1W7yks*l{{69qlRT!70!IjG7cT@eA$`Y!BREL;`jGp*CmQFB0(qJ?PRVVszObpa_pf$o;QOK#)#EGw>+DF9@LF zA3O2T7RBgA`JEUQ{xTvr67g+4DE9xe6z&B8?AkZe!dyJbc;5S~6vVf6fgSy~Qn-f~ zBtB7iF=G^HUPOef+6K5CM1jk*0B+u!sYY=b1 { + for (const child of children) { + try { + child.kill() + } catch { + /* ignore */ + } + } +} + +process.on("SIGINT", () => { + cleanup() + process.exit(130) +}) +process.on("SIGTERM", () => { + cleanup() + process.exit(143) +}) + +const exitCode = await Promise.race([sidecar.exited, miniapp.exited]) +cleanup() +process.exit(exitCode ?? 0) + +function spawn(name: string, command: string[], env: Record): Bun.Subprocess { + console.log(`[dev-hud] ${name}: ${command.join(" ")}`) + const child = Bun.spawn(command, { + cwd: root, + env: {...process.env, ...env}, + stdout: "inherit", + stderr: "inherit", + stdin: "inherit", + }) + children.push(child) + return child +} + +function localIpAddress(): string | null { + const nets = networkInterfaces() + for (const name of ["en0", "en1", "bridge100"]) { + const found = nets[name]?.find((entry) => entry.family === "IPv4" && !entry.internal) + if (found) return found.address + } + for (const entries of Object.values(nets)) { + const found = entries?.find((entry) => entry.family === "IPv4" && !entry.internal) + if (found) return found.address + } + return null +} diff --git a/miniapps/dev-hud/sidecar/index.ts b/miniapps/dev-hud/sidecar/index.ts new file mode 100644 index 0000000000..10cb239926 --- /dev/null +++ b/miniapps/dev-hud/sidecar/index.ts @@ -0,0 +1,969 @@ +import {existsSync, mkdirSync, readFileSync, renameSync, writeFileSync} from "fs" +import {homedir, networkInterfaces} from "os" +import {dirname, join} from "path" +import {Database} from "bun:sqlite" + +import type { + CheckState, + CodexTaskSummary, + CodexThreadState, + CodexThreadSummary, + CodexTurnSummary, + DevHudNotification, + DevHudStatus, + NotificationKind, + NotificationSeverity, + PullRequestSummary, + PullRequestCheckItem, + PullRequestReviewNote, + ReviewState, + SourceStatus, +} from "../src/shared/types" + +const PORT = Number(process.env.DEV_HUD_PORT ?? 3147) +const CODEX_HOME = process.env.CODEX_HOME ?? join(homedir(), ".codex") +const GITHUB_LIMIT = Number(process.env.DEV_HUD_GITHUB_LIMIT ?? 8) +const THREAD_LIMIT = Number(process.env.DEV_HUD_CODEX_THREAD_LIMIT ?? 8) +const CODEX_TURN_LIMIT = Number(process.env.DEV_HUD_CODEX_TURN_LIMIT ?? 24) +const NOTIFICATION_STORE_PATH = process.env.DEV_HUD_NOTIFICATION_STORE_PATH ?? join(CODEX_HOME, "dev-hud-notifications.json") +const PUBLIC_ENDPOINT = process.env.DEV_HUD_PUBLIC_ENDPOINT ?? `http://${localIpAddress() ?? "127.0.0.1"}:${PORT}` +const RECENT_CODEX_FINISH_WINDOW_MS = Number(process.env.DEV_HUD_CODEX_FINISH_WINDOW_MS ?? 5 * 60 * 1000) + +const CORS_HEADERS = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET,OPTIONS", + "Access-Control-Allow-Headers": "Content-Type,Accept", +} + +const MAX_NOTIFICATIONS = 80 + +interface DiffBaseline { + prs: Map + tasks: Map + threads: Map + turns: Map +} + +let previousBaseline: DiffBaseline | null = null +let notificationEvents: DevHudNotification[] = loadNotificationEvents() + +Bun.serve({ + hostname: "0.0.0.0", + port: PORT, + async fetch(request) { + const url = new URL(request.url) + if (request.method === "OPTIONS") return new Response(null, {headers: CORS_HEADERS}) + if (url.pathname === "/health") return json({ok: true, endpoint: PUBLIC_ENDPOINT}) + if (url.pathname === "/api/status") return json(await collectStatus()) + return json({ + name: "Dev HUD sidecar", + endpoints: ["/health", "/api/status"], + endpoint: PUBLIC_ENDPOINT, + }) + }, +}) + +console.log(`[dev-hud] sidecar listening on ${PUBLIC_ENDPOINT}`) +console.log(`[dev-hud] codex home: ${CODEX_HOME}`) +console.log(`[dev-hud] notifications: ${NOTIFICATION_STORE_PATH}`) + +async function collectStatus(): Promise { + const [github, codex] = await Promise.all([collectGitHub(), collectCodex()]) + const events = + github.status.state === "ok" && codex.status.state === "ok" + ? diffStatus(github.openPrs, codex.tasks, codex.threads, codex.turns) + : notificationEvents + return { + generatedAt: Date.now(), + endpoint: PUBLIC_ENDPOINT, + github, + codex, + notifications: { + status: sourceStatus("ok", `${notificationEvents.length} event${notificationEvents.length === 1 ? "" : "s"}`), + events, + }, + } +} + +async function collectGitHub(): Promise { + try { + const searchRaw = await runProcess("gh", [ + "search", + "prs", + "--author", + "@me", + "--state", + "open", + "--archived=false", + "--sort", + "updated", + "--order", + "desc", + "--limit", + String(GITHUB_LIMIT), + "--json", + "repository,number,title,url,isDraft,state,updatedAt", + ]) + const searchResults = JSON.parse(searchRaw) as SearchPullRequest[] + const enriched = await Promise.all(searchResults.map((pr) => enrichPullRequest(pr).catch(() => fromSearchPullRequest(pr)))) + return { + status: sourceStatus("ok", `${enriched.length} open PR${enriched.length === 1 ? "" : "s"}`), + openPrs: enriched, + } + } catch (err) { + return { + status: sourceStatus("error", errorMessage(err)), + openPrs: [], + } + } +} + +async function enrichPullRequest(pr: SearchPullRequest): Promise { + const repo = pr.repository.nameWithOwner + const raw = await runProcess("gh", [ + "pr", + "view", + String(pr.number), + "--repo", + repo, + "--json", + "number,title,url,isDraft,reviewDecision,mergeStateStatus,statusCheckRollup,headRefName,baseRefName,updatedAt,reviewRequests,latestReviews", + ]) + const detail = JSON.parse(raw) as PullRequestDetail + const requestedReviewers = normalizeReviewRequests(detail.reviewRequests) + const reviewComments = await collectReviewComments(repo, pr.number, detail.latestReviews) + return { + id: `${repo}#${detail.number}`, + repo, + number: detail.number, + title: sanitizeTitle(detail.title), + url: detail.url, + isDraft: Boolean(detail.isDraft), + updatedAt: detail.updatedAt ?? pr.updatedAt ?? null, + branch: detail.headRefName ?? null, + base: detail.baseRefName ?? null, + mergeState: detail.mergeStateStatus ?? null, + reviewState: normalizeReviewState(detail.reviewDecision, requestedReviewers.length), + requestedReviewers, + checks: summarizeChecks(detail.statusCheckRollup ?? []), + reviewComments, + } +} + +function fromSearchPullRequest(pr: SearchPullRequest): PullRequestSummary { + const repo = pr.repository.nameWithOwner + return { + id: `${repo}#${pr.number}`, + repo, + number: pr.number, + title: sanitizeTitle(pr.title), + url: pr.url, + isDraft: Boolean(pr.isDraft), + updatedAt: pr.updatedAt ?? null, + branch: null, + base: null, + mergeState: null, + reviewState: "unknown", + requestedReviewers: [], + checks: {state: "unknown", total: 0, success: 0, pending: 0, failure: 0, skipped: 0, items: []}, + reviewComments: [], + } +} + +async function collectCodex(): Promise { + try { + const names = readSessionNames() + const tasks = readCodexTasks(names) + const commandsByThread = new Map() + for (const task of tasks) { + if (!task.running) continue + const current = commandsByThread.get(task.threadId) ?? [] + current.push(task.command) + commandsByThread.set(task.threadId, current) + } + const threads = readCodexThreads(names, commandsByThread) + const turns = readCompletedCodexTurns(names) + const runningCount = tasks.filter((task) => task.running).length + return { + status: sourceStatus("ok", `${threads.length} threads, ${runningCount} running task${runningCount === 1 ? "" : "s"}`), + threads, + tasks, + turns, + } + } catch (err) { + return { + status: sourceStatus("error", errorMessage(err)), + threads: [], + tasks: [], + turns: [], + } + } +} + +function readCodexThreads(names: Map, commandsByThread: Map): CodexThreadSummary[] { + const statePath = join(CODEX_HOME, "state_5.sqlite") + if (!existsSync(statePath)) throw new Error(`Codex state not found at ${statePath}`) + const db = new Database(statePath, {readonly: true}) + try { + const rows = db + .query( + `select id, title, cwd, preview, git_branch, updated_at, updated_at_ms, recency_at_ms + from threads + where archived = 0 + order by recency_at_ms desc + limit ?`, + ) + .all(THREAD_LIMIT) as ThreadRow[] + return rows.map((row) => { + const updatedAt = Number(row.updated_at_ms || row.recency_at_ms || row.updated_at * 1000 || 0) + const runningCommands = commandsByThread.get(row.id) ?? [] + return { + id: row.id, + title: sanitizeTitle(names.get(row.id) ?? row.title ?? row.preview ?? "Untitled thread"), + cwd: row.cwd, + branch: row.git_branch ? sanitizeTitle(row.git_branch) : null, + updatedAt, + state: threadState(updatedAt, runningCommands.length > 0), + runningCommands, + } + }) + } finally { + db.close() + } +} + +function readCodexTasks(names: Map): CodexTaskSummary[] { + const processPath = join(CODEX_HOME, "process_manager", "chat_processes.json") + if (!existsSync(processPath)) return [] + const raw = JSON.parse(readFileSync(processPath, "utf8")) as ProcessRow[] + const tasks = raw + .filter((row) => !isDevHudSelfTask(row)) + .map((row) => { + const pid = typeof row.osPid === "number" ? row.osPid : null + const running = pid !== null && processAlive(pid) + return { + id: row.id, + threadId: row.conversationId, + title: sanitizeTitle(names.get(row.conversationId) ?? row.chatTitle ?? "Codex thread"), + cwd: row.cwd, + command: sanitizeCommand(row.command), + startedAt: Number(row.startedAtMs ?? 0), + updatedAt: Number(row.updatedAtMs ?? row.startedAtMs ?? 0), + pid, + running, + } satisfies CodexTaskSummary + }) + .filter((task) => task.running) + .sort((a, b) => b.startedAt - a.startedAt) + return tasks.slice(0, 8) +} + +function readCompletedCodexTurns(names: Map): CodexTurnSummary[] { + const logsPath = join(CODEX_HOME, "logs_2.sqlite") + if (!existsSync(logsPath)) return [] + const sinceSeconds = Math.floor((Date.now() - RECENT_CODEX_FINISH_WINDOW_MS) / 1000) + const db = new Database(logsPath, {readonly: true}) + try { + const rows = db + .query( + `select id, thread_id, ts, feedback_log_body + from logs + where ts >= ? + and thread_id is not null + and target = 'codex_core::session::turn' + and feedback_log_body like '%post sampling token usage%' + and feedback_log_body like '%needs_follow_up=false%' + order by ts desc, ts_nanos desc, id desc + limit ?`, + ) + .all(sinceSeconds, CODEX_TURN_LIMIT) as CodexTurnRow[] + + const turns: CodexTurnSummary[] = [] + const seenTurnIds = new Set() + for (const row of rows) { + const body = row.feedback_log_body ?? "" + const turnId = body.match(/\bturn_id=([0-9a-f-]+)/)?.[1] + const threadId = row.thread_id ?? body.match(/\bthread\.id=([0-9a-f-]+)/)?.[1] + if (!turnId || !threadId || seenTurnIds.has(turnId)) continue + seenTurnIds.add(turnId) + turns.push({ + id: turnId, + threadId, + title: sanitizeTitle(names.get(threadId) ?? "Codex thread"), + completedAt: Number(row.ts) * 1000, + }) + } + return turns + } finally { + db.close() + } +} + +function isDevHudSelfTask(row: ProcessRow): boolean { + if (!row.cwd.endsWith("/miniapps/dev-hud")) return false + return /(?:sidecar\/index\.ts|scripts\/dev\.ts|miniapp:dev|bun run dev)\b/.test(row.command) +} + +function readSessionNames(): Map { + const indexPath = join(CODEX_HOME, "session_index.jsonl") + const names = new Map() + if (!existsSync(indexPath)) return names + for (const line of readFileSync(indexPath, "utf8").split(/\r?\n/)) { + if (!line.trim()) continue + try { + const row = JSON.parse(line) as {id?: string; thread_name?: string} + if (row.id && row.thread_name) names.set(row.id, sanitizeTitle(row.thread_name)) + } catch { + /* ignore malformed rows */ + } + } + return names +} + +function diffStatus( + prs: PullRequestSummary[], + tasks: CodexTaskSummary[], + threads: CodexThreadSummary[], + turns: CodexTurnSummary[], +): DevHudNotification[] { + const current: DiffBaseline = { + prs: new Map(prs.map((pr) => [pr.id, pr])), + tasks: new Map(tasks.filter((task) => task.running).map((task) => [task.id, task])), + threads: new Map(threads.map((thread) => [thread.id, thread])), + turns: new Map(turns.map((turn) => [turn.id, turn])), + } + + if (!previousBaseline) { + previousBaseline = current + return notificationEvents + } + + const newEvents: DevHudNotification[] = [] + const finishedTaskThreadIds = new Set() + + for (const pr of current.prs.values()) { + const before = previousBaseline.prs.get(pr.id) + if (!before) { + newEvents.push( + notification({ + source: "github", + kind: "github_pr_opened", + severity: "info", + interrupt: false, + title: `PR #${pr.number} opened`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: null, + after: "open", + }), + ) + continue + } + + if (before.checks.state !== pr.checks.state) { + if (pr.checks.state === "failure") { + newEvents.push( + notification({ + source: "github", + kind: "github_checks_failed", + severity: "critical", + interrupt: true, + title: `PR #${pr.number} checks failed`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: before.checks.state, + after: pr.checks.state, + }), + ) + } else if (before.checks.state === "failure" && pr.checks.state === "success") { + newEvents.push( + notification({ + source: "github", + kind: "github_checks_recovered", + severity: "success", + interrupt: true, + title: `PR #${pr.number} checks recovered`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: before.checks.state, + after: pr.checks.state, + }), + ) + } else { + newEvents.push( + notification({ + source: "github", + kind: "github_checks_changed", + severity: pr.checks.state === "pending" ? "warning" : "info", + interrupt: false, + title: `PR #${pr.number} checks ${pr.checks.state}`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: before.checks.state, + after: pr.checks.state, + }), + ) + } + } + + if (before.reviewState !== pr.reviewState) { + if (pr.reviewState === "changes_requested") { + newEvents.push( + notification({ + source: "github", + kind: "github_changes_requested", + severity: "critical", + interrupt: true, + title: `Changes requested on #${pr.number}`, + message: latestReviewMessage(pr) ?? pr.title, + target: {type: "pr", id: pr.id}, + before: before.reviewState, + after: pr.reviewState, + }), + ) + } else if (pr.reviewState === "approved") { + newEvents.push( + notification({ + source: "github", + kind: "github_approved", + severity: "success", + interrupt: true, + title: `PR #${pr.number} approved`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: before.reviewState, + after: pr.reviewState, + }), + ) + } else { + newEvents.push( + notification({ + source: "github", + kind: "github_review_changed", + severity: "info", + interrupt: false, + title: `PR #${pr.number} review changed`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: before.reviewState, + after: pr.reviewState, + }), + ) + } + } + + const beforeMerge = normalizeMergeState(before.mergeState) + const afterMerge = normalizeMergeState(pr.mergeState) + if (beforeMerge !== afterMerge) { + if (isBlockedMergeState(afterMerge)) { + newEvents.push( + notification({ + source: "github", + kind: "github_merge_blocked", + severity: "warning", + interrupt: true, + title: `PR #${pr.number} merge blocked`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: beforeMerge, + after: afterMerge, + }), + ) + } else if (isBlockedMergeState(beforeMerge) && afterMerge === "CLEAN") { + newEvents.push( + notification({ + source: "github", + kind: "github_merge_recovered", + severity: "success", + interrupt: true, + title: `PR #${pr.number} mergeable again`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: beforeMerge, + after: afterMerge, + }), + ) + } + } + } + + for (const pr of previousBaseline.prs.values()) { + if (current.prs.has(pr.id)) continue + newEvents.push( + notification({ + source: "github", + kind: "github_pr_removed", + severity: "info", + interrupt: false, + title: `PR #${pr.number} left open list`, + message: pr.title, + target: {type: "pr", id: pr.id}, + before: "open", + after: "removed", + }), + ) + } + + for (const task of previousBaseline.tasks.values()) { + if (current.tasks.has(task.id)) continue + finishedTaskThreadIds.add(task.threadId) + newEvents.push( + notification({ + source: "codex", + kind: "codex_task_finished", + severity: "success", + interrupt: true, + title: "Codex task finished", + message: `${task.title}: ${task.command}`, + target: {type: "codexThread", id: task.threadId}, + before: "running", + after: "finished", + }), + ) + } + + for (const turn of current.turns.values()) { + if (previousBaseline.turns.has(turn.id) || finishedTaskThreadIds.has(turn.threadId)) continue + const thread = current.threads.get(turn.threadId) + newEvents.push( + notification({ + source: "codex", + kind: "codex_turn_finished", + severity: "success", + interrupt: true, + title: "Codex turn finished", + message: thread?.title ?? turn.title, + target: {type: "codexThread", id: turn.threadId}, + before: null, + after: "finished", + }), + ) + } + + previousBaseline = current + if (newEvents.length > 0) { + notificationEvents = [...newEvents, ...notificationEvents].slice(0, MAX_NOTIFICATIONS) + persistNotificationEvents(notificationEvents) + } + return notificationEvents +} + +function loadNotificationEvents(): DevHudNotification[] { + if (!existsSync(NOTIFICATION_STORE_PATH)) return [] + try { + const parsed = JSON.parse(readFileSync(NOTIFICATION_STORE_PATH, "utf8")) as unknown + const rawEvents = Array.isArray(parsed) + ? parsed + : isRecord(parsed) && Array.isArray(parsed.events) + ? parsed.events + : [] + return rawEvents + .filter(isStoredNotification) + .map((event) => ({ + ...event, + title: sanitizeTitle(event.title), + message: sanitizeBody(event.message), + before: event.before ? sanitizeTitle(event.before) : null, + after: event.after ? sanitizeTitle(event.after) : null, + })) + .sort((a, b) => b.timestamp - a.timestamp) + .slice(0, MAX_NOTIFICATIONS) + } catch (err) { + console.log(`DevHUD: failed to load notifications: ${errorMessage(err)}`) + return [] + } +} + +function persistNotificationEvents(events: DevHudNotification[]): void { + try { + mkdirSync(dirname(NOTIFICATION_STORE_PATH), {recursive: true}) + const tmpPath = `${NOTIFICATION_STORE_PATH}.${process.pid}.tmp` + writeFileSync(tmpPath, JSON.stringify({version: 1, events}, null, 2)) + renameSync(tmpPath, NOTIFICATION_STORE_PATH) + } catch (err) { + console.log(`DevHUD: failed to persist notifications: ${errorMessage(err)}`) + } +} + +function isStoredNotification(value: unknown): value is DevHudNotification { + if (!isRecord(value) || !isRecord(value.target)) return false + const targetType = value.target.type + return ( + typeof value.id === "string" && + typeof value.timestamp === "number" && + typeof value.source === "string" && + typeof value.severity === "string" && + typeof value.kind === "string" && + typeof value.interrupt === "boolean" && + typeof value.title === "string" && + typeof value.message === "string" && + (targetType === "pr" || targetType === "codexThread") && + typeof value.target.id === "string" && + (typeof value.before === "string" || value.before === null) && + (typeof value.after === "string" || value.after === null) + ) +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} + +function notification(input: Omit): DevHudNotification { + const timestamp = Date.now() + const idSource = [ + input.kind, + input.target.type, + input.target.id, + input.before ?? "", + input.after ?? "", + timestamp, + ].join(":") + return { + id: `${input.kind}-${hashString(idSource)}`, + timestamp, + ...input, + message: sanitizeBody(input.message), + title: sanitizeTitle(input.title), + before: input.before ? sanitizeTitle(input.before) : null, + after: input.after ? sanitizeTitle(input.after) : null, + } +} + +function latestReviewMessage(pr: PullRequestSummary): string | null { + const latest = pr.reviewComments.find((comment) => comment.body.trim().length > 0) + if (!latest) return null + return `${latest.author}: ${latest.body}` +} + +function normalizeMergeState(state: string | null): string { + return sanitizeTitle((state ?? "UNKNOWN").toUpperCase()) +} + +function isBlockedMergeState(state: string): boolean { + return state === "BLOCKED" || state === "DIRTY" || state === "UNKNOWN" +} + +function hashString(input: string): string { + let hash = 5381 + for (let i = 0; i < input.length; i += 1) { + hash = (hash * 33) ^ input.charCodeAt(i) + } + return (hash >>> 0).toString(36) +} + +async function collectReviewComments( + repo: string, + prNumber: number, + latestReviews: ReviewSummary[] | undefined, +): Promise { + const notes: PullRequestReviewNote[] = [] + for (const review of latestReviews ?? []) { + const body = sanitizeBody(review.body ?? "") + if (!body) continue + notes.push({ + id: `review-${review.id || review.submittedAt || notes.length}`, + type: "review", + author: sanitizeTitle(review.author?.login ?? "reviewer"), + body, + state: review.state ? sanitizeTitle(review.state.toLowerCase()) : null, + path: null, + line: null, + url: null, + createdAt: review.submittedAt ?? null, + }) + } + + try { + const raw = await runProcess("gh", ["api", `repos/${repo}/pulls/${prNumber}/comments`], 15_000) + const inlineComments = JSON.parse(raw) as InlineReviewComment[] + for (const comment of inlineComments.slice(-16).reverse()) { + notes.push({ + id: `inline-${comment.id}`, + type: "inline", + author: sanitizeTitle(comment.user?.login ?? "reviewer"), + body: sanitizeBody(comment.body ?? ""), + state: null, + path: comment.path ? sanitizeTitle(comment.path) : null, + line: typeof comment.line === "number" ? comment.line : typeof comment.original_line === "number" ? comment.original_line : null, + url: comment.html_url ?? null, + createdAt: comment.created_at ?? null, + }) + } + } catch { + /* Review comments are best-effort; PR state should still render. */ + } + + return notes.filter((note) => note.body.length > 0).slice(0, 20) +} + +function summarizeChecks(items: StatusCheckItem[]): PullRequestSummary["checks"] { + let success = 0 + let pending = 0 + let failure = 0 + let skipped = 0 + const checkItems: PullRequestCheckItem[] = [] + + for (const item of items) { + const state = normalizeCheckItem(item) + if (state === "success") success += 1 + else if (state === "pending") pending += 1 + else if (state === "failure") failure += 1 + else skipped += 1 + checkItems.push(checkItemSummary(item, state === "skipped" ? "unknown" : state)) + } + + const total = items.length + const state: CheckState = total === 0 ? "unknown" : failure > 0 ? "failure" : pending > 0 ? "pending" : "success" + return {state, total, success, pending, failure, skipped, items: checkItems} +} + +function checkItemSummary(item: StatusCheckItem, state: CheckState): PullRequestCheckItem { + if (item.__typename === "StatusContext") { + return { + name: sanitizeTitle(item.context ?? "status"), + workflow: null, + state, + url: item.targetUrl ?? null, + } + } + return { + name: sanitizeTitle(item.name ?? "check"), + workflow: item.workflowName ? sanitizeTitle(item.workflowName) : null, + state, + url: item.detailsUrl ?? null, + } +} + +function normalizeCheckItem(item: StatusCheckItem): "success" | "pending" | "failure" | "skipped" { + if (item.__typename === "StatusContext") { + const state = String(item.state ?? "").toUpperCase() + if (state === "SUCCESS") return "success" + if (state === "PENDING" || state === "EXPECTED") return "pending" + if (state === "ERROR" || state === "FAILURE") return "failure" + return "skipped" + } + const status = String(item.status ?? "").toUpperCase() + const conclusion = String(item.conclusion ?? "").toUpperCase() + if (status && status !== "COMPLETED") return "pending" + if (conclusion === "SUCCESS" || conclusion === "NEUTRAL") return "success" + if (conclusion === "SKIPPED") return "skipped" + if (conclusion === "FAILURE" || conclusion === "CANCELLED" || conclusion === "TIMED_OUT" || conclusion === "ACTION_REQUIRED") { + return "failure" + } + return "pending" +} + +function normalizeReviewRequests(items: ReviewRequest[] | undefined): string[] { + if (!Array.isArray(items)) return [] + return items + .map((item) => ("login" in item ? item.login : "name" in item ? item.name : null)) + .filter((value): value is string => typeof value === "string" && value.length > 0) + .map(sanitizeTitle) +} + +function normalizeReviewState(decision: string | null | undefined, requestedCount: number): ReviewState { + const normalized = String(decision ?? "").toUpperCase() + if (normalized === "APPROVED") return "approved" + if (normalized === "CHANGES_REQUESTED") return "changes_requested" + if (normalized === "REVIEW_REQUIRED") return "review_required" + if (requestedCount > 0) return "review_required" + if (!normalized) return "none" + return "unknown" +} + +function threadState(updatedAt: number, running: boolean): CodexThreadState { + if (running) return "running" + const ageMs = Date.now() - updatedAt + if (ageMs < 2 * 60 * 60 * 1000) return "done" + if (ageMs < 48 * 60 * 60 * 1000) return "recent" + return "stale" +} + +async function runProcess(command: string, args: string[], timeoutMs = 15_000): Promise { + const proc = Bun.spawn([command, ...args], { + stdout: "pipe", + stderr: "pipe", + env: process.env, + }) + const stdoutPromise = new Response(proc.stdout).text() + const stderrPromise = new Response(proc.stderr).text() + let timedOut = false + const timeout = setTimeout(() => { + timedOut = true + proc.kill() + }, timeoutMs) + + const exitCode = await proc.exited.finally(() => clearTimeout(timeout)) + const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]) + if (timedOut) throw new Error(`${command} timed out`) + if (exitCode !== 0) throw new Error(trimError(stderr) || `${command} exited ${exitCode}`) + return stdout +} + +function processAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} + +function sourceStatus(state: SourceStatus["state"], message: string | null): SourceStatus { + return {state, message, updatedAt: Date.now()} +} + +function json(body: unknown, init?: ResponseInit): Response { + return new Response(JSON.stringify(body, null, 2), { + ...init, + headers: { + "Content-Type": "application/json", + "Cache-Control": "no-store", + ...CORS_HEADERS, + ...(init?.headers ?? {}), + }, + }) +} + +function sanitizeTitle(value: string): string { + return truncate(oneLine(redact(value)), 96) +} + +function sanitizeCommand(value: string): string { + return truncate(oneLine(redact(value)), 120) +} + +function sanitizeBody(value: string): string { + return truncate(redact(value).replace(/\r\n/g, "\n").trim(), 900) +} + +function redact(value: string): string { + return value + .replace(/\b(?:gho|ghp|github_pat)_[A-Za-z0-9_]+/g, "[github-token]") + .replace(/\blin_api_[A-Za-z0-9]+/g, "[linear-token]") + .replace(/\bsk-[A-Za-z0-9_-]+/g, "[api-token]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[jwt]") + .replace(/\b([A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASS|API_KEY|ACCESS_KEY|PRIVATE_KEY)[A-Z0-9_]*)=(?:"[^"]*"|'[^']*'|[^\s]+)/gi, "$1=[redacted]") + .replace(/([?&](?:token|key|secret|password)=)[^&\s]+/gi, "$1[redacted]") + .replace(/\bBearer\s+[A-Za-z0-9._-]+/gi, "Bearer [redacted]") +} + +function oneLine(value: string): string { + return value.replace(/\s+/g, " ").trim() +} + +function truncate(value: string, max: number): string { + if (value.length <= max) return value + return `${value.slice(0, Math.max(0, max - 3)).trimEnd()}...` +} + +function trimError(value: string): string { + return truncate(oneLine(redact(value)), 220) +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? trimError(err.message) : trimError(String(err)) +} + +function localIpAddress(): string | null { + const nets = networkInterfaces() + for (const name of ["en0", "en1", "bridge100"]) { + const found = nets[name]?.find((entry) => entry.family === "IPv4" && !entry.internal) + if (found) return found.address + } + for (const entries of Object.values(nets)) { + const found = entries?.find((entry) => entry.family === "IPv4" && !entry.internal) + if (found) return found.address + } + return null +} + +interface SearchPullRequest { + repository: {nameWithOwner: string} + number: number + title: string + url: string + isDraft?: boolean + updatedAt?: string +} + +interface PullRequestDetail { + number: number + title: string + url: string + isDraft?: boolean + reviewDecision?: string + mergeStateStatus?: string + statusCheckRollup?: StatusCheckItem[] + headRefName?: string + baseRefName?: string + updatedAt?: string + reviewRequests?: ReviewRequest[] + latestReviews?: ReviewSummary[] +} + +type StatusCheckItem = + | { + __typename: "CheckRun" + name?: string + workflowName?: string + detailsUrl?: string + status?: string + conclusion?: string | null + } + | { + __typename: "StatusContext" + context?: string + targetUrl?: string + state?: string + } + +type ReviewRequest = {__typename?: string; login?: string; name?: string} + +interface ReviewSummary { + id?: string + author?: {login?: string} + body?: string + submittedAt?: string + state?: string +} + +interface InlineReviewComment { + id: number + user?: {login?: string} + body?: string + path?: string + line?: number | null + original_line?: number | null + html_url?: string + created_at?: string +} + +interface ThreadRow { + id: string + title: string + cwd: string + preview: string + git_branch: string | null + updated_at: number + updated_at_ms: number | null + recency_at_ms: number | null +} + +interface ProcessRow { + id: string + conversationId: string + chatTitle?: string | null + cwd: string + command: string + osPid?: number | null + startedAtMs?: number + updatedAtMs?: number +} + +interface CodexTurnRow { + id: number + thread_id: string | null + ts: number + feedback_log_body: string | null +} diff --git a/miniapps/dev-hud/src/background/index.ts b/miniapps/dev-hud/src/background/index.ts new file mode 100644 index 0000000000..593df97f4d --- /dev/null +++ b/miniapps/dev-hud/src/background/index.ts @@ -0,0 +1,806 @@ +import {registerMiniapp} from "@mentra/miniapp/background" +import type {MiniappSession, TouchData} from "@mentra/miniapp/background" + +import type {Channels} from "../shared/channels" +import type { + CodexTaskSummary, + CodexThreadSummary, + DevHudNotification, + DevHudSnapshot, + DevHudStatus, + DevHudView, + PullRequestSummary, +} from "../shared/types" + +type Send = (channel: C, payload: Channels[C]) => void +type On = (channel: C, cb: (payload: Channels[C]) => void) => () => void + +const SETTINGS_KEY = "dev-hud:settings" +const DEFAULT_ENDPOINT = process.env.MENTRA_PUBLIC_DEV_HUD_ENDPOINT?.trim() || "http://127.0.0.1:3147" +const POLL_MS = 10_000 +const DISPLAY_MS = 28_000 +const NOTIFICATION_DISPLAY_MS = 12_000 +const MAIN_MENU_ROWS = ["Summary", "Pull Requests", "Codex Threads", "Notifications", "Refresh"] as const +const EMPTY_PR_ROWS = ["No open PRs", "Refresh"] as const +const EMPTY_CODEX_ROWS = ["No Codex threads", "Refresh"] as const +const EMPTY_NOTIFICATION_ROWS = ["No notifications", "Refresh"] as const +const LOADING_ROWS = ["Loading status", "Refresh"] as const +const LIST_OPTIONS = { + x: 24, + y: 16, + width: 528, + height: 256, + borderWidth: 1, + borderColor: 13, + borderRadius: 8, + paddingLength: 8, + itemWidth: 500, + showSelectionBorder: true, +} as const + +type GlassScreen = "focus" | "main" | "summary" | "prList" | "codexList" | "prDetail" | "codexDetail" | "notifications" + +interface StoredSettings { + endpoint?: string + selectedView?: DevHudView +} + +class DevHudController { + private readonly unsubs: Array<() => void> = [] + private endpoint = DEFAULT_ENDPOINT + private selectedView: DevHudView = "summary" + private screen: GlassScreen = "main" + private status: DevHudStatus | null = null + private loading = false + private lastError: string | null = null + private pollTimer: ReturnType | null = null + private interruptTimer: ReturnType | null = null + private activeInterrupt: DevHudNotification | null = null + private interruptActiveUntil = 0 + private ignoreTouchUntil = 0 + private deliveredNotificationIds = new Set() + private selectedIndex = 0 + private activeRows: string[] = [] + private activePrs: PullRequestSummary[] = [] + private activeThreads: CodexThreadSummary[] = [] + private activeNotifications: DevHudNotification[] = [] + private selectedPr: PullRequestSummary | null = null + private selectedThread: CodexThreadSummary | null = null + private ui!: { + send: Send + on: On + onOpen: (cb: () => void) => () => void + } + + constructor(private readonly session: MiniappSession) {} + + async start(): Promise { + this.ui = this.session.ui as unknown as { + send: Send + on: On + onOpen: (cb: () => void) => () => void + } + await this.loadSettings() + this.screen = screenForView(this.selectedView) + this.wireUI() + this.unsubs.push(this.session.input.onTouch((event) => this.handleTouch(event))) + this.session.display.showTextWall("Dev HUD\nLoading local status...", {durationMs: DISPLAY_MS}) + await this.refresh({forceRender: true}) + this.pollTimer = setInterval(() => { + void this.refresh() + }, POLL_MS) + } + + stop(): void { + if (this.pollTimer) clearInterval(this.pollTimer) + this.pollTimer = null + this.clearInterruptTimer() + for (const unsub of this.unsubs) { + try { + unsub() + } catch { + /* ignore */ + } + } + this.unsubs.length = 0 + } + + private wireUI(): void { + this.unsubs.push(this.ui.onOpen(() => this.sendSnapshot())) + this.unsubs.push(this.ui.on("devhud:request-snapshot", () => this.sendSnapshot())) + this.unsubs.push(this.ui.on("devhud:refresh", () => void this.refresh({forceRender: true}))) + this.unsubs.push( + this.ui.on("devhud:set-endpoint", ({endpoint}) => { + void this.setEndpoint(endpoint) + }), + ) + this.unsubs.push( + this.ui.on("devhud:set-view", ({view}) => { + void this.setView(view) + }), + ) + this.unsubs.push(this.ui.on("devhud:show-target", (target) => this.showTargetFromUI(target))) + this.unsubs.push(this.ui.on("devhud:close-detail-on-glasses", ({view}) => this.closeDetailFromUI(view))) + } + + private async loadSettings(): Promise { + try { + const raw = await this.session.storage.get(SETTINGS_KEY) + if (!raw) return + const parsed = JSON.parse(raw) as StoredSettings + if (typeof parsed.endpoint === "string" && parsed.endpoint.trim()) { + this.endpoint = normalizeEndpoint(parsed.endpoint) + } + if (isDevHudView(parsed.selectedView)) this.selectedView = parsed.selectedView + } catch (err) { + console.log("DevHUD: failed to load settings", err) + } + } + + private async saveSettings(): Promise { + try { + await this.session.storage.set( + SETTINGS_KEY, + JSON.stringify({endpoint: this.endpoint, selectedView: this.selectedView} satisfies StoredSettings), + ) + } catch (err) { + console.log("DevHUD: failed to save settings", err) + } + } + + private async setEndpoint(endpoint: string): Promise { + this.endpoint = normalizeEndpoint(endpoint) + await this.saveSettings() + this.sendSnapshot() + await this.refresh({forceRender: true}) + } + + private async setView(view: DevHudView): Promise { + this.selectedView = view + this.screen = screenForView(view) + this.selectedPr = null + this.selectedThread = null + await this.saveSettings() + this.sendSnapshot() + this.renderDisplay() + } + + private async refresh(options: {forceRender?: boolean} = {}): Promise { + if (this.loading) return + this.setLoading(true) + const hadStatus = this.status !== null + try { + const res = await fetch(`${this.endpoint}/api/status`, {headers: {"Accept": "application/json"}}) + if (!res.ok) throw new Error(`sidecar ${res.status}`) + const status = (await res.json()) as DevHudStatus + this.status = status + this.lastError = null + this.ui.send("devhud:status", status) + const didInterrupt = hadStatus ? this.deliverNotifications(status) : this.rememberDeliveredNotifications(status) + if (!didInterrupt && this.shouldRenderAfterRefresh(hadStatus, options.forceRender === true)) { + this.renderDisplay() + } + } catch (err) { + const message = err instanceof Error ? err.message : "Status refresh failed" + this.lastError = message + this.ui.send("devhud:error", {message}) + this.session.display.showTextWall(`Dev HUD\nSidecar offline\n${this.endpoint}`, {durationMs: DISPLAY_MS}) + console.log(`DevHUD: refresh failed: ${message}`) + } finally { + this.setLoading(false) + this.sendSnapshot() + } + } + + private shouldRenderAfterRefresh(hadStatus: boolean, forceRender: boolean): boolean { + if (forceRender || !hadStatus) return true + // Recreating a native list resets the visible G2 highlight. Keep list + // screens stable during background polling; explicit refreshes redraw them. + return this.screen !== "focus" && this.screen !== "prList" && this.screen !== "codexList" + } + + private setLoading(loading: boolean): void { + this.loading = loading + this.ui.send("devhud:loading", {loading}) + } + + private sendSnapshot(): void { + const snapshot: DevHudSnapshot = { + endpoint: this.endpoint, + status: this.status, + loading: this.loading, + lastError: this.lastError, + selectedView: this.selectedView, + } + this.ui.send("devhud:snapshot", snapshot) + } + + private renderDisplay(): void { + if (Date.now() < this.interruptActiveUntil) return + if (this.screen === "focus") { + this.showFocus() + } else if (this.screen === "main") { + this.showMainMenu() + } else if (this.screen === "summary") { + this.showSummary() + } else if (this.screen === "prList") { + this.showPrList() + } else if (this.screen === "codexList") { + this.showCodexList() + } else if (this.screen === "prDetail") { + this.showSelectedPrDetail() + } else if (this.screen === "codexDetail") { + this.showSelectedThreadDetail() + } else { + this.showNotifications() + } + } + + private handleTouch(event: TouchData): void { + if (Date.now() < this.ignoreTouchUntil) return + + if (Date.now() < this.interruptActiveUntil) { + if (event.kind === "double_click") { + this.dismissInterrupt() + this.renderDisplay() + } else if (event.kind === "click") { + const interrupt = this.activeInterrupt + this.dismissInterrupt() + if (interrupt) this.openNotificationTarget(interrupt) + else this.renderDisplay() + } + return + } + + if (event.kind === "double_click") { + this.handleDoubleClick() + return + } + + if (this.screen === "focus") { + this.ignoreTouchUntil = Date.now() + 750 + this.openScreen("main", this.selectedView) + return + } + + if (event.kind !== "click") { + this.updateSelectedIndex(event) + return + } + + this.updateSelectedIndex(event) + if (this.screen === "main") { + this.handleMainMenuClick() + } else if (this.screen === "prList") { + this.handlePrListClick() + } else if (this.screen === "codexList") { + this.handleCodexListClick() + } else if (this.screen === "notifications") { + this.handleNotificationsClick() + } + } + + private handleMainMenuClick(): void { + const row = this.activeRows[this.selectedIndex] + if (row === "Summary") { + this.openScreen("summary", "summary") + } else if (row === "Pull Requests") { + this.openScreen("prList", "github") + } else if (row === "Codex Threads") { + this.openScreen("codexList", "codex") + } else if (row === "Notifications") { + this.openScreen("notifications", "notifications") + } else if (row === "Refresh") { + this.session.display.showTextWall("Dev HUD\nRefreshing...", {durationMs: 2_000}) + void this.refresh({forceRender: true}) + } + } + + private handlePrListClick(): void { + if (this.activePrs.length === 0) { + const row = this.activeRows[this.selectedIndex] + if (row === "Refresh") void this.refresh({forceRender: true}) + return + } + + const pr = this.activePrs[this.selectedIndex] ?? this.activePrs[0] + if (!pr) return + this.selectedPr = pr + this.screen = "prDetail" + this.ui.send("devhud:open-target", {type: "pr", id: pr.id}) + this.showSelectedPrDetail() + } + + private handleCodexListClick(): void { + if (this.activeThreads.length === 0) { + const row = this.activeRows[this.selectedIndex] + if (row === "Refresh") void this.refresh({forceRender: true}) + return + } + + const thread = this.activeThreads[this.selectedIndex] ?? this.activeThreads[0] + if (!thread) return + this.selectedThread = thread + this.screen = "codexDetail" + this.ui.send("devhud:open-target", {type: "codexThread", id: thread.id}) + this.showSelectedThreadDetail() + } + + private handleNotificationsClick(): void { + if (this.activeNotifications.length === 0) { + const row = this.activeRows[this.selectedIndex] + if (row === "Refresh") void this.refresh({forceRender: true}) + return + } + + const event = this.activeNotifications[this.selectedIndex] ?? this.activeNotifications[0] + if (!event) return + this.openNotificationTarget(event) + } + + private handleDoubleClick(): void { + if (this.screen === "prDetail") { + this.screen = "prList" + this.selectedPr = null + this.ui.send("devhud:close-detail", {}) + this.renderDisplay() + return + } + if (this.screen === "codexDetail") { + this.screen = "codexList" + this.selectedThread = null + this.ui.send("devhud:close-detail", {}) + this.renderDisplay() + return + } + if (this.screen === "summary") { + this.openScreen("main", "summary") + return + } + if (this.screen === "prList" || this.screen === "codexList" || this.screen === "notifications") { + this.openScreen("main", "summary") + return + } + if (this.screen === "main") { + this.openScreen("focus", this.selectedView) + } + } + + private updateSelectedIndex(event: TouchData): void { + const rows = this.activeRows + if (rows.length === 0) { + this.selectedIndex = 0 + return + } + + if (event.selectedItemName) { + const fromName = rows.indexOf(event.selectedItemName) + if (fromName >= 0) { + this.selectedIndex = fromName + return + } + } + + if (typeof event.selectedItemIndex === "number" && Number.isFinite(event.selectedItemIndex)) { + this.selectedIndex = clampIndex(Math.trunc(event.selectedItemIndex), rows.length) + } + } + + private openScreen(screen: GlassScreen, view: DevHudView): void { + this.screen = screen + this.selectedView = view + this.selectedPr = null + this.selectedThread = null + void this.saveSettings() + this.sendSnapshot() + this.renderDisplay() + } + + private openNotificationTarget(event: DevHudNotification): void { + if (event.target.type === "pr") { + const pr = this.status?.github.openPrs.find((item) => item.id === event.target.id) + this.selectedView = "github" + this.screen = "prDetail" + this.selectedPr = pr ?? null + this.selectedThread = null + } else { + const thread = this.status?.codex.threads.find((item) => item.id === event.target.id) + this.selectedView = "codex" + this.screen = "codexDetail" + this.selectedThread = thread ?? null + this.selectedPr = null + } + this.ui.send("devhud:open-target", event.target) + void this.saveSettings() + this.sendSnapshot() + this.renderDisplay() + } + + private showTargetFromUI(target: {type: "pr"; id: string} | {type: "codexThread"; id: string}): void { + if (target.type === "pr") { + const pr = this.status?.github.openPrs.find((item) => item.id === target.id) + this.selectedView = "github" + this.screen = "prDetail" + this.selectedPr = pr ?? null + this.selectedThread = null + } else { + const thread = this.status?.codex.threads.find((item) => item.id === target.id) + this.selectedView = "codex" + this.screen = "codexDetail" + this.selectedThread = thread ?? null + this.selectedPr = null + } + void this.saveSettings() + this.sendSnapshot() + this.renderDisplay() + } + + private closeDetailFromUI(view: "github" | "codex"): void { + this.selectedView = view + if (view === "github") { + this.screen = "prList" + this.selectedPr = null + } else { + this.screen = "codexList" + this.selectedThread = null + } + void this.saveSettings() + this.sendSnapshot() + this.renderDisplay() + } + + private showNativeList(rows: readonly string[]): void { + this.selectedIndex = 0 + this.activeRows = [...rows] + this.session.display.showSelectableList(this.activeRows, LIST_OPTIONS) + } + + private showMainMenu(): void { + this.activePrs = [] + this.activeThreads = [] + this.activeNotifications = [] + this.showNativeList(MAIN_MENU_ROWS) + } + + private showFocus(): void { + this.activePrs = [] + this.activeThreads = [] + this.activeNotifications = [] + this.activeRows = [] + this.selectedIndex = 0 + this.session.display.clear() + } + + private showSummary(): void { + this.activePrs = [] + this.activeThreads = [] + this.activeNotifications = [] + this.activeRows = [] + if (!this.status) { + this.session.display.showTextWall("Summary\nLoading status...", {durationMs: DISPLAY_MS}) + return + } + this.session.display.showTextWall(formatSummaryDisplay(this.status), { + durationMs: DISPLAY_MS, + breakMode: "word", + }) + } + + private showPrList(): void { + this.activeThreads = [] + this.activeNotifications = [] + this.activePrs = this.status?.github.openPrs ?? [] + if (!this.status) { + this.showNativeList(LOADING_ROWS) + return + } + if (this.activePrs.length === 0) { + this.showNativeList(EMPTY_PR_ROWS) + return + } + this.showNativeList(this.activePrs.map(formatPrRow)) + } + + private showCodexList(): void { + this.activePrs = [] + this.activeNotifications = [] + this.activeThreads = this.status?.codex.threads ?? [] + if (!this.status) { + this.showNativeList(LOADING_ROWS) + return + } + if (this.activeThreads.length === 0) { + this.showNativeList(EMPTY_CODEX_ROWS) + return + } + this.showNativeList(this.activeThreads.map(formatThreadRow)) + } + + private showSelectedPrDetail(): void { + if (this.status && this.selectedPr) { + this.selectedPr = this.status.github.openPrs.find((pr) => pr.id === this.selectedPr?.id) ?? this.selectedPr + } + if (!this.selectedPr) { + this.screen = "prList" + this.showPrList() + return + } + this.session.display.showTextWall(formatPrDetail(this.selectedPr), {breakMode: "word"}) + } + + private showSelectedThreadDetail(): void { + if (this.status && this.selectedThread) { + this.selectedThread = + this.status.codex.threads.find((thread) => thread.id === this.selectedThread?.id) ?? this.selectedThread + } + if (!this.selectedThread) { + this.screen = "codexList" + this.showCodexList() + return + } + const tasks = this.status?.codex.tasks.filter((task) => task.threadId === this.selectedThread?.id) ?? [] + this.session.display.showTextWall(formatThreadDetail(this.selectedThread, tasks), {breakMode: "word"}) + } + + private showNotifications(): void { + this.activePrs = [] + this.activeThreads = [] + this.activeNotifications = this.status?.notifications.events ?? [] + if (!this.status) { + this.showNativeList(LOADING_ROWS) + return + } + if (this.activeNotifications.length === 0) { + this.showNativeList(EMPTY_NOTIFICATION_ROWS) + return + } + this.showNativeList(this.activeNotifications.map(formatNotificationRow)) + } + + private deliverNotifications(status: DevHudStatus): boolean { + let interrupt: DevHudNotification | null = null + for (const event of status.notifications.events.slice().reverse()) { + if (this.deliveredNotificationIds.has(event.id)) continue + this.deliveredNotificationIds.add(event.id) + this.ui.send("devhud:notification", event) + if (event.interrupt) interrupt = event + } + if (this.deliveredNotificationIds.size > 300) { + this.deliveredNotificationIds = new Set(Array.from(this.deliveredNotificationIds).slice(-150)) + } + if (!interrupt) return false + this.activeInterrupt = interrupt + this.interruptActiveUntil = Date.now() + NOTIFICATION_DISPLAY_MS + this.clearInterruptTimer() + this.session.display.showTextWall(formatNotificationDisplay(interrupt), { + durationMs: NOTIFICATION_DISPLAY_MS, + breakMode: "word", + }) + this.interruptTimer = setTimeout(() => { + this.activeInterrupt = null + this.interruptActiveUntil = 0 + this.interruptTimer = null + this.renderDisplay() + }, NOTIFICATION_DISPLAY_MS) + return true + } + + private rememberDeliveredNotifications(status: DevHudStatus): false { + for (const event of status.notifications.events) { + this.deliveredNotificationIds.add(event.id) + } + return false + } + + private clearInterruptTimer(): void { + if (!this.interruptTimer) return + clearTimeout(this.interruptTimer) + this.interruptTimer = null + } + + private dismissInterrupt(): void { + this.activeInterrupt = null + this.interruptActiveUntil = 0 + this.clearInterruptTimer() + } +} + +function formatNotificationDisplay(event: DevHudNotification): string { + return [`Dev HUD Alert`, event.title, truncate(event.message, 78)].join("\n") +} + +function formatSummaryDisplay(status: DevHudStatus): string { + const openPrs = status.github.openPrs.length + const failingPrs = status.github.openPrs.filter((pr) => pr.checks.state === "failure").length + const pendingPrs = status.github.openPrs.filter((pr) => pr.checks.state === "pending").length + const runningTasks = status.codex.tasks.filter((task) => task.running) + const latestEvent = status.notifications.events[0] + const lines = [ + "Summary", + `PRs: ${openPrs} open, ${failingPrs} failing, ${pendingPrs} pending`, + `Codex: ${status.codex.threads.length} threads, ${runningTasks.length} running`, + ] + if (runningTasks[0]) { + lines.push(`Task: ${truncate(oneLine(runningTasks[0].command), 82)}`) + } + if (latestEvent) { + lines.push(`Latest: ${truncate(oneLine(latestEvent.title), 80)}`) + } else { + lines.push("Latest: no changes yet") + } + return lines.join("\n") +} + +function formatPrRow(pr: PullRequestSummary): string { + return `#${pr.number} ${statusEmoji(pr)} ${truncate(oneLine(pr.title), 42)}` +} + +function formatThreadRow(thread: CodexThreadSummary): string { + return `${threadStateLabel(thread.state)} ${truncate(oneLine(thread.title), 38)} [${shortId(thread.id)}]` +} + +function formatNotificationRow(event: DevHudNotification): string { + return `${eventLabel(event)} ${formatClock(event.timestamp)} ${truncate(oneLine(event.title), 34)} [${shortEventId(event.id)}]` +} + +function formatPrDetail(pr: PullRequestSummary): string { + const lines = [ + `PR #${pr.number} ${statusEmoji(pr)}`, + truncate(oneLine(pr.title), 92), + `${pr.repo} ${pr.branch ?? "unknown"} -> ${pr.base ?? "unknown"}`, + `Checks: ${formatCheckState(pr.checks.state)} ${pr.checks.success}/${pr.checks.total} ok, ${pr.checks.failure} fail, ${pr.checks.pending} pending`, + `Review: ${formatReviewState(pr.reviewState)}`, + ] + + const failedOrPending = pr.checks.items + .filter((item) => item.state === "failure" || item.state === "pending") + .slice(0, 3) + for (const item of failedOrPending) { + lines.push(`${item.state.toUpperCase()}: ${truncate(oneLine(item.name), 72)}`) + } + + const comments = pr.reviewComments.filter((comment) => oneLine(comment.body).length > 0) + if (comments.length > 0) { + lines.push("Review comments:") + for (const comment of comments.slice(0, 3)) { + const location = comment.path ? ` ${basename(comment.path)}${comment.line ? `:${comment.line}` : ""}` : "" + lines.push(`${comment.author}${location}`) + lines.push(truncate(oneLine(comment.body), 92)) + } + if (comments.length > 3) lines.push(`+${comments.length - 3} more comments`) + } else { + lines.push("Review comments: none") + } + + return lines.join("\n") +} + +function formatThreadDetail(thread: CodexThreadSummary, tasks: CodexTaskSummary[]): string { + const lines = [ + `Codex ${threadStateLabel(thread.state)}`, + truncate(oneLine(thread.title), 92), + `Branch: ${thread.branch ?? "unknown"}`, + `CWD: ${basename(thread.cwd)}`, + `Updated ${formatClock(thread.updatedAt)}`, + ] + + const runningTasks = tasks.filter((task) => task.running) + if (runningTasks.length > 0) { + lines.push("Running:") + for (const task of runningTasks.slice(0, 3)) { + lines.push(truncate(oneLine(task.command), 96)) + } + } else if (thread.runningCommands.length > 0) { + lines.push("Recent commands:") + for (const command of thread.runningCommands.slice(0, 3)) { + lines.push(truncate(oneLine(command), 96)) + } + } else { + lines.push("No running tasks.") + } + + return lines.join("\n") +} + +function formatNotificationsDisplay(status: DevHudStatus): string { + const lines = ["Notifications"] + const events = status.notifications.events.slice(0, 4) + if (events.length === 0) lines.push("No changes yet") + for (const event of events) { + lines.push(`${eventLabel(event)} ${truncate(event.title, 44)}`) + } + return lines.join("\n") +} + +function statusEmoji(pr: PullRequestSummary): string { + if (pr.isDraft) return "DRAFT" + if (pr.checks.state === "failure") return "FAIL" + if (pr.reviewState === "changes_requested") return "FIX" + if (pr.checks.state === "pending") return "WAIT" + if (pr.reviewState === "approved") return "OK" + return "OPEN" +} + +function screenForView(view: DevHudView): GlassScreen { + if (view === "github") return "prList" + if (view === "codex") return "codexList" + if (view === "notifications") return "notifications" + return "main" +} + +function clampIndex(index: number, length: number): number { + if (length <= 0) return 0 + return Math.max(0, Math.min(index, length - 1)) +} + +function formatCheckState(state: PullRequestSummary["checks"]["state"]): string { + if (state === "success") return "passing" + if (state === "failure") return "failing" + if (state === "pending") return "pending" + return "unknown" +} + +function formatReviewState(state: PullRequestSummary["reviewState"]): string { + if (state === "changes_requested") return "changes requested" + if (state === "review_required") return "review required" + if (state === "approved") return "approved" + if (state === "none") return "none" + return "unknown" +} + +function threadStateLabel(state: CodexThreadSummary["state"]): string { + if (state === "running") return "RUN" + if (state === "done" || state === "active") return "DONE" + if (state === "recent") return "RECENT" + return "STALE" +} + +function normalizeEndpoint(endpoint: string): string { + const trimmed = endpoint.trim() + if (!trimmed) return DEFAULT_ENDPOINT + return trimmed.replace(/\/+$/, "") +} + +function formatClock(timestamp: number): string { + return new Date(timestamp).toLocaleTimeString([], {hour: "numeric", minute: "2-digit"}) +} + +function truncate(text: string, max: number): string { + const compact = oneLine(text) + if (compact.length <= max) return compact + return `${compact.slice(0, Math.max(0, max - 1)).trimEnd()}...` +} + +function oneLine(text: string): string { + return text.replace(/\s+/g, " ").trim() +} + +function basename(path: string): string { + const parts = path.split("/").filter(Boolean) + return parts[parts.length - 1] ?? path +} + +function shortId(id: string): string { + return id.replace(/-/g, "").slice(0, 6) +} + +function shortEventId(id: string): string { + return id.replace(/[^a-z0-9]/gi, "").slice(-5) +} + +function isDevHudView(value: unknown): value is DevHudView { + return value === "summary" || value === "github" || value === "codex" || value === "notifications" +} + +function eventLabel(event: DevHudNotification): string { + if (event.severity === "critical") return "!" + if (event.severity === "warning") return "?" + if (event.severity === "success") return "OK" + return "-" +} + +registerMiniapp((session) => { + const controller = new DevHudController(session) + void controller.start() + session.onBeforeDisconnect(() => controller.stop()) +}) diff --git a/miniapps/dev-hud/src/shared/channels.ts b/miniapps/dev-hud/src/shared/channels.ts new file mode 100644 index 0000000000..09d17affd9 --- /dev/null +++ b/miniapps/dev-hud/src/shared/channels.ts @@ -0,0 +1,22 @@ +import type {DevHudNotification, DevHudSnapshot, DevHudStatus, DevHudView, NotificationTarget} from "./types" + +export interface Channels { + "devhud:snapshot": DevHudSnapshot + "devhud:status": DevHudStatus + "devhud:notification": DevHudNotification + "devhud:open-target": NotificationTarget + "devhud:close-detail": Record + "devhud:show-target": NotificationTarget + "devhud:close-detail-on-glasses": {view: Extract} + "devhud:error": {message: string} + "devhud:loading": {loading: boolean} + "devhud:request-snapshot": Record + "devhud:refresh": Record + "devhud:set-endpoint": {endpoint: string} + "devhud:set-view": {view: DevHudView} +} + +declare global { + // eslint-disable-next-line no-var + var mentra: import("@mentra/miniapp/ui").MentraTyped +} diff --git a/miniapps/dev-hud/src/shared/types.ts b/miniapps/dev-hud/src/shared/types.ts new file mode 100644 index 0000000000..ad23ab689b --- /dev/null +++ b/miniapps/dev-hud/src/shared/types.ts @@ -0,0 +1,157 @@ +export type SourceState = "ok" | "loading" | "error" | "disabled" + +export interface SourceStatus { + state: SourceState + message: string | null + updatedAt: number | null +} + +export type CheckState = "success" | "pending" | "failure" | "unknown" +export type ReviewState = "approved" | "changes_requested" | "review_required" | "none" | "unknown" + +export interface PullRequestCheckItem { + name: string + workflow: string | null + state: CheckState + url: string | null +} + +export interface PullRequestReviewNote { + id: string + type: "review" | "inline" + author: string + body: string + state: string | null + path: string | null + line: number | null + url: string | null + createdAt: string | null +} + +export interface PullRequestSummary { + id: string + repo: string + number: number + title: string + url: string + isDraft: boolean + updatedAt: string | null + branch: string | null + base: string | null + mergeState: string | null + reviewState: ReviewState + requestedReviewers: string[] + checks: { + state: CheckState + total: number + success: number + pending: number + failure: number + skipped: number + items: PullRequestCheckItem[] + } + reviewComments: PullRequestReviewNote[] +} + +export type CodexThreadState = "running" | "done" | "active" | "recent" | "stale" + +export interface CodexThreadSummary { + id: string + title: string + cwd: string + branch: string | null + updatedAt: number + state: CodexThreadState + runningCommands: string[] +} + +export interface CodexTaskSummary { + id: string + threadId: string + title: string + cwd: string + command: string + startedAt: number + updatedAt: number + pid: number | null + running: boolean +} + +export interface CodexTurnSummary { + id: string + threadId: string + title: string + completedAt: number +} + +export interface DevHudStatus { + generatedAt: number + endpoint: string + github: { + status: SourceStatus + openPrs: PullRequestSummary[] + } + codex: { + status: SourceStatus + threads: CodexThreadSummary[] + tasks: CodexTaskSummary[] + turns: CodexTurnSummary[] + } + notifications: { + status: SourceStatus + events: DevHudNotification[] + } +} + +export interface DevHudSnapshot { + endpoint: string + status: DevHudStatus | null + loading: boolean + lastError: string | null + selectedView: DevHudView +} + +export type DevHudView = "summary" | "github" | "codex" | "notifications" + +export type NotificationSource = "github" | "codex" | "system" +export type NotificationSeverity = "info" | "success" | "warning" | "critical" + +export type NotificationKind = + | "github_pr_opened" + | "github_pr_removed" + | "github_checks_failed" + | "github_checks_recovered" + | "github_checks_changed" + | "github_changes_requested" + | "github_approved" + | "github_review_changed" + | "github_merge_blocked" + | "github_merge_recovered" + | "codex_task_started" + | "codex_task_finished" + | "codex_turn_finished" + | "codex_thread_finished" + +export type NotificationTarget = + | { + type: "pr" + id: string + } + | { + type: "codexThread" + id: string + } + +export interface DevHudNotification { + id: string + timestamp: number + source: NotificationSource + severity: NotificationSeverity + kind: NotificationKind + interrupt: boolean + title: string + message: string + target: NotificationTarget + before: string | null + after: string | null +} diff --git a/miniapps/dev-hud/src/ui/App.tsx b/miniapps/dev-hud/src/ui/App.tsx new file mode 100644 index 0000000000..2af1e6ff8b --- /dev/null +++ b/miniapps/dev-hud/src/ui/App.tsx @@ -0,0 +1,698 @@ +import {useEffect, useMemo, useState, type ReactNode} from "react" +import {useSafeArea} from "@mentra/miniapp/ui" +import { + AlertCircle, + Bell, + CheckCircle2, + ChevronLeft, + CircleDashed, + Clock3, + ExternalLink, + GitPullRequest, + Loader2, + MessageSquare, + RadioTower, + RefreshCw, + TerminalSquare, +} from "lucide-react" + +import type { + CheckState, + CodexTaskSummary, + CodexThreadSummary, + DevHudNotification, + DevHudSnapshot, + DevHudStatus, + DevHudView, + NotificationTarget, + PullRequestCheckItem, + PullRequestReviewNote, + PullRequestSummary, + ReviewState, +} from "../shared/types" + +const EMPTY_SNAPSHOT: DevHudSnapshot = { + endpoint: "", + status: null, + loading: true, + lastError: null, + selectedView: "summary", +} + +type DetailTarget = NotificationTarget + +export function App() { + const {insets, capsuleMenu} = useSafeArea() + const [snapshot, setSnapshot] = useState(EMPTY_SNAPSHOT) + const [endpointDraft, setEndpointDraft] = useState("") + const [detailTarget, setDetailTarget] = useState(null) + + useEffect(() => { + const unsubs = [ + mentra.on("devhud:snapshot", (next: DevHudSnapshot) => { + setSnapshot(next) + setEndpointDraft(next.endpoint) + }), + mentra.on("devhud:status", (status: DevHudStatus) => { + setSnapshot((current) => ({...current, status, lastError: null})) + }), + mentra.on("devhud:notification", (event: DevHudNotification) => { + setSnapshot((current) => { + if (!current.status) return current + const exists = current.status.notifications.events.some((item) => item.id === event.id) + if (exists) return current + return { + ...current, + status: { + ...current.status, + notifications: { + ...current.status.notifications, + events: [event, ...current.status.notifications.events], + }, + }, + } + }) + }), + mentra.on("devhud:open-target", (target: DetailTarget) => { + openTarget(target, {syncGlasses: false}) + }), + mentra.on("devhud:close-detail", () => { + closeDetail({syncGlasses: false}) + }), + mentra.on("devhud:error", ({message}) => { + setSnapshot((current) => ({...current, lastError: message})) + }), + mentra.on("devhud:loading", ({loading}) => { + setSnapshot((current) => ({...current, loading})) + }), + ] + mentra.send("devhud:request-snapshot", {}) + return () => { + for (const unsub of unsubs) unsub() + } + }, []) + + const status = snapshot.status + const counts = useMemo(() => summarize(status), [status]) + const headerPaddingRight = capsuleMenu ? Math.max(20, capsuleMenu.width + 20) : 20 + + const openTarget = (target: DetailTarget, options: {syncGlasses?: boolean} = {}) => { + setDetailTarget(target) + const view: DevHudView = target.type === "pr" ? "github" : "codex" + setSnapshot((current) => ({...current, selectedView: view})) + if (options.syncGlasses !== false) { + mentra.send("devhud:show-target", target) + } + } + + const closeDetail = (options: {syncGlasses?: boolean} = {}) => { + const target = detailTarget + setDetailTarget(null) + if (options.syncGlasses === false || !target) return + mentra.send("devhud:close-detail-on-glasses", {view: target.type === "pr" ? "github" : "codex"}) + } + + const selectView = (view: DevHudView) => { + setDetailTarget(null) + setSnapshot((current) => ({...current, selectedView: view})) + mentra.send("devhud:set-view", {view}) + } + + return ( +
+
+
+
+
+
+
+ +
+
+

Dev HUD

+

+ {status ? `Updated ${formatTime(status.generatedAt)}` : "Waiting for local sidecar"} +

+
+
+
+ +
+
+ + + +
+ {snapshot.lastError ? : null} + {detailTarget && status ? ( + + ) : ( + <> + {snapshot.selectedView === "summary" ? : null} + {snapshot.selectedView === "github" ? openTarget({type: "pr", id})} /> : null} + {snapshot.selectedView === "codex" ? ( + openTarget({type: "codexThread", id})} + /> + ) : null} + {snapshot.selectedView === "notifications" ? ( + + ) : null} + + )} +
+ +
+
{ + event.preventDefault() + mentra.send("devhud:set-endpoint", {endpoint: endpointDraft}) + }}> + setEndpointDraft(event.currentTarget.value)} + className="h-10 min-w-0 flex-1 rounded-md border border-neutral-200 bg-neutral-50 px-3 text-sm text-neutral-900 outline-none focus:border-[#1f7a6d]" + spellCheck={false} + /> + +
+
+
+
+ ) +} + +function ViewSwitch({ + selectedView, + notificationCount, + onSelect, +}: { + selectedView: DevHudView + notificationCount: number + onSelect: (view: DevHudView) => void +}) { + return ( + + ) +} + +function TabButton({active, label, onClick, children}: {active: boolean; label: string; onClick: () => void; children: ReactNode}) { + return ( + + ) +} + +function SummaryView({ + status, + counts, + onOpenTarget, +}: { + status: DevHudStatus | null + counts: SummaryCounts + onOpenTarget: (target: DetailTarget) => void +}) { + if (!status) return + const runningTasks = status.codex.tasks.filter((task) => task.running) + const latestThread = status.codex.threads[0] + return ( +
+
+ 0 ? "bad" : "good"} /> + 0 ? "warn" : "quiet"} /> + 0 ? "bad" : "quiet"} /> + event.severity === "critical") ? "bad" : "quiet"} /> +
+ + {status.notifications.events[0] ? : null} + {status.github.openPrs[0] ? onOpenTarget({type: "pr", id: status.github.openPrs[0].id})} /> : null} + {latestThread ? onOpenTarget({type: "codexThread", id: latestThread.id})} /> : null} + {runningTasks.length > 0 ? onOpenTarget({type: "codexThread", id: runningTasks[0].threadId})} /> : null} +
+ ) +} + +function GitHubView({prs, onOpenPr}: {prs: PullRequestSummary[]; onOpenPr: (id: string) => void}) { + if (prs.length === 0) return + return ( +
+ {prs.map((pr) => ( + onOpenPr(pr.id)} /> + ))} +
+ ) +} + +function CodexView({ + threads, + tasks, + onOpenThread, +}: { + threads: CodexThreadSummary[] + tasks: CodexTaskSummary[] + onOpenThread: (id: string) => void +}) { + const runningTasks = tasks.filter((task) => task.running) + return ( +
+
+ } label="Running Tasks" /> + {runningTasks.length === 0 ? ( + + ) : ( + runningTasks.map((task) => onOpenThread(task.threadId)} />) + )} +
+
+ } label="Recent Threads" /> + {threads.map((thread) => ( + onOpenThread(thread.id)} /> + ))} +
+
+ ) +} + +function NotificationsView({events, onOpenTarget}: {events: DevHudNotification[]; onOpenTarget: (target: DetailTarget) => void}) { + if (events.length === 0) return + return ( +
+ {events.map((event) => ( + + ))} +
+ ) +} + +function DetailView({status, target, onBack}: {status: DevHudStatus; target: DetailTarget; onBack: () => void}) { + if (target.type === "pr") { + const pr = status.github.openPrs.find((item) => item.id === target.id) + return + } + const thread = status.codex.threads.find((item) => item.id === target.id) + const tasks = status.codex.tasks.filter((task) => task.threadId === target.id) + return +} + +function PullRequestDetail({pr, onBack}: {pr?: PullRequestSummary; onBack: () => void}) { + if (!pr) return + const checkPresentation = checkStatePresentation(pr.checks.state) + const reviewPresentation = reviewStatePresentation(pr.reviewState, pr.requestedReviewers.length) + return ( +
+ + + + ${pr.base ?? "base"}` : "Unknown"} /> + +
+ + + {pr.isDraft ? : null} + {pr.mergeState ? : null} +
+ + Open on GitHub + + + +
+ } label="Checks" /> + {pr.checks.items.length === 0 ? : null} + {pr.checks.items.map((item) => ( + + ))} +
+ +
+ } label="Review Comments" /> + {pr.reviewComments.length === 0 ? : null} + {pr.reviewComments.map((comment) => ( + + ))} +
+
+ ) +} + +function CodexThreadDetail({thread, tasks, onBack}: {thread?: CodexThreadSummary; tasks: CodexTaskSummary[]; onBack: () => void}) { + if (!thread) return + return ( +
+ + + + + + + +
+ } label="Running Commands" /> + {tasks.length === 0 ? : null} + {tasks.map((task) => ( + + ))} +
+
+ ) +} + +function PullRequestRow({pr, compact = false, onOpen}: {pr: PullRequestSummary; compact?: boolean; onOpen: () => void}) { + const checkPresentation = checkStatePresentation(pr.checks.state) + const reviewPresentation = reviewStatePresentation(pr.reviewState, pr.requestedReviewers.length) + return ( + + ) +} + +function ThreadRow({thread, compact = false, onOpen}: {thread: CodexThreadSummary; compact?: boolean; onOpen: () => void}) { + return ( + + ) +} + +function TaskRow({task, onOpen}: {task: CodexTaskSummary; onOpen?: () => void}) { + const content = ( +
+ +
+

{task.title}

+

{task.command}

+

{basename(task.cwd)}

+
+
+ ) + if (!onOpen) return
{content}
+ return ( + + ) +} + +function NotificationRow({event, onOpenTarget}: {event: DevHudNotification; onOpenTarget: (target: DetailTarget) => void}) { + const tone = event.severity === "critical" ? "bad" : event.severity === "warning" ? "warn" : event.severity === "success" ? "good" : "quiet" + return ( + + ) +} + +function CheckRow({item}: {item: PullRequestCheckItem}) { + const presentation = checkStatePresentation(item.state) + const content = ( +
+
+ {presentation.icon} +
+
+

{item.name}

+

{item.workflow ?? "status"}

+
+
+ ) + if (!item.url) return
{content}
+ return ( + + {content} + + ) +} + +function ReviewComment({comment}: {comment: PullRequestReviewNote}) { + const body = ( + <> +
+

{comment.author}

+ +
+ {comment.path ? ( +

+ {comment.path} + {comment.line ? `:${comment.line}` : ""} +

+ ) : null} +

{comment.body}

+ + ) + if (!comment.url) return
{body}
+ return ( + + {body} + + ) +} + +function DetailHeader({label, title, onBack}: {label: string; title: string; onBack: () => void}) { + return ( +
+ +

{label}

+

{title}

+
+ ) +} + +function MissingDetail({title, onBack}: {title: string; onBack: () => void}) { + return ( +
+ + +
+ ) +} + +function InfoCard({children}: {children: ReactNode}) { + return
{children}
+} + +function InfoRow({label, value}: {label: string; value: string}) { + return ( +
+ {label} + {value} +
+ ) +} + +function Metric({label, value, tone}: {label: string; value: number; tone: ChipTone}) { + const color = tone === "bad" ? "#d73a31" : tone === "warn" ? "#9a6a00" : tone === "good" ? "#1f7a6d" : "#737373" + return ( +
+
{label}
+
+ {value} +
+
+ ) +} + +function ErrorBanner({message, endpoint}: {message: string; endpoint: string}) { + return ( +
+
+ +
+

Sidecar unavailable

+

{message}

+

{endpoint}

+
+
+
+ ) +} + +function EmptyState({title, detail, compact = false}: {title: string; detail: string; compact?: boolean}) { + return ( +
+

{title}

+

{detail}

+
+ ) +} + +function SectionTitle({icon, label}: {icon: ReactNode; label: string}) { + return ( +
+ {icon} + {label} +
+ ) +} + +type ChipTone = "good" | "warn" | "bad" | "quiet" + +function StatusChip({label, tone}: {label: string; tone: ChipTone}) { + return {label} +} + +function toneClass(tone: ChipTone): string { + if (tone === "good") return "bg-[#e6f3ef] text-[#126253]" + if (tone === "warn") return "bg-[#fff3ca] text-[#815900]" + if (tone === "bad") return "bg-[#ffe5e5] text-[#b92323]" + return "bg-neutral-100 text-neutral-600" +} + +function toneBg(tone: ChipTone): string { + if (tone === "good") return "bg-[#e6f3ef] text-[#126253]" + if (tone === "warn") return "bg-[#fff3ca] text-[#815900]" + if (tone === "bad") return "bg-[#ffe5e5] text-[#b92323]" + return "bg-neutral-100 text-neutral-600" +} + +function checkStatePresentation(state: CheckState): {label: string; tone: ChipTone; bg: string; icon: ReactNode} { + if (state === "success") { + return {label: "checks ok", tone: "good", bg: "bg-[#e6f3ef]", icon: } + } + if (state === "failure") { + return {label: "checks fail", tone: "bad", bg: "bg-[#ffe5e5]", icon: } + } + if (state === "pending") { + return {label: "checks pending", tone: "warn", bg: "bg-[#fff3ca]", icon: } + } + return {label: "checks unknown", tone: "quiet", bg: "bg-neutral-100", icon: } +} + +function reviewStatePresentation(state: ReviewState, requestedCount: number): string { + if (state === "approved") return "approved" + if (state === "changes_requested") return "changes requested" + if (state === "review_required") return requestedCount > 0 ? `${requestedCount} requested` : "review" + if (state === "none") return "no review" + return "review unknown" +} + +function threadStateTone(state: CodexThreadSummary["state"]): ChipTone { + if (state === "done" || state === "active") return "good" + if (state === "running") return "warn" + return "quiet" +} + +function threadStateText(state: CodexThreadSummary["state"]): string { + if (state === "active") return "done" + return state +} + +interface SummaryCounts { + openPrs: number + failedPrs: number +} + +function summarize(status: DevHudStatus | null): SummaryCounts { + const prs = status?.github.openPrs ?? [] + return { + openPrs: prs.length, + failedPrs: prs.filter((pr) => pr.checks.state === "failure").length, + } +} + +function isBadMergeState(state: string): boolean { + const normalized = state.toUpperCase() + return normalized === "BLOCKED" || normalized === "DIRTY" || normalized === "UNKNOWN" +} + +function formatTime(timestamp: number): string { + return new Date(timestamp).toLocaleTimeString([], {hour: "numeric", minute: "2-digit"}) +} + +function formatDateTime(value: string): string { + const date = new Date(value) + if (Number.isNaN(date.getTime())) return value + return `${date.toLocaleDateString([], {month: "short", day: "numeric"})} ${date.toLocaleTimeString([], {hour: "numeric", minute: "2-digit"})}` +} + +function basename(path: string): string { + const normalized = path.replace(/\/+$/, "") + return normalized.slice(normalized.lastIndexOf("/") + 1) || normalized +} diff --git a/miniapps/dev-hud/src/ui/env.d.ts b/miniapps/dev-hud/src/ui/env.d.ts new file mode 100644 index 0000000000..d7e961ee22 --- /dev/null +++ b/miniapps/dev-hud/src/ui/env.d.ts @@ -0,0 +1 @@ +declare module "*.css" diff --git a/miniapps/dev-hud/src/ui/index.css b/miniapps/dev-hud/src/ui/index.css new file mode 100644 index 0000000000..f705f85c09 --- /dev/null +++ b/miniapps/dev-hud/src/ui/index.css @@ -0,0 +1,40 @@ +@import "tailwindcss"; + +:root { + font-family: + Inter, + ui-sans-serif, + system-ui, + -apple-system, + BlinkMacSystemFont, + "Segoe UI", + sans-serif; + color: #171717; + background: #f5f6f4; +} + +html, +body, +#root { + height: 100%; +} + +body { + margin: 0; + overflow: hidden; + -webkit-font-smoothing: antialiased; +} + +button, +input { + font: inherit; +} + +button { + -webkit-tap-highlight-color: transparent; +} + +.selectable-text { + user-select: text; + -webkit-user-select: text; +} diff --git a/miniapps/dev-hud/src/ui/index.html b/miniapps/dev-hud/src/ui/index.html new file mode 100644 index 0000000000..3e5f2d4750 --- /dev/null +++ b/miniapps/dev-hud/src/ui/index.html @@ -0,0 +1,13 @@ + + + + + + Dev HUD + + + +
+ + + diff --git a/miniapps/dev-hud/src/ui/main.tsx b/miniapps/dev-hud/src/ui/main.tsx new file mode 100644 index 0000000000..993ed31ed7 --- /dev/null +++ b/miniapps/dev-hud/src/ui/main.tsx @@ -0,0 +1,17 @@ +import {createRoot} from "react-dom/client" +import {MentraProvider} from "@mentra/miniapp/ui" +import "../shared/channels" + +import {App} from "./App" +import "./index.css" + +const root = document.getElementById("root") +if (!root) throw new Error("Root element not found") + +createRoot(root).render( + + + , +) + +mentra.ready() diff --git a/miniapps/dev-hud/tsconfig.json b/miniapps/dev-hud/tsconfig.json new file mode 100644 index 0000000000..f1ef32830d --- /dev/null +++ b/miniapps/dev-hud/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2020", "DOM"], + "jsx": "react-jsx", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true, + "types": ["bun"] + }, + "include": ["src/**/*", "sidecar/**/*", "scripts/**/*", "build.ts"] +} From 31a06785222ea2791af27680cdf382a9333f24ce Mon Sep 17 00:00:00 2001 From: PhilippeFerreiraDeSousa Date: Thu, 25 Jun 2026 09:55:00 -0700 Subject: [PATCH 3/4] Add selectable list demo miniapp --- bun.lock | 11 ++ miniapps/selectable-list-demo/.gitignore | 3 + miniapps/selectable-list-demo/build.ts | 42 +++++ miniapps/selectable-list-demo/miniapp.json | 22 +++ miniapps/selectable-list-demo/package.json | 17 ++ .../src/background/index.ts | 168 ++++++++++++++++++ .../src/shared/channels.ts | 27 +++ .../selectable-list-demo/src/ui/index.html | 37 ++++ miniapps/selectable-list-demo/src/ui/main.ts | 43 +++++ .../selectable-list-demo/src/ui/styles.css | 159 +++++++++++++++++ miniapps/selectable-list-demo/tsconfig.json | 13 ++ 11 files changed, 542 insertions(+) create mode 100644 miniapps/selectable-list-demo/.gitignore create mode 100644 miniapps/selectable-list-demo/build.ts create mode 100644 miniapps/selectable-list-demo/miniapp.json create mode 100644 miniapps/selectable-list-demo/package.json create mode 100644 miniapps/selectable-list-demo/src/background/index.ts create mode 100644 miniapps/selectable-list-demo/src/shared/channels.ts create mode 100644 miniapps/selectable-list-demo/src/ui/index.html create mode 100644 miniapps/selectable-list-demo/src/ui/main.ts create mode 100644 miniapps/selectable-list-demo/src/ui/styles.css create mode 100644 miniapps/selectable-list-demo/tsconfig.json diff --git a/bun.lock b/bun.lock index 60f62641e1..5f3f3fb87b 100644 --- a/bun.lock +++ b/bun.lock @@ -243,6 +243,15 @@ "tailwindcss": "^4.2.4", }, }, + "miniapps/selectable-list-demo": { + "name": "selectable-list-demo", + "dependencies": { + "@mentra/miniapp": "workspace:*", + }, + "devDependencies": { + "@mentra/miniapp-cli": "workspace:*", + }, + }, "miniapps/teleprompter": { "name": "teleprompter-miniapp", "dependencies": { @@ -1558,6 +1567,8 @@ "secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="], + "selectable-list-demo": ["selectable-list-demo@workspace:miniapps/selectable-list-demo"], + "semver": ["semver@6.3.1", "", { "bin": "bin/semver.js" }, ""], "set-cookie-parser": ["set-cookie-parser@2.7.2", "", {}, "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw=="], diff --git a/miniapps/selectable-list-demo/.gitignore b/miniapps/selectable-list-demo/.gitignore new file mode 100644 index 0000000000..299df1e19f --- /dev/null +++ b/miniapps/selectable-list-demo/.gitignore @@ -0,0 +1,3 @@ +dist/ +build/ +node_modules/ diff --git a/miniapps/selectable-list-demo/build.ts b/miniapps/selectable-list-demo/build.ts new file mode 100644 index 0000000000..04e1c3fb55 --- /dev/null +++ b/miniapps/selectable-list-demo/build.ts @@ -0,0 +1,42 @@ +/** + * Minimal two-layer miniapp build. + * + * Emits: + * dist/background/index.js - JSContext bundle, evaluated as a classic script + * dist/ui/index.html - tiny WebView companion UI + */ + +import {rm} from "fs/promises" + +const distDir = "./dist" + +await rm(distDir, {recursive: true, force: true}) + +const backgroundResult = await Bun.build({ + entrypoints: ["./src/background/index.ts"], + outdir: `${distDir}/background`, + target: "browser", + format: "iife", + minify: false, +}) + +if (!backgroundResult.success) { + console.error("Background build failed:") + for (const log of backgroundResult.logs) console.error(log) + process.exit(1) +} + +const uiResult = await Bun.build({ + entrypoints: ["./src/ui/index.html"], + outdir: `${distDir}/ui`, + target: "browser", + minify: true, +}) + +if (!uiResult.success) { + console.error("UI build failed:") + for (const log of uiResult.logs) console.error(log) + process.exit(1) +} + +console.log("built selectable-list-demo -> dist/background/index.js + dist/ui") diff --git a/miniapps/selectable-list-demo/miniapp.json b/miniapps/selectable-list-demo/miniapp.json new file mode 100644 index 0000000000..88d0cbeb8b --- /dev/null +++ b/miniapps/selectable-list-demo/miniapp.json @@ -0,0 +1,22 @@ +{ + "$schema": "./node_modules/@mentra/miniapp-cli/schema/miniapp.schema.json", + "packageName": "com.mentra.selectable_list_demo", + "version": "0.1.0", + "name": "Selectable List Demo", + "description": "Minimal local miniapp that renders a native scrollable selectable list on G2 and reports selected rows back through touch events.", + "type": "standard", + "sdkVersion": "0.3.0", + "minHostVersion": "1.42.0", + "port": 3150, + "entry": { + "background": "background/index.js", + "ui": "ui/index.html" + }, + "hardwareRequirements": [ + { + "type": "DISPLAY", + "level": "REQUIRED", + "description": "Shows the native selectable list on display glasses" + } + ] +} diff --git a/miniapps/selectable-list-demo/package.json b/miniapps/selectable-list-demo/package.json new file mode 100644 index 0000000000..96048b0fd4 --- /dev/null +++ b/miniapps/selectable-list-demo/package.json @@ -0,0 +1,17 @@ +{ + "name": "selectable-list-demo", + "private": true, + "description": "Minimal MentraOS miniapp demonstrating native selectable lists on G2", + "scripts": { + "dev": "mentra-miniapp dev", + "build": "bun run build.ts", + "pack": "mentra-miniapp pack", + "typecheck": "bun x tsc --noEmit" + }, + "dependencies": { + "@mentra/miniapp": "workspace:*" + }, + "devDependencies": { + "@mentra/miniapp-cli": "workspace:*" + } +} diff --git a/miniapps/selectable-list-demo/src/background/index.ts b/miniapps/selectable-list-demo/src/background/index.ts new file mode 100644 index 0000000000..b7500f8e91 --- /dev/null +++ b/miniapps/selectable-list-demo/src/background/index.ts @@ -0,0 +1,168 @@ +/** + * Background JSContext entry - native selectable list demo. + * + * The glasses render a firmware-owned list. The miniapp receives row clicks + * through session.input.onTouch(...), including selectedItemIndex/name on G2. + */ + +import { + registerMiniapp, + type MiniappSession, + type TouchData, + type UIModule, +} from "@mentra/miniapp/background" + +import type {Channels, ListDemoItem, ListDemoSnapshot} from "../shared/channels" + +const ITEMS: ListDemoItem[] = [ + { + id: "weather", + label: "Weather", + detail: "Shows a short forecast card, then returns to the list.", + }, + { + id: "timer", + label: "Timer", + detail: "Pretends to start a five minute focus timer.", + }, + { + id: "messages", + label: "Messages", + detail: "Represents opening a recent message thread.", + }, + { + id: "music", + label: "Music", + detail: "Represents choosing a playback control surface.", + }, + { + id: "settings", + label: "Settings", + detail: "Represents a nested settings page.", + }, + { + id: "help", + label: "Help", + detail: "Explains that G2 scrolls and highlights the rows natively.", + }, +] + +const ROW_LABELS = ITEMS.map((item, index) => `${index + 1}. ${item.label}`) + +class SelectableListDemo { + private readonly ui: UIModule + private selectedIndex = 0 + private displayMode: "list" | "detail" = "list" + private lastEvent = "List shown. Scroll on G2, tap a row to select it." + private lastSelectedItemName: string | undefined + private returnTimer: ReturnType | undefined + + constructor(private readonly session: MiniappSession) { + this.ui = session.ui as unknown as UIModule + } + + start(): void { + this.ui.onOpen(() => this.pushSnapshot()) + this.ui.on("list-demo:show-list", () => this.showList("List restored from UI.")) + this.ui.on("list-demo:show-detail", () => this.showDetail("Detail restored from UI.")) + + this.session.input.onTouch((event) => this.handleTouch(event)) + this.showList() + } + + private handleTouch(event: TouchData): void { + this.updateSelectedIndex(event) + this.lastSelectedItemName = event.selectedItemName + + if (event.kind === "click") { + this.showDetail(`Selected ${ITEMS[this.selectedIndex]?.label ?? "row"}.`) + return + } + + if (event.kind === "double_click") { + this.showList("Double click returned to the list.") + return + } + + if (event.kind === "scroll_top") { + this.lastEvent = "Reached the top of the native list." + } else if (event.kind === "scroll_bottom") { + this.lastEvent = "Reached the bottom of the native list." + } else { + this.lastEvent = `Touch event: ${event.kind}` + } + this.pushSnapshot() + } + + private updateSelectedIndex(event: TouchData): void { + if (event.selectedItemName) { + const fromName = ROW_LABELS.indexOf(event.selectedItemName) + if (fromName >= 0) { + this.selectedIndex = fromName + return + } + } + + if (typeof event.selectedItemIndex === "number" && Number.isFinite(event.selectedItemIndex)) { + const raw = Math.trunc(event.selectedItemIndex) + this.selectedIndex = Math.min(Math.max(raw, 0), ITEMS.length - 1) + } + } + + private showList(lastEvent = "List shown. Scroll on G2, tap a row to select it."): void { + this.clearReturnTimer() + // G2 recreates the native list with the first row highlighted. Keep the + // demo's logical selection aligned until the next list event says otherwise. + this.selectedIndex = 0 + this.lastSelectedItemName = undefined + this.displayMode = "list" + this.lastEvent = lastEvent + this.session.display.showSelectableList(ROW_LABELS, { + x: 24, + y: 16, + width: 528, + height: 256, + borderWidth: 1, + borderColor: 13, + borderRadius: 8, + paddingLength: 8, + itemWidth: 500, + showSelectionBorder: true, + }) + this.pushSnapshot() + } + + private showDetail(lastEvent = "Detail shown."): void { + this.clearReturnTimer() + const item = ITEMS[this.selectedIndex] ?? ITEMS[0]! + this.displayMode = "detail" + this.lastEvent = lastEvent + this.session.display.showTextWall(`Selected: ${item.label}\n\n${item.detail}\n\nDouble click to return.`) + this.returnTimer = setTimeout(() => this.showList("Auto-returned to the list after showing detail."), 5000) + this.pushSnapshot() + } + + private pushSnapshot(): void { + const selected = ITEMS[this.selectedIndex] ?? ITEMS[0]! + const snapshot: ListDemoSnapshot = { + items: ITEMS, + selectedIndex: this.selectedIndex, + selectedLabel: selected.label, + displayMode: this.displayMode, + lastEvent: this.lastEvent, + lastSelectedItemName: this.lastSelectedItemName, + } + this.ui.send("list-demo:snapshot", snapshot) + } + + private clearReturnTimer(): void { + if (this.returnTimer) { + clearTimeout(this.returnTimer) + this.returnTimer = undefined + } + } +} + +registerMiniapp((session) => { + new SelectableListDemo(session).start() +}) diff --git a/miniapps/selectable-list-demo/src/shared/channels.ts b/miniapps/selectable-list-demo/src/shared/channels.ts new file mode 100644 index 0000000000..3126476d4b --- /dev/null +++ b/miniapps/selectable-list-demo/src/shared/channels.ts @@ -0,0 +1,27 @@ +import type {MentraTyped} from "@mentra/miniapp/ui" + +export interface ListDemoItem { + id: string + label: string + detail: string +} + +export interface ListDemoSnapshot { + items: ListDemoItem[] + selectedIndex: number + selectedLabel: string + displayMode: "list" | "detail" + lastEvent: string + lastSelectedItemName?: string +} + +export interface Channels { + "list-demo:snapshot": ListDemoSnapshot + "list-demo:show-list": Record + "list-demo:show-detail": Record +} + +declare global { + // eslint-disable-next-line no-var + var mentra: MentraTyped +} diff --git a/miniapps/selectable-list-demo/src/ui/index.html b/miniapps/selectable-list-demo/src/ui/index.html new file mode 100644 index 0000000000..55ebe6e911 --- /dev/null +++ b/miniapps/selectable-list-demo/src/ui/index.html @@ -0,0 +1,37 @@ + + + + + + Selectable List Demo + + + +
+
+
+

G2 Native List

+

Selectable List Demo

+
+ Waiting +
+ +
+

Selected row

+

Waiting for background...

+

Open this miniapp on glasses to show the list.

+
+ +
+

Rows sent to glasses

+
    +
    + + +
    + + + diff --git a/miniapps/selectable-list-demo/src/ui/main.ts b/miniapps/selectable-list-demo/src/ui/main.ts new file mode 100644 index 0000000000..e62fb1ea78 --- /dev/null +++ b/miniapps/selectable-list-demo/src/ui/main.ts @@ -0,0 +1,43 @@ +import "../shared/channels" +import type {ListDemoSnapshot} from "../shared/channels" + +const mode = document.getElementById("mode") +const selected = document.getElementById("selected") +const event = document.getElementById("event") +const items = document.getElementById("items") +const showList = document.getElementById("show-list") +const showDetail = document.getElementById("show-detail") + +function render(snapshot: ListDemoSnapshot): void { + if (mode) mode.textContent = snapshot.displayMode === "list" ? "List" : "Detail" + if (selected) selected.textContent = `${snapshot.selectedIndex + 1}. ${snapshot.selectedLabel}` + if (event) { + event.textContent = snapshot.lastSelectedItemName + ? `${snapshot.lastEvent} Firmware item: ${snapshot.lastSelectedItemName}` + : snapshot.lastEvent + } + if (items) { + items.innerHTML = "" + for (const [index, item] of snapshot.items.entries()) { + const row = document.createElement("li") + row.className = index === snapshot.selectedIndex ? "active" : "" + const label = document.createElement("span") + label.textContent = item.label + const detail = document.createElement("small") + detail.textContent = item.detail + row.append(label, detail) + items.appendChild(row) + } + } +} + +showList?.addEventListener("click", () => { + mentra.send("list-demo:show-list", {}) +}) + +showDetail?.addEventListener("click", () => { + mentra.send("list-demo:show-detail", {}) +}) + +mentra.on("list-demo:snapshot", render) +mentra.ready() diff --git a/miniapps/selectable-list-demo/src/ui/styles.css b/miniapps/selectable-list-demo/src/ui/styles.css new file mode 100644 index 0000000000..df32f6d931 --- /dev/null +++ b/miniapps/selectable-list-demo/src/ui/styles.css @@ -0,0 +1,159 @@ +* { + box-sizing: border-box; +} + +html, +body { + width: 100%; + height: 100%; + margin: 0; + background: #f4f6f8; + color: #17202a; + font-family: + Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +body { + overflow: hidden; +} + +button { + font: inherit; +} + +.screen { + min-height: 100%; + display: grid; + grid-template-rows: auto auto 1fr auto; + gap: 12px; + padding: 18px; +} + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.eyebrow { + margin: 0 0 4px; + color: #487089; + font-size: 12px; + font-weight: 700; + text-transform: uppercase; +} + +h1 { + margin: 0; + font-size: 25px; + line-height: 1.05; + letter-spacing: 0; +} + +.mode { + flex: 0 0 auto; + min-width: 70px; + padding: 7px 10px; + border: 1px solid #cbd7df; + border-radius: 8px; + background: white; + color: #27485c; + font-size: 13px; + font-weight: 700; + text-align: center; +} + +.panel { + min-width: 0; + border: 1px solid #d8e0e5; + border-radius: 8px; + background: white; + padding: 14px; + box-shadow: 0 1px 2px rgba(27, 40, 51, 0.05); +} + +.label { + margin: 0 0 8px; + color: #617789; + font-size: 12px; + font-weight: 700; + text-transform: uppercase; +} + +.selected { + margin: 0; + color: #102433; + font-size: 24px; + font-weight: 800; + line-height: 1.1; +} + +.event { + margin: 8px 0 0; + color: #536879; + font-size: 14px; + line-height: 1.35; +} + +.list-panel { + min-height: 0; + overflow: hidden; +} + +.items { + height: 100%; + margin: 0; + padding: 0; + display: grid; + gap: 8px; + overflow: auto; + list-style: none; +} + +.items li { + padding: 10px; + border: 1px solid #e1e7eb; + border-radius: 8px; + background: #f9fbfc; +} + +.items li.active { + border-color: #147d85; + background: #e8f8f8; +} + +.items span { + display: block; + color: #183040; + font-size: 16px; + font-weight: 800; +} + +.items small { + display: block; + margin-top: 3px; + color: #617789; + font-size: 12px; + line-height: 1.3; +} + +.actions { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} + +.actions button { + min-height: 46px; + border: 0; + border-radius: 8px; + background: #147d85; + color: white; + font-size: 15px; + font-weight: 800; +} + +.actions button:last-child { + background: #28384a; +} diff --git a/miniapps/selectable-list-demo/tsconfig.json b/miniapps/selectable-list-demo/tsconfig.json new file mode 100644 index 0000000000..a4c7687075 --- /dev/null +++ b/miniapps/selectable-list-demo/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2020", "DOM"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src/**/*"] +} From cd5682bf1a513232f57ba73836dd53d94e5e6394 Mon Sep 17 00:00:00 2001 From: PhilippeFerreiraDeSousa Date: Fri, 26 Jun 2026 10:49:45 -0700 Subject: [PATCH 4/4] Restore G2 dashboard double-tap fallback --- mobile/jest.setup.js | 1 + .../src/services/LocalMiniappRuntime.ts | 10 +++++ mobile/src/services/MantleManager.test.ts | 40 +++++++++++++++++++ mobile/src/services/MantleManager.ts | 18 +++++++++ 4 files changed, 69 insertions(+) diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 4f82692104..27e01ac43f 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -354,6 +354,7 @@ jest.mock("@mentra/island", () => { forwardEvent: jest.fn(), getAppStatus: jest.fn(() => null), handleRawMessage: jest.fn(), + hasTouchSubscriberForGesture: jest.fn(() => false), initialize: jest.fn(), wireStreamingStatusFanout: jest.fn(), }, diff --git a/mobile/modules/island/src/services/LocalMiniappRuntime.ts b/mobile/modules/island/src/services/LocalMiniappRuntime.ts index 7d95e37694..a35a02e153 100644 --- a/mobile/modules/island/src/services/LocalMiniappRuntime.ts +++ b/mobile/modules/island/src/services/LocalMiniappRuntime.ts @@ -347,6 +347,16 @@ class LocalMiniappRuntime { private constructor() {} + /** + * Returns true when at least one connected miniapp is subscribed to touch + * events broadly or to the given gesture-specific touch stream. + */ + public hasTouchSubscriberForGesture(gestureName: string): boolean { + if (this.streamSubscribers.has(MiniappStreamType.TOUCH_EVENT)) return true + const kind = normalizeTouchGestureKind(gestureName) + return typeof kind === "string" && this.streamSubscribers.has(`${MiniappStreamType.TOUCH_EVENT}:${kind}`) + } + /** * Generate an HMAC-signed local session token for browser fallback auth. * Token format: base64(JSON({userId, packageName, exp})).base64(HMAC-SHA256(payload, secret)) diff --git a/mobile/src/services/MantleManager.test.ts b/mobile/src/services/MantleManager.test.ts index 3cf484896e..96dd50e57a 100644 --- a/mobile/src/services/MantleManager.test.ts +++ b/mobile/src/services/MantleManager.test.ts @@ -8,6 +8,7 @@ import {useCoreStore} from "@/stores/core" import {useDisplayStore} from "@/stores/display" import {isGlassesConnected, useGlassesStore} from "@/stores/glasses" import {SETTINGS, useSettingsStore} from "@/stores/settings" +import {localMiniappRuntime} from "@mentra/island" import {crustModuleMock, emitCrustEvent, resetCrustModuleMock} from "@/test-utils/mockCrustModule" import { bluetoothSdkMock, @@ -193,6 +194,7 @@ describe("MantleManager", () => { resetMantleTestState() jest.clearAllTimers() jest.clearAllMocks() + jest.restoreAllMocks() }) afterAll(() => { @@ -319,6 +321,44 @@ describe("MantleManager", () => { ) }) + it("opens the G2 native dashboard when double tap is not claimed by a local miniapp", async () => { + const hasTouchSubscriber = jest.spyOn(localMiniappRuntime, "hasTouchSubscriberForGesture").mockReturnValue(false) + + emitBluetoothSdkEvent("touch_event", { + type: "touch_event", + deviceModel: "Even Realities G2", + gestureName: "double_tap", + timestamp: 999, + }) + + await waitFor(() => { + expect(hasTouchSubscriber).toHaveBeenCalledWith("double_tap") + expect(bluetoothSdkMock.showDashboard).toHaveBeenCalledTimes(1) + }) + expect(socketComms.sendTouchEvent).toHaveBeenCalledWith( + expect.objectContaining({ + deviceModel: "Even Realities G2", + gestureName: "double_tap", + }), + ) + }) + + it("does not open the G2 native dashboard when a local miniapp claims double tap", async () => { + const hasTouchSubscriber = jest.spyOn(localMiniappRuntime, "hasTouchSubscriberForGesture").mockReturnValue(true) + + emitBluetoothSdkEvent("touch_event", { + type: "touch_event", + deviceModel: "Even Realities G2", + gestureName: "double_tap", + timestamp: 999, + }) + + await waitFor(() => { + expect(hasTouchSubscriber).toHaveBeenCalledWith("double_tap") + }) + expect(bluetoothSdkMock.showDashboard).not.toHaveBeenCalled() + }) + it("syncs notification enablement and blocklist settings to Crust only", async () => { ;(bluetoothSdkMock.updateBluetoothSettings as jest.Mock).mockClear() ;(crustModuleMock.setNotificationConfig as jest.Mock).mockClear() diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index 22343b8a63..17a5770bd8 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -1,4 +1,5 @@ import BluetoothSdk, {ButtonPressEvent, BluetoothStatus, OtaStatus} from "@mentra/bluetooth-sdk-internal" +import type {TouchEvent} from "@mentra/bluetooth-sdk-internal" import CrustModule from "@mentra/crust" import {Asset} from "expo-asset" import * as Calendar from "expo-calendar" @@ -707,6 +708,7 @@ class MantleManager { BluetoothSdk.addListener("touch_event", (event) => { socketComms.sendTouchEvent(event) localMiniappRuntime.forwardEvent("touch_event", event) + void this.handleNativeDashboardTouchFallback(event) }), ) @@ -1353,6 +1355,18 @@ class MantleManager { } } + private async handleNativeDashboardTouchFallback(event: TouchEvent) { + if (event.deviceModel !== "Even Realities G2") return + if (!isDoubleClickGesture(event.gestureName)) return + + const useNativeDashboard = await useSettingsStore.getState().getSetting(SETTINGS.use_native_dashboard.key) + if (!useNativeDashboard) return + if (localMiniappRuntime.hasTouchSubscriberForGesture(event.gestureName)) return + + console.log("MANTLE: G2 double tap opening native dashboard") + BluetoothSdk.showDashboard() + } + public async resetDisplayTimeout() { if (this.clearTextTimeout) { // console.log("MANTLE: canceling pending timeout") @@ -1417,5 +1431,9 @@ class MantleManager { } } +function isDoubleClickGesture(gestureName: string): boolean { + return gestureName === "double_tap" || gestureName === "double_click" +} + const mantle = MantleManager.getInstance() export default mantle