From 473afa7a787af1d21f99b2a6dbf59b7d1a3996b9 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 08:30:16 -0700 Subject: [PATCH 01/80] island: add the island namespace + glasses.wifi facade (first host API) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce the namespaced `island` object — the "(A) host toolkit API" from the Phase-1 contract (OS-1622). Additive: it lives alongside the flat named exports and grows one facade at a time. First facade: `island.glasses.wifi` (scan / connect / forget) over the bluetooth-sdk passthrough — the reference pattern the rest of the (A) domains copy. The wifi screens (scan, connecting) now call `island.glasses.wifi.*` and no longer import `@mentra/bluetooth-sdk` directly (the native-import boundary). status()/onStatus() are deferred to the glasses-store migration — wifi status derives from multiple sources in the glasses connection state, so the read-model lands with that store, not here. connect() propagates bluetooth-sdk's coded errors unchanged so callers keep their error mapping. Jest suite for the facade (by-path import, real code under the CI runner); the @mentra/island mock gains `island`. The bootstrap front door (island.configure/start/stop) and remaining domains follow in later PRs. --- mobile/jest.setup.js | 11 +++++ .../modules/island/src/facades/glassesWifi.ts | 42 +++++++++++++++++++ mobile/modules/island/src/index.ts | 5 +++ mobile/modules/island/src/island.ts | 20 +++++++++ mobile/src/__tests__/glassesWifi.test.ts | 34 +++++++++++++++ mobile/src/app/wifi/connecting.tsx | 4 +- mobile/src/app/wifi/scan.tsx | 6 +-- 7 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 mobile/modules/island/src/facades/glassesWifi.ts create mode 100644 mobile/modules/island/src/island.ts create mode 100644 mobile/src/__tests__/glassesWifi.test.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 12993d2ca9..795ff47566 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -249,6 +249,17 @@ jest.mock("@mentra/island", () => { return { __esModule: true, + // The namespaced (A) host API. Mirrors the real `island` object; facades are + // jest.fn()s so screen tests can assert delegation without native btsdk. + island: { + glasses: { + wifi: { + scan: jest.fn(() => Promise.resolve([])), + connect: jest.fn(() => Promise.resolve()), + forget: jest.fn(() => Promise.resolve()), + }, + }, + }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), clearInterval: jest.fn((id) => clearInterval(id)), diff --git a/mobile/modules/island/src/facades/glassesWifi.ts b/mobile/modules/island/src/facades/glassesWifi.ts new file mode 100644 index 0000000000..19babe3df3 --- /dev/null +++ b/mobile/modules/island/src/facades/glassesWifi.ts @@ -0,0 +1,42 @@ +/** + * glasses.wifi facade — the first typed island facade over the bluetooth-sdk + * passthrough, and the reference pattern the rest of the (A) host API copies. + * + * A facade wraps the raw `@mentra/bluetooth-sdk` surface into a small, typed, + * device-agnostic API (the `doX()` action shape here; `getX()`/`onX()` read-models + * come with domains that own their state). The host UI calls `island.glasses.wifi.*` + * instead of importing bluetooth-sdk directly — that's the native-import boundary + * the toolkit is built around. + * + * Scope note: this is actions only (`scan`/`connect`/`forget`). The wifi *status* + * read-model (`status()`/`onStatus()`) is deliberately deferred — it's derived from + * multiple sources in the glasses connection state (the `wifi_status_change` event + * *and* legacy connection-info fields), so it lands with the glasses-store migration, + * not here. `connect()` propagates bluetooth-sdk's coded errors unchanged so callers + * keep their existing error mapping. + */ +import BluetoothSdk from "@mentra/bluetooth-sdk" +import type {WifiSearchResult} from "@mentra/bluetooth-sdk" + +export type {WifiSearchResult} + +export const glassesWifi = { + /** Scan for nearby wifi networks. Request/response — resolves with the results. */ + scan(): Promise { + return BluetoothSdk.requestWifiScan() + }, + + /** + * Send wifi credentials to the glasses. Resolves on success; rejects with the + * bluetooth-sdk coded error (`bluetooth_powered_off`, `request_timeout`, …) on + * failure, propagated unchanged for the caller to map. + */ + async connect(ssid: string, password: string): Promise { + await BluetoothSdk.sendWifiCredentials(ssid, password) + }, + + /** Forget a saved network on the glasses. */ + async forget(ssid: string): Promise { + await BluetoothSdk.forgetWifiNetwork(ssid) + }, +} diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 09a8e73600..b05f4213db 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -77,6 +77,11 @@ export { type ConnectButtonAction, } from "./services/ConnectionCoordinator" +// The namespaced OEM-facing toolkit API (the "(A) host API"). Additive — grows one +// facade at a time alongside the flat exports below. See ./island. +export {island} from "./island" +export type {WifiSearchResult} from "./facades/glassesWifi" + // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. // This is the escape hatch for what no island facade models yet (OTA, raw device diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts new file mode 100644 index 0000000000..7a9e8efe6a --- /dev/null +++ b/mobile/modules/island/src/island.ts @@ -0,0 +1,20 @@ +/** + * `island` — the namespaced OEM-facing toolkit API (the "(A) host API" from the + * Phase-1 contract). Host UI calls `island..()`; each domain is a + * typed facade over the runtime. + * + * This is additive: it grows one facade at a time and lives *alongside* the flat + * named exports in `index.ts` during the migration. The flat exports (and the + * `BluetoothSdk` passthrough) stay until every screen has moved onto `island.*`. + * + * First member: `island.glasses.wifi`. The bootstrap front door + * (`island.configure`/`start`/`stop`) and the remaining domains attach here in + * follow-up PRs. + */ +import {glassesWifi} from "./facades/glassesWifi" + +export const island = { + glasses: { + wifi: glassesWifi, + }, +} diff --git a/mobile/src/__tests__/glassesWifi.test.ts b/mobile/src/__tests__/glassesWifi.test.ts new file mode 100644 index 0000000000..43b428c0ff --- /dev/null +++ b/mobile/src/__tests__/glassesWifi.test.ts @@ -0,0 +1,34 @@ +// Imports the real glasses.wifi facade by path (not via "@mentra/island", which +// jest mocks) so the actual delegation runs under the mobile jest CI runner. +// @mentra/bluetooth-sdk is globally mocked in jest.setup.js — the facade calls it. +import BluetoothSdk from "@mentra/bluetooth-sdk" +import {glassesWifi} from "../../modules/island/src/facades/glassesWifi" + +describe("glassesWifi facade", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it("scan() delegates to requestWifiScan and returns its results", async () => { + const networks = [{ssid: "home", requiresPassword: true, signalStrength: -40}] + ;(BluetoothSdk.requestWifiScan as jest.Mock).mockResolvedValueOnce(networks) + await expect(glassesWifi.scan()).resolves.toEqual(networks) + expect(BluetoothSdk.requestWifiScan).toHaveBeenCalledTimes(1) + }) + + it("connect() delegates to sendWifiCredentials with ssid + password", async () => { + await glassesWifi.connect("home", "hunter2") + expect(BluetoothSdk.sendWifiCredentials).toHaveBeenCalledWith("home", "hunter2") + }) + + it("connect() propagates the bluetooth-sdk coded error unchanged", async () => { + const err = Object.assign(new Error("bt off"), {code: "bluetooth_powered_off"}) + ;(BluetoothSdk.sendWifiCredentials as jest.Mock).mockRejectedValueOnce(err) + await expect(glassesWifi.connect("home", "x")).rejects.toBe(err) + }) + + it("forget() delegates to forgetWifiNetwork", async () => { + await glassesWifi.forget("home") + expect(BluetoothSdk.forgetWifiNetwork).toHaveBeenCalledWith("home") + }) +}) diff --git a/mobile/src/app/wifi/connecting.tsx b/mobile/src/app/wifi/connecting.tsx index f362959324..f05e93256e 100644 --- a/mobile/src/app/wifi/connecting.tsx +++ b/mobile/src/app/wifi/connecting.tsx @@ -1,4 +1,4 @@ -import BluetoothSdk from "@mentra/bluetooth-sdk" +import {island} from "@mentra/island" import {useLocalSearchParams} from "expo-router" import {useEffect, useState, useCallback} from "react" import {ActivityIndicator, View} from "react-native" @@ -81,7 +81,7 @@ export default function WifiConnectingScreen() { const attemptConnection = async () => { try { console.log("Attempting to send wifi credentials to Core", ssid, password) - await BluetoothSdk.sendWifiCredentials(ssid, password) + await island.glasses.wifi.connect(ssid, password) // Save credentials ONLY on successful connection if checkbox was checked. // This ensures we never save wrong passwords. diff --git a/mobile/src/app/wifi/scan.tsx b/mobile/src/app/wifi/scan.tsx index 0620a061e7..3f5d38da68 100644 --- a/mobile/src/app/wifi/scan.tsx +++ b/mobile/src/app/wifi/scan.tsx @@ -1,4 +1,4 @@ -import BluetoothSdk, {WifiSearchResult} from "@mentra/bluetooth-sdk" +import {island, type WifiSearchResult} from "@mentra/island" import {useFocusEffect} from "expo-router" import {useCallback, useEffect, useMemo, useState} from "react" import {ActivityIndicator, ScrollView, TouchableOpacity, View} from "react-native" @@ -81,7 +81,7 @@ export default function WifiScanScreen() { setNetworks([]) try { - const scanResults = await BluetoothSdk.requestWifiScan() + const scanResults = await island.glasses.wifi.scan() console.log(`WIFI_SCAN: Received ${scanResults.length} WiFi scan results`) setNetworks( scanResults.map((network) => ({ @@ -118,7 +118,7 @@ export default function WifiScanScreen() { onPress: async () => { try { console.log(`WIFI_SCAN: Forgetting network: ${selectedNetwork.ssid}`) - await BluetoothSdk.forgetWifiNetwork(selectedNetwork.ssid) + await island.glasses.wifi.forget(selectedNetwork.ssid) // Also remove from local saved credentials WifiCredentialsService.removeCredentials(selectedNetwork.ssid) setSavedNetworks((prev) => prev.filter((ssid) => ssid !== selectedNetwork.ssid)) From ff4f0fac8a89bf852b651717bf424bcb5b60324e Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 09:29:33 -0700 Subject: [PATCH 02/80] island: add bootstrap front door + glasses.display.mirror read facade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more pieces of the (A) host API on the island namespace. Bootstrap front door — island.configure({auth, config?, analytics?}) + start() + stop(), backed by an island config singleton (runtime/bootstrap.ts). The host now hands island its auth provider at boot (MantleManager.init). This is ADDITIVE: the three transitional config seams (configureRuntime/Launcher/Island) still wire the host adapters during the migration and collapse into this front door only as each domain lands in island. island.display.mirror — island's own read-model of the latest processed display event (current() / onMirror(cb)), fed in PARALLEL with the legacy host useDisplayStore from both display paths (LocalDisplayManager + the cloud path in SocketComms). This is the safe "facade" half. The "callback" half — flipping the preview UI onto onMirror, adding view tracking, deleting useDisplayStore + the setDisplayEvent adapter, and migrating the display tests (which are coupled to v1 SocketComms + MantleManager) — is its own focused PR. Jest: @mentra/island mock gains the new members; by-path suites cover the bootstrap lifecycle + the mirror read-model. --- mobile/jest.setup.js | 14 +++- .../island/src/facades/displayMirror.ts | 40 ++++++++++ mobile/modules/island/src/index.ts | 8 ++ mobile/modules/island/src/island.ts | 13 ++- .../modules/island/src/runtime/bootstrap.ts | 79 +++++++++++++++++++ .../src/services/LocalDisplayManager.ts | 3 + .../__tests__/islandBootstrapMirror.test.ts | 52 ++++++++++++ mobile/src/services/MantleManager.ts | 17 ++++ mobile/src/services/SocketComms.ts | 4 +- 9 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 mobile/modules/island/src/facades/displayMirror.ts create mode 100644 mobile/modules/island/src/runtime/bootstrap.ts create mode 100644 mobile/src/__tests__/islandBootstrapMirror.test.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 795ff47566..9bb987c2ef 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -249,9 +249,12 @@ jest.mock("@mentra/island", () => { return { __esModule: true, - // The namespaced (A) host API. Mirrors the real `island` object; facades are - // jest.fn()s so screen tests can assert delegation without native btsdk. + // The namespaced (A) host API. Mirrors the real `island` object; members are + // jest.fn()s so host/screen tests can assert delegation without native btsdk. island: { + configure: jest.fn(), + start: jest.fn(() => Promise.resolve()), + stop: jest.fn(() => Promise.resolve()), glasses: { wifi: { scan: jest.fn(() => Promise.resolve([])), @@ -259,6 +262,13 @@ jest.mock("@mentra/island", () => { forget: jest.fn(() => Promise.resolve()), }, }, + display: { + mirror: { + ingest: jest.fn(), + current: jest.fn(() => null), + onMirror: jest.fn(() => () => {}), + }, + }, }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), diff --git a/mobile/modules/island/src/facades/displayMirror.ts b/mobile/modules/island/src/facades/displayMirror.ts new file mode 100644 index 0000000000..e94a4e9014 --- /dev/null +++ b/mobile/modules/island/src/facades/displayMirror.ts @@ -0,0 +1,40 @@ +/** + * glasses display mirror — island's read-model of the latest display event sent + * to the glasses, for a phone-side preview (`island.display.mirror`). + * + * This is the additive "facade" half of inverting the mirror: island owns this + * read-model and is fed every processed display event (from the local miniapp + * display path and the cloud display path), in PARALLEL with the legacy host + * `useDisplayStore`. Nothing has to read island yet, so no consumer/test churn. + * + * The follow-up "callback" half flips the preview UI (`GlassesDisplayMirror`, + * `SimulatedGlassesControls`) onto `onMirror`/`current`, adds view tracking here, + * deletes the host `useDisplayStore` + the `setDisplayEvent` RuntimeHooks adapter, + * and migrates the display tests. That part is test-coupled to v1 `SocketComms` + * and `MantleManager`, so it's isolated to its own PR. + */ +export type DisplayMirrorEvent = Record & {view?: string} + +let latestEvent: DisplayMirrorEvent | null = null +const listeners = new Set<(event: DisplayMirrorEvent) => void>() + +export const displayMirror = { + /** Feed a processed display event (the object `DisplayProcessor` produces). */ + ingest(event: DisplayMirrorEvent): void { + latestEvent = event + for (const cb of listeners) cb(event) + }, + + /** The most recent processed display event, or null before the first one. */ + current(): DisplayMirrorEvent | null { + return latestEvent + }, + + /** Subscribe to display events; returns an unsubscribe. */ + onMirror(cb: (event: DisplayMirrorEvent) => void): () => void { + listeners.add(cb) + return () => { + listeners.delete(cb) + } + }, +} diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index b05f4213db..98bae19831 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -81,6 +81,14 @@ export { // facade at a time alongside the flat exports below. See ./island. export {island} from "./island" export type {WifiSearchResult} from "./facades/glassesWifi" +export type {DisplayMirrorEvent} from "./facades/displayMirror" +export type { + IslandConfigureOptions, + IslandAuth, + IslandConfigValues, + IslandAnalytics, + SubjectTokenType, +} from "./runtime/bootstrap" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 7a9e8efe6a..be7f4bf02b 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -6,15 +6,20 @@ * This is additive: it grows one facade at a time and lives *alongside* the flat * named exports in `index.ts` during the migration. The flat exports (and the * `BluetoothSdk` passthrough) stay until every screen has moved onto `island.*`. - * - * First member: `island.glasses.wifi`. The bootstrap front door - * (`island.configure`/`start`/`stop`) and the remaining domains attach here in - * follow-up PRs. */ +import {configure, start, stop} from "./runtime/bootstrap" import {glassesWifi} from "./facades/glassesWifi" +import {displayMirror} from "./facades/displayMirror" export const island = { + /** Front door — hand island auth + config, then start/stop the runtime. */ + configure, + start, + stop, glasses: { wifi: glassesWifi, }, + display: { + mirror: displayMirror, + }, } diff --git a/mobile/modules/island/src/runtime/bootstrap.ts b/mobile/modules/island/src/runtime/bootstrap.ts new file mode 100644 index 0000000000..107e44ebd7 --- /dev/null +++ b/mobile/modules/island/src/runtime/bootstrap.ts @@ -0,0 +1,79 @@ +/** + * Bootstrap — island's single front door (`island.configure` / `start` / `stop`). + * + * The Phase-1 contract's end state is: the host hands island its auth + config in + * ONE call and island owns the rest. This is the additive first step of that: a + * config holder the host populates, plus the lifecycle verbs. It lives ALONGSIDE + * the three transitional config seams (`configureRuntime` / `configureLauncher` / + * `configureIsland`) — those still carry the host adapters during the migration + * and collapse into this front door only as each domain lands in island. + * + * So today `configure()` stores auth/config/analytics for island to read, and + * `start()`/`stop()` mark the lifecycle; the host's existing boot is unchanged. + * Future domain PRs route their consumers through `getAuth()` / `getConfigValues()` + * / `getAnalytics()` and retire the matching adapter. + */ + +export type SubjectTokenType = "supabase" | "authing" | (string & {}) + +export interface IslandAuth { + /** Returns the host's current (auto-refreshed) subject token for the backend. */ + getSubjectToken: () => Promise<{token: string; type: SubjectTokenType}> +} + +export interface IslandConfigValues { + /** cloud-v2 core service base URL (defaults resolved by the cloud client). */ + coreUrl?: string + /** cloud-v2 runtime service base URL. */ + runtimeUrl?: string + /** OEM identifier (Mentra is OEM #0); reserved for OEM auth/telemetry. */ + oemId?: string +} + +export type IslandAnalytics = (event: string, props?: Record) => void + +export interface IslandConfigureOptions { + /** REQUIRED — the only must-have seam. The host owns login; island owns the rest. */ + auth: IslandAuth + config?: IslandConfigValues + analytics?: IslandAnalytics +} + +let options: IslandConfigureOptions | null = null +let started = false + +/** Hand island its auth + config + analytics. Call once, before `start()`. */ +export function configure(opts: IslandConfigureOptions): void { + options = opts +} + +/** Mark the runtime started. Idempotent. */ +export async function start(): Promise { + if (started) return + if (!options) { + throw new Error("island.start() called before island.configure()") + } + started = true +} + +/** Tear down. Idempotent. */ +export async function stop(): Promise { + started = false +} + +export function isStarted(): boolean { + return started +} + +/** The host-supplied auth provider, or null if not configured yet. */ +export function getAuth(): IslandAuth | null { + return options?.auth ?? null +} + +export function getConfigValues(): IslandConfigValues { + return options?.config ?? {} +} + +export function getAnalytics(): IslandAnalytics | null { + return options?.analytics ?? null +} diff --git a/mobile/modules/island/src/services/LocalDisplayManager.ts b/mobile/modules/island/src/services/LocalDisplayManager.ts index bc9a58e293..7481fa0274 100644 --- a/mobile/modules/island/src/services/LocalDisplayManager.ts +++ b/mobile/modules/island/src/services/LocalDisplayManager.ts @@ -20,6 +20,7 @@ */ import displayProcessor from "./DisplayProcessor" +import {displayMirror} from "../facades/displayMirror" import {getRuntimeHooks} from "../runtime/config" import {BgTimer} from "../utils/timers" @@ -388,6 +389,8 @@ class LocalDisplayManager { }) } getRuntimeHooks().setDisplayEvent?.(JSON.stringify(processedEvent)) + // Feed island's own mirror read-model in parallel with the legacy host store. + displayMirror.ingest(processedEvent) } catch (err) { console.error(`${LOG_TAG}: native display failed:`, err) } diff --git a/mobile/src/__tests__/islandBootstrapMirror.test.ts b/mobile/src/__tests__/islandBootstrapMirror.test.ts new file mode 100644 index 0000000000..ab056ce1ef --- /dev/null +++ b/mobile/src/__tests__/islandBootstrapMirror.test.ts @@ -0,0 +1,52 @@ +// Imports the real bootstrap + displayMirror by path (not via "@mentra/island", +// which jest mocks) so the actual logic runs under the mobile jest CI runner. +import * as bootstrap from "../../modules/island/src/runtime/bootstrap" +import {displayMirror} from "../../modules/island/src/facades/displayMirror" + +describe("island bootstrap front door", () => { + beforeEach(async () => { + await bootstrap.stop() + }) + + it("start() before configure() rejects (this test runs first, nothing configured yet)", async () => { + await expect(bootstrap.start()).rejects.toThrow(/configure/) + }) + + it("configure() stores auth/config/analytics for island to read", () => { + const auth = {getSubjectToken: async () => ({token: "t", type: "supabase" as const})} + const analytics = jest.fn() + bootstrap.configure({auth, config: {coreUrl: "http://core", oemId: "mentra"}, analytics}) + expect(bootstrap.getAuth()).toBe(auth) + expect(bootstrap.getConfigValues()).toEqual({coreUrl: "http://core", oemId: "mentra"}) + expect(bootstrap.getAnalytics()).toBe(analytics) + }) + + it("start()/stop() toggle isStarted (idempotent)", async () => { + bootstrap.configure({auth: {getSubjectToken: async () => ({token: "t", type: "supabase"})}}) + await bootstrap.start() + await bootstrap.start() + expect(bootstrap.isStarted()).toBe(true) + await bootstrap.stop() + expect(bootstrap.isStarted()).toBe(false) + }) +}) + +describe("island.display.mirror read-model", () => { + it("ingest() sets current() and notifies subscribers", () => { + const seen: unknown[] = [] + const unsub = displayMirror.onMirror((e) => seen.push(e)) + const event = {view: "main", layout: {layoutType: "text_wall", text: "hi"}} + displayMirror.ingest(event) + expect(displayMirror.current()).toBe(event) + expect(seen).toEqual([event]) + unsub() + }) + + it("onMirror() unsubscribe stops delivery", () => { + const cb = jest.fn() + const unsub = displayMirror.onMirror(cb) + unsub() + displayMirror.ingest({view: "dashboard"}) + expect(cb).not.toHaveBeenCalled() + }) +}) diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index ee34ccc6ae..97a21dbaab 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -28,6 +28,7 @@ import { configureRuntime, getRuntimeHooks, displayProcessor, + island, localMiniappRuntime, localSttFallbackCoordinator, micStateCoordinator, @@ -165,6 +166,22 @@ class MantleManager { } this.initialized = true + // Island front door: hand island the host's auth provider, then mark it + // started. Additive — the `configure*` adapter seams below still wire the + // host services during the migration; consumers move onto this incrementally. + island.configure({ + auth: { + getSubjectToken: async () => { + const res = await mentraAuth.getSession() + if (res.is_error() || !res.value.token) { + throw new Error("island.configure: no session token available") + } + return {token: res.value.token, type: "supabase"} + }, + }, + }) + void island.start() + // iOS: require a second swipe across the bottom edge to invoke the Home // indicator / app switcher, so users don't accidentally background the // app mid-glasses-session. No-op on Android. diff --git a/mobile/src/services/SocketComms.ts b/mobile/src/services/SocketComms.ts index 0021e1ed03..e2cb03931a 100644 --- a/mobile/src/services/SocketComms.ts +++ b/mobile/src/services/SocketComms.ts @@ -1,6 +1,6 @@ import {type RgbLedControlResponseEvent, type TouchEvent} from "@mentra/bluetooth-sdk" import BluetoothSdk from "@mentra/bluetooth-sdk-internal" -import {displayProcessor, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" +import {displayProcessor, island, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" import audioPlaybackService from "@/services/AudioPlaybackService" import mantle from "@/services/MantleManager" @@ -487,6 +487,8 @@ class SocketComms { BluetoothSdk.displayEvent(processedEvent) const displayEventStr = JSON.stringify(processedEvent) useDisplayStore.getState().setDisplayEvent(displayEventStr) + // Feed island's own mirror read-model in parallel with the legacy host store. + island.display.mirror.ingest(processedEvent) } private handle_set_location_tier(msg: any) { From 17e9ed5740e133e2e622b2bc99a2d23545cbd606 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 09:56:11 -0700 Subject: [PATCH 03/80] island: move the glasses device-state store into island (+ wifi status facade) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The glasses store is the source of device state (connection, wifi, battery, OTA, hotspot) that every device facade needs, so it moves into island — and the wifi status read-model that was blocked on it now lands too. - stores/glasses.ts -> modules/island/src/stores/glasses.ts. btsdk type imports switch to the relative _internal path (standalone-build-safe, like the passthrough); connection predicates come from island's GlassesReadiness. - @/stores/glasses becomes a thin re-export shim — the ~46 existing importers are unchanged (the escape hatch from the Phase-1 plan). Also surfaced as island.glassesStore (the raw store) for the app to keep using directly. - jest: the @mentra/island mock provides the REAL store via jest.requireActual so every consumer's tests keep their real setState/getState/subscribe behavior. - island.glasses.wifi gains status() / onStatus() reading the island-owned store — the wifi facade is now complete. Verified: typecheck 0 new errors across all 46 consumers; jest 265 pass (the real store runs in-test via requireActual). The island standalone build uses the proven relative-_internal pattern; CI confirms it (local build is blocked only by the unrelated cloud-v2 zod dep not being installed in this checkout). --- mobile/jest.setup.js | 10 + .../modules/island/src/facades/glassesWifi.ts | 25 +- mobile/modules/island/src/index.ts | 13 + mobile/modules/island/src/island.ts | 7 + mobile/modules/island/src/stores/glasses.ts | 309 +++++++++++++++++ mobile/src/__tests__/glassesWifi.test.ts | 17 + mobile/src/stores/glasses.ts | 325 +----------------- 7 files changed, 388 insertions(+), 318 deletions(-) create mode 100644 mobile/modules/island/src/stores/glasses.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 9bb987c2ef..bc536b9688 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -233,6 +233,10 @@ jest.mock("@dr.pogodin/react-native-fs", () => ({ // (react-native-share, expo-battery/clipboard/location, etc.). Tests that // only need a handful of exports get stubs here; specific tests can override. jest.mock("@mentra/island", () => { + // The glasses store moved into island; tests + the @/stores/glasses shim need its + // REAL behavior (setState/getState/subscribe), so pull the actual store in. It's + // pure (zustand + type-only btsdk imports), so it loads cleanly under the mock. + const realGlasses = jest.requireActual("./modules/island/src/stores/glasses") const appStatusState = { apps: [], refresh: jest.fn(), @@ -249,6 +253,9 @@ jest.mock("@mentra/island", () => { return { __esModule: true, + // Real glasses store + its selectors/helpers (useGlassesStore, selectors, + // waitForGlassesState, getGlasesInfoPartial, getGlassesSystemTimeMs, predicates). + ...realGlasses, // The namespaced (A) host API. Mirrors the real `island` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. island: { @@ -260,6 +267,8 @@ jest.mock("@mentra/island", () => { scan: jest.fn(() => Promise.resolve([])), connect: jest.fn(() => Promise.resolve()), forget: jest.fn(() => Promise.resolve()), + status: jest.fn(() => ({state: "disconnected"})), + onStatus: jest.fn(() => () => {}), }, }, display: { @@ -269,6 +278,7 @@ jest.mock("@mentra/island", () => { onMirror: jest.fn(() => () => {}), }, }, + glassesStore: realGlasses.useGlassesStore, }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), diff --git a/mobile/modules/island/src/facades/glassesWifi.ts b/mobile/modules/island/src/facades/glassesWifi.ts index 19babe3df3..155ac3fd14 100644 --- a/mobile/modules/island/src/facades/glassesWifi.ts +++ b/mobile/modules/island/src/facades/glassesWifi.ts @@ -8,17 +8,16 @@ * instead of importing bluetooth-sdk directly — that's the native-import boundary * the toolkit is built around. * - * Scope note: this is actions only (`scan`/`connect`/`forget`). The wifi *status* - * read-model (`status()`/`onStatus()`) is deliberately deferred — it's derived from - * multiple sources in the glasses connection state (the `wifi_status_change` event - * *and* legacy connection-info fields), so it lands with the glasses-store migration, - * not here. `connect()` propagates bluetooth-sdk's coded errors unchanged so callers - * keep their existing error mapping. + * `connect()` propagates bluetooth-sdk's coded errors unchanged so callers keep + * their existing error mapping. The `status()`/`onStatus()` read-model reads the + * glasses-state store island now owns (wifi status is derived there from the + * device's connection info). */ import BluetoothSdk from "@mentra/bluetooth-sdk" -import type {WifiSearchResult} from "@mentra/bluetooth-sdk" +import type {WifiSearchResult, WifiStatus} from "@mentra/bluetooth-sdk" +import {useGlassesStore} from "../stores/glasses" -export type {WifiSearchResult} +export type {WifiSearchResult, WifiStatus} export const glassesWifi = { /** Scan for nearby wifi networks. Request/response — resolves with the results. */ @@ -39,4 +38,14 @@ export const glassesWifi = { async forget(ssid: string): Promise { await BluetoothSdk.forgetWifiNetwork(ssid) }, + + /** The glasses' current wifi connection status (snapshot). */ + status(): WifiStatus { + return useGlassesStore.getState().wifi + }, + + /** Subscribe to wifi-status changes; returns an unsubscribe. */ + onStatus(cb: (status: WifiStatus) => void): () => void { + return useGlassesStore.subscribe((s) => s.wifi, cb) + }, } diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 98bae19831..65313b3566 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -90,6 +90,19 @@ export type { SubjectTokenType, } from "./runtime/bootstrap" +// Glasses device-state store — moved into island so island owns device state. +// Re-exported through the host's @/stores/glasses shim so the app keeps its +// imports. Also surfaced as island.glassesStore (the escape hatch). Predicates +// (isGlassesConnected/…) come from GlassesReadiness above, not re-exported here. +export { + useGlassesStore, + selectGlassesConnected, + selectGlassesReady, + getGlasesInfoPartial, + getGlassesSystemTimeMs, + waitForGlassesState, +} from "./stores/glasses" + // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. // This is the escape hatch for what no island facade models yet (OTA, raw device diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index be7f4bf02b..1ccf794fcc 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -10,6 +10,7 @@ import {configure, start, stop} from "./runtime/bootstrap" import {glassesWifi} from "./facades/glassesWifi" import {displayMirror} from "./facades/displayMirror" +import {useGlassesStore} from "./stores/glasses" export const island = { /** Front door — hand island auth + config, then start/stop the runtime. */ @@ -22,4 +23,10 @@ export const island = { display: { mirror: displayMirror, }, + /** + * Escape hatch — the raw glasses device-state zustand store, exposed so the + * Mentra app keeps using it directly instead of rewriting every screen onto + * typed facades. Prefer a facade where one exists. + */ + glassesStore: useGlassesStore, } diff --git a/mobile/modules/island/src/stores/glasses.ts b/mobile/modules/island/src/stores/glasses.ts new file mode 100644 index 0000000000..ef199d09ee --- /dev/null +++ b/mobile/modules/island/src/stores/glasses.ts @@ -0,0 +1,309 @@ +import type { + GlassesConnectionStatus, + GlassesStatus, + HotspotStatus, + OtaProgress, + OtaStatus, + OtaUpdateInfo, + WifiStatus, +} from "../../../bluetooth-sdk/build/_internal" +import {isGlassesConnected, isGlassesLinkLayerBusy, isGlassesReady} from "../services/GlassesReadiness" +import {create} from "zustand" +import {subscribeWithSelector} from "zustand/middleware" + +// Re-export the connection predicates (single source of truth lives in +// GlassesReadiness) so importers of this store keep getting them. +export {isGlassesConnected, isGlassesLinkLayerBusy, isGlassesReady} + +export const selectGlassesConnected = (state: {connection: GlassesConnectionStatus}) => + isGlassesConnected(state.connection) + +export const selectGlassesReady = (state: {connection: GlassesConnectionStatus}) => isGlassesReady(state.connection) + +interface GlassesState extends GlassesStatus { + systemTimeMs: number + wifiStatusKnown: boolean + setGlassesInfo: (info: GlassesInfoUpdate) => void + setBatteryInfo: (batteryLevel: number, charging: boolean, caseBatteryLevel: number, caseCharging: boolean) => void + setWifiInfo: (connected: boolean, ssid: string) => void + setHotspotInfo: (enabled: boolean, ssid: string, password: string, ip: string) => void + // OTA methods + otaStatus: OtaStatus | null + setOtaStatus: (status: OtaStatus | null) => void + setOtaUpdateAvailable: (info: OtaUpdateInfo | null) => void + setOtaProgress: (progress: OtaProgress | null) => void + setOtaInProgress: (inProgress: boolean) => void + setMtkUpdatedThisSession: (updated: boolean) => void + clearOtaState: () => void + reset: () => void + mtkUpdatedThisSession: boolean +} + +type LegacyWifiFields = { + wifiConnected?: boolean + wifiSsid?: string + wifiLocalIp?: string +} + +type LegacyHotspotFields = { + hotspotEnabled?: boolean + hotspotSsid?: string + hotspotPassword?: string + hotspotGatewayIp?: string + hotspotLocalIp?: string +} + +type GlassesInfoUpdate = Partial & LegacyWifiFields & LegacyHotspotFields + +function wifiFromLegacyFields(info: LegacyWifiFields): WifiStatus | null { + if (info.wifiConnected === true) { + const ssid = info.wifiSsid?.trim() + const localIp = info.wifiLocalIp?.trim() + return ssid ? {state: "connected", ssid, ...(localIp ? {localIp} : {})} : null + } + if (info.wifiConnected === false) { + return {state: "disconnected"} + } + return null +} + +function hotspotFromLegacyFields(info: LegacyHotspotFields): HotspotStatus | null { + if (info.hotspotEnabled === true) { + const ssid = info.hotspotSsid?.trim() + const password = info.hotspotPassword?.trim() + const localIp = (info.hotspotGatewayIp ?? info.hotspotLocalIp)?.trim() + return ssid && password && localIp ? {state: "enabled", ssid, password, localIp} : null + } + if (info.hotspotEnabled === false) { + return {state: "disabled"} + } + return null +} + +export const getGlasesInfoPartial = (state: GlassesStatus) => { + const wifi = state.wifi + const connected = isGlassesConnected(state.connection) + return { + batteryLevel: state.batteryLevel, + charging: state.charging, + caseBatteryLevel: state.caseBatteryLevel, + caseCharging: state.caseCharging, + connected, + wifiConnected: wifi.state === "connected", + wifiSsid: wifi.state === "connected" ? wifi.ssid : "", + deviceModel: state.deviceModel, + // Cloud GlassesInfo uses modelName, map from deviceModel so the cloud + // knows which device is connected when it receives connection state updates + modelName: state.deviceModel || null, + } +} + +interface GlassesStore extends GlassesStatus { + systemTimeMs: number + mtkUpdatedThisSession: boolean + wifiStatusKnown: boolean + otaStatus: OtaStatus | null +} + +const initialState: GlassesStore = { + // state: + connection: {state: "disconnected"}, + micEnabled: false, + bluetoothClassicConnected: false, + signalStrength: -1, + signalStrengthUpdatedAt: 0, + voiceActivityDetectionEnabled: true, + // device info + deviceModel: "", + androidVersion: "", + firmwareVersion: "", + bluetoothMacAddress: "", + leftMacAddress: "", + rightMacAddress: "", + buildNumber: "", + systemTimeMs: 0, + otaVersionUrl: "", + appVersion: "", + bluetoothName: "", + serialNumber: "", + style: "", + color: "", + mtkFirmwareVersion: "", + besFirmwareVersion: "", + // wifi info + wifi: {state: "disconnected"}, + wifiStatusKnown: false, + // battery info + batteryLevel: -1, + charging: false, + caseBatteryLevel: -1, + caseCharging: false, + caseOpen: false, + caseRemoved: true, + // hotspot info + hotspot: {state: "disabled"}, + // OTA update info + otaStatus: null, + otaUpdateAvailable: null, + otaProgress: null, + otaInProgress: false, + mtkUpdatedThisSession: false, + // ring: + controllerConnected: false, + controllerFullyBooted: false, + controllerMacAddress: "", + controllerBatteryLevel: -1, + controllerSignalStrength: -1, +} + +export const useGlassesStore = create()( + subscribeWithSelector((set) => ({ + ...initialState, + + setGlassesInfo: (info) => + set((state) => { + const { + wifiConnected, + wifiSsid, + wifiLocalIp, + hotspotEnabled, + hotspotSsid, + hotspotPassword, + hotspotGatewayIp, + hotspotLocalIp, + wifi, + hotspot, + ...sdkInfo + } = info + const wifiUpdate = wifi ?? wifiFromLegacyFields({wifiConnected, wifiSsid, wifiLocalIp}) + const hotspotUpdate = + hotspot ?? + hotspotFromLegacyFields({hotspotEnabled, hotspotSsid, hotspotPassword, hotspotGatewayIp, hotspotLocalIp}) + const hasWifiInfoUpdate = + Object.prototype.hasOwnProperty.call(info, "wifi") || + Object.prototype.hasOwnProperty.call(info, "wifiConnected") || + Object.prototype.hasOwnProperty.call(info, "wifiSsid") || + Object.prototype.hasOwnProperty.call(info, "wifiLocalIp") + const next = { + ...state, + ...sdkInfo, + ...(wifiUpdate ? {wifi: wifiUpdate} : {}), + ...(hotspotUpdate ? {hotspot: hotspotUpdate} : {}), + ...(hasWifiInfoUpdate ? {wifiStatusKnown: true} : {}), + } + if (!isGlassesConnected(next.connection)) { + next.wifiStatusKnown = false + } + return next + }), + + setBatteryInfo: (batteryLevel, charging, caseBatteryLevel, caseCharging) => + set({ + batteryLevel, + charging, + caseBatteryLevel, + caseCharging, + }), + + setWifiInfo: (connected, ssid) => + set(() => { + const trimmedSsid = ssid.trim() + if (connected && !trimmedSsid) { + return {} + } + const wifi: WifiStatus = connected ? {state: "connected", ssid: trimmedSsid} : {state: "disconnected"} + return { + wifi, + wifiStatusKnown: true, + } + }), + + setHotspotInfo: (enabled: boolean, ssid: string, password: string, ip: string) => + set(() => { + const hotspot = hotspotFromLegacyFields({ + hotspotEnabled: enabled, + hotspotSsid: ssid, + hotspotPassword: password, + hotspotGatewayIp: ip, + }) + return hotspot ? {hotspot} : {} + }), + + // OTA methods + setOtaStatus: (status: OtaStatus | null) => set({otaStatus: status}), + + setOtaUpdateAvailable: (info: OtaUpdateInfo | null) => set({otaUpdateAvailable: info}), + + setOtaProgress: (progress: OtaProgress | null) => + set((state) => { + const otaInProgress = progress !== null && progress.status !== "FINISHED" && progress.status !== "FAILED" + console.log("🔍 GLASSES STORE: setOtaProgress called with:", JSON.stringify(progress)) + console.log("🔍 GLASSES STORE: otaInProgress =", otaInProgress) + + // Never allow progress to regress within the same stage+currentUpdate — except a new + // work wave (STARTED) or the step after FINISHED (multi-hop APK: 27→31→36 re-downloads from 0). + const prev = state.otaProgress + const sameWave = + progress && + prev && + progress.stage === prev.stage && + progress.currentUpdate === prev.currentUpdate && + progress.progress < prev.progress + if (sameWave) { + const nextIsNewWave = progress.status === "STARTED" || prev.status === "FINISHED" + if (!nextIsNewWave) { + return {otaProgress: {...progress, progress: prev.progress}, otaInProgress} + } + } + + return {otaProgress: progress, otaInProgress} + }), + + setOtaInProgress: (inProgress: boolean) => set({otaInProgress: inProgress}), + + setMtkUpdatedThisSession: (updated: boolean) => set({mtkUpdatedThisSession: updated}), + + clearOtaState: () => + set({ + otaUpdateAvailable: null, + otaProgress: null, + otaInProgress: false, + // Note: mtkUpdatedThisSession is NOT cleared here - it stays true until glasses disconnect/reboot + }), + + reset: () => set(initialState), + })), +) + +export function getGlassesSystemTimeMs(): number { + return useGlassesStore.getState().systemTimeMs ?? 0 +} + +export const waitForGlassesState = ( + key: K, + predicate: (value: GlassesState[K]) => boolean, + timeoutMs = 1000, +): Promise => { + return new Promise((resolve) => { + const state = useGlassesStore.getState() + if (predicate(state[key])) { + resolve(true) + return + } + + const unsubscribe = useGlassesStore.subscribe( + (s) => s[key], + (value) => { + if (predicate(value)) { + unsubscribe() + resolve(true) + } + }, + ) + + setTimeout(() => { + unsubscribe() + resolve(predicate(useGlassesStore.getState()[key])) + }, timeoutMs) + }) +} diff --git a/mobile/src/__tests__/glassesWifi.test.ts b/mobile/src/__tests__/glassesWifi.test.ts index 43b428c0ff..9c39935094 100644 --- a/mobile/src/__tests__/glassesWifi.test.ts +++ b/mobile/src/__tests__/glassesWifi.test.ts @@ -3,6 +3,7 @@ // @mentra/bluetooth-sdk is globally mocked in jest.setup.js — the facade calls it. import BluetoothSdk from "@mentra/bluetooth-sdk" import {glassesWifi} from "../../modules/island/src/facades/glassesWifi" +import {useGlassesStore} from "../../modules/island/src/stores/glasses" describe("glassesWifi facade", () => { beforeEach(() => { @@ -31,4 +32,20 @@ describe("glassesWifi facade", () => { await glassesWifi.forget("home") expect(BluetoothSdk.forgetWifiNetwork).toHaveBeenCalledWith("home") }) + + it("status() reads the glasses store's wifi state", () => { + useGlassesStore.setState({wifi: {state: "connected", ssid: "home"}}) + expect(glassesWifi.status()).toEqual({state: "connected", ssid: "home"}) + }) + + it("onStatus() fires on wifi change and stops after unsubscribe", () => { + useGlassesStore.setState({wifi: {state: "disconnected"}}) + const cb = jest.fn() + const unsub = glassesWifi.onStatus(cb) + useGlassesStore.setState({wifi: {state: "connected", ssid: "home"}}) + expect(cb).toHaveBeenCalledWith({state: "connected", ssid: "home"}, {state: "disconnected"}) + unsub() + useGlassesStore.setState({wifi: {state: "disconnected"}}) + expect(cb).toHaveBeenCalledTimes(1) + }) }) diff --git a/mobile/src/stores/glasses.ts b/mobile/src/stores/glasses.ts index d4313d6ce2..d41455046f 100644 --- a/mobile/src/stores/glasses.ts +++ b/mobile/src/stores/glasses.ts @@ -1,310 +1,15 @@ -import type { - GlassesConnectionStatus, - GlassesStatus, - HotspotStatus, - OtaProgress, - OtaStatus, - OtaUpdateInfo, - WifiStatus, -} from "@mentra/bluetooth-sdk-internal" -import {isGlassesConnected, isGlassesLinkLayerBusy, isGlassesReady} from "@mentra/island" -import {create} from "zustand" -import {subscribeWithSelector} from "zustand/middleware" - -// The connection predicates live in @mentra/island (which re-exports the single -// source of truth from @mentra/bluetooth-sdk). Re-export them here so existing -// importers of @/stores/glasses keep working unchanged. -export {isGlassesConnected, isGlassesLinkLayerBusy, isGlassesReady} - -export const selectGlassesConnected = (state: {connection: GlassesConnectionStatus}) => - isGlassesConnected(state.connection) - -export const selectGlassesReady = (state: {connection: GlassesConnectionStatus}) => isGlassesReady(state.connection) - -interface GlassesState extends GlassesStatus { - systemTimeMs: number - wifiStatusKnown: boolean - setGlassesInfo: (info: GlassesInfoUpdate) => void - setBatteryInfo: (batteryLevel: number, charging: boolean, caseBatteryLevel: number, caseCharging: boolean) => void - setWifiInfo: (connected: boolean, ssid: string) => void - setHotspotInfo: (enabled: boolean, ssid: string, password: string, ip: string) => void - // OTA methods - otaStatus: OtaStatus | null - setOtaStatus: (status: OtaStatus | null) => void - setOtaUpdateAvailable: (info: OtaUpdateInfo | null) => void - setOtaProgress: (progress: OtaProgress | null) => void - setOtaInProgress: (inProgress: boolean) => void - setMtkUpdatedThisSession: (updated: boolean) => void - clearOtaState: () => void - reset: () => void - mtkUpdatedThisSession: boolean -} - -type LegacyWifiFields = { - wifiConnected?: boolean - wifiSsid?: string - wifiLocalIp?: string -} - -type LegacyHotspotFields = { - hotspotEnabled?: boolean - hotspotSsid?: string - hotspotPassword?: string - hotspotGatewayIp?: string - hotspotLocalIp?: string -} - -type GlassesInfoUpdate = Partial & LegacyWifiFields & LegacyHotspotFields - -function wifiFromLegacyFields(info: LegacyWifiFields): WifiStatus | null { - if (info.wifiConnected === true) { - const ssid = info.wifiSsid?.trim() - const localIp = info.wifiLocalIp?.trim() - return ssid ? {state: "connected", ssid, ...(localIp ? {localIp} : {})} : null - } - if (info.wifiConnected === false) { - return {state: "disconnected"} - } - return null -} - -function hotspotFromLegacyFields(info: LegacyHotspotFields): HotspotStatus | null { - if (info.hotspotEnabled === true) { - const ssid = info.hotspotSsid?.trim() - const password = info.hotspotPassword?.trim() - const localIp = (info.hotspotGatewayIp ?? info.hotspotLocalIp)?.trim() - return ssid && password && localIp ? {state: "enabled", ssid, password, localIp} : null - } - if (info.hotspotEnabled === false) { - return {state: "disabled"} - } - return null -} - -export const getGlasesInfoPartial = (state: GlassesStatus) => { - const wifi = state.wifi - const connected = isGlassesConnected(state.connection) - return { - batteryLevel: state.batteryLevel, - charging: state.charging, - caseBatteryLevel: state.caseBatteryLevel, - caseCharging: state.caseCharging, - connected, - wifiConnected: wifi.state === "connected", - wifiSsid: wifi.state === "connected" ? wifi.ssid : "", - deviceModel: state.deviceModel, - // Cloud GlassesInfo uses modelName, map from deviceModel so the cloud - // knows which device is connected when it receives connection state updates - modelName: state.deviceModel || null, - } -} - -interface GlassesStore extends GlassesStatus { - systemTimeMs: number - mtkUpdatedThisSession: boolean - wifiStatusKnown: boolean - otaStatus: OtaStatus | null -} - -const initialState: GlassesStore = { - // state: - connection: {state: "disconnected"}, - micEnabled: false, - bluetoothClassicConnected: false, - signalStrength: -1, - signalStrengthUpdatedAt: 0, - voiceActivityDetectionEnabled: true, - // device info - deviceModel: "", - androidVersion: "", - firmwareVersion: "", - bluetoothMacAddress: "", - leftMacAddress: "", - rightMacAddress: "", - buildNumber: "", - systemTimeMs: 0, - otaVersionUrl: "", - appVersion: "", - bluetoothName: "", - serialNumber: "", - style: "", - color: "", - mtkFirmwareVersion: "", - besFirmwareVersion: "", - // wifi info - wifi: {state: "disconnected"}, - wifiStatusKnown: false, - // battery info - batteryLevel: -1, - charging: false, - caseBatteryLevel: -1, - caseCharging: false, - caseOpen: false, - caseRemoved: true, - // hotspot info - hotspot: {state: "disabled"}, - // OTA update info - otaStatus: null, - otaUpdateAvailable: null, - otaProgress: null, - otaInProgress: false, - mtkUpdatedThisSession: false, - // ring: - controllerConnected: false, - controllerFullyBooted: false, - controllerMacAddress: "", - controllerBatteryLevel: -1, - controllerSignalStrength: -1, -} - -export const useGlassesStore = create()( - subscribeWithSelector((set) => ({ - ...initialState, - - setGlassesInfo: (info) => - set((state) => { - const { - wifiConnected, - wifiSsid, - wifiLocalIp, - hotspotEnabled, - hotspotSsid, - hotspotPassword, - hotspotGatewayIp, - hotspotLocalIp, - wifi, - hotspot, - ...sdkInfo - } = info - const wifiUpdate = wifi ?? wifiFromLegacyFields({wifiConnected, wifiSsid, wifiLocalIp}) - const hotspotUpdate = - hotspot ?? - hotspotFromLegacyFields({hotspotEnabled, hotspotSsid, hotspotPassword, hotspotGatewayIp, hotspotLocalIp}) - const hasWifiInfoUpdate = - Object.prototype.hasOwnProperty.call(info, "wifi") || - Object.prototype.hasOwnProperty.call(info, "wifiConnected") || - Object.prototype.hasOwnProperty.call(info, "wifiSsid") || - Object.prototype.hasOwnProperty.call(info, "wifiLocalIp") - const next = { - ...state, - ...sdkInfo, - ...(wifiUpdate ? {wifi: wifiUpdate} : {}), - ...(hotspotUpdate ? {hotspot: hotspotUpdate} : {}), - ...(hasWifiInfoUpdate ? {wifiStatusKnown: true} : {}), - } - if (!isGlassesConnected(next.connection)) { - next.wifiStatusKnown = false - } - return next - }), - - setBatteryInfo: (batteryLevel, charging, caseBatteryLevel, caseCharging) => - set({ - batteryLevel, - charging, - caseBatteryLevel, - caseCharging, - }), - - setWifiInfo: (connected, ssid) => - set(() => { - const trimmedSsid = ssid.trim() - if (connected && !trimmedSsid) { - return {} - } - const wifi: WifiStatus = connected ? {state: "connected", ssid: trimmedSsid} : {state: "disconnected"} - return { - wifi, - wifiStatusKnown: true, - } - }), - - setHotspotInfo: (enabled: boolean, ssid: string, password: string, ip: string) => - set(() => { - const hotspot = hotspotFromLegacyFields({ - hotspotEnabled: enabled, - hotspotSsid: ssid, - hotspotPassword: password, - hotspotGatewayIp: ip, - }) - return hotspot ? {hotspot} : {} - }), - - // OTA methods - setOtaStatus: (status: OtaStatus | null) => set({otaStatus: status}), - - setOtaUpdateAvailable: (info: OtaUpdateInfo | null) => set({otaUpdateAvailable: info}), - - setOtaProgress: (progress: OtaProgress | null) => - set((state) => { - const otaInProgress = progress !== null && progress.status !== "FINISHED" && progress.status !== "FAILED" - console.log("🔍 GLASSES STORE: setOtaProgress called with:", JSON.stringify(progress)) - console.log("🔍 GLASSES STORE: otaInProgress =", otaInProgress) - - // Never allow progress to regress within the same stage+currentUpdate — except a new - // work wave (STARTED) or the step after FINISHED (multi-hop APK: 27→31→36 re-downloads from 0). - const prev = state.otaProgress - const sameWave = - progress && - prev && - progress.stage === prev.stage && - progress.currentUpdate === prev.currentUpdate && - progress.progress < prev.progress - if (sameWave) { - const nextIsNewWave = progress.status === "STARTED" || prev.status === "FINISHED" - if (!nextIsNewWave) { - return {otaProgress: {...progress, progress: prev.progress}, otaInProgress} - } - } - - return {otaProgress: progress, otaInProgress} - }), - - setOtaInProgress: (inProgress: boolean) => set({otaInProgress: inProgress}), - - setMtkUpdatedThisSession: (updated: boolean) => set({mtkUpdatedThisSession: updated}), - - clearOtaState: () => - set({ - otaUpdateAvailable: null, - otaProgress: null, - otaInProgress: false, - // Note: mtkUpdatedThisSession is NOT cleared here - it stays true until glasses disconnect/reboot - }), - - reset: () => set(initialState), - })), -) - -export function getGlassesSystemTimeMs(): number { - return useGlassesStore.getState().systemTimeMs ?? 0 -} - -export const waitForGlassesState = ( - key: K, - predicate: (value: GlassesState[K]) => boolean, - timeoutMs = 1000, -): Promise => { - return new Promise((resolve) => { - const state = useGlassesStore.getState() - if (predicate(state[key])) { - resolve(true) - return - } - - const unsubscribe = useGlassesStore.subscribe( - (s) => s[key], - (value) => { - if (predicate(value)) { - unsubscribe() - resolve(true) - } - }, - ) - - setTimeout(() => { - unsubscribe() - resolve(predicate(useGlassesStore.getState()[key])) - }, timeoutMs) - }) -} +// The glasses device-state store moved into @mentra/island (island owns device +// state). This shim re-exports it so the ~46 existing `@/stores/glasses` importers +// keep working unchanged — the escape hatch from the Phase-1 plan. New code can use +// `island.glassesStore` / the typed facades directly instead. +export { + isGlassesConnected, + isGlassesLinkLayerBusy, + isGlassesReady, + selectGlassesConnected, + selectGlassesReady, + getGlasesInfoPartial, + getGlassesSystemTimeMs, + useGlassesStore, + waitForGlassesState, +} from "@mentra/island" From 76dd70d13d4b38d7fa455616887c27a7a5f4005c Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 10:04:23 -0700 Subject: [PATCH 04/80] island: rename the public namespace island.* -> toolkit.* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OEM-facing API object is now `toolkit` (toolkit.glasses.wifi, toolkit.configure / start / stop, toolkit.display.mirror, toolkit.glassesStore). The code module stays @mentra/island — only the public surface is renamed, since the product is the OEM Integration Toolkit. Done now while the namespace has only a handful of consumers. All consumers updated: the wifi screens, SocketComms, MantleManager, the jest mock, and the runtime-visible bootstrap error string. The flat exports (useGlassesStore, BluetoothSdk, …) are unchanged. --- mobile/jest.setup.js | 4 ++-- mobile/modules/island/src/facades/displayMirror.ts | 2 +- mobile/modules/island/src/facades/glassesWifi.ts | 2 +- mobile/modules/island/src/index.ts | 4 ++-- mobile/modules/island/src/island.ts | 9 +++++---- mobile/modules/island/src/runtime/bootstrap.ts | 4 ++-- mobile/src/__tests__/islandBootstrapMirror.test.ts | 2 +- mobile/src/app/wifi/connecting.tsx | 4 ++-- mobile/src/app/wifi/scan.tsx | 6 +++--- mobile/src/services/MantleManager.ts | 8 ++++---- mobile/src/services/SocketComms.ts | 4 ++-- mobile/src/stores/glasses.ts | 2 +- 12 files changed, 26 insertions(+), 25 deletions(-) diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index bc536b9688..7de40b5d91 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -256,9 +256,9 @@ jest.mock("@mentra/island", () => { // Real glasses store + its selectors/helpers (useGlassesStore, selectors, // waitForGlassesState, getGlasesInfoPartial, getGlassesSystemTimeMs, predicates). ...realGlasses, - // The namespaced (A) host API. Mirrors the real `island` object; members are + // The namespaced (A) host API. Mirrors the real `toolkit` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. - island: { + toolkit: { configure: jest.fn(), start: jest.fn(() => Promise.resolve()), stop: jest.fn(() => Promise.resolve()), diff --git a/mobile/modules/island/src/facades/displayMirror.ts b/mobile/modules/island/src/facades/displayMirror.ts index e94a4e9014..cf482cb045 100644 --- a/mobile/modules/island/src/facades/displayMirror.ts +++ b/mobile/modules/island/src/facades/displayMirror.ts @@ -1,6 +1,6 @@ /** * glasses display mirror — island's read-model of the latest display event sent - * to the glasses, for a phone-side preview (`island.display.mirror`). + * to the glasses, for a phone-side preview (`toolkit.display.mirror`). * * This is the additive "facade" half of inverting the mirror: island owns this * read-model and is fed every processed display event (from the local miniapp diff --git a/mobile/modules/island/src/facades/glassesWifi.ts b/mobile/modules/island/src/facades/glassesWifi.ts index 155ac3fd14..e8cd722798 100644 --- a/mobile/modules/island/src/facades/glassesWifi.ts +++ b/mobile/modules/island/src/facades/glassesWifi.ts @@ -4,7 +4,7 @@ * * A facade wraps the raw `@mentra/bluetooth-sdk` surface into a small, typed, * device-agnostic API (the `doX()` action shape here; `getX()`/`onX()` read-models - * come with domains that own their state). The host UI calls `island.glasses.wifi.*` + * come with domains that own their state). The host UI calls `toolkit.glasses.wifi.*` * instead of importing bluetooth-sdk directly — that's the native-import boundary * the toolkit is built around. * diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 65313b3566..3418e5a66f 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -79,7 +79,7 @@ export { // The namespaced OEM-facing toolkit API (the "(A) host API"). Additive — grows one // facade at a time alongside the flat exports below. See ./island. -export {island} from "./island" +export {toolkit} from "./island" export type {WifiSearchResult} from "./facades/glassesWifi" export type {DisplayMirrorEvent} from "./facades/displayMirror" export type { @@ -92,7 +92,7 @@ export type { // Glasses device-state store — moved into island so island owns device state. // Re-exported through the host's @/stores/glasses shim so the app keeps its -// imports. Also surfaced as island.glassesStore (the escape hatch). Predicates +// imports. Also surfaced as toolkit.glassesStore (the escape hatch). Predicates // (isGlassesConnected/…) come from GlassesReadiness above, not re-exported here. export { useGlassesStore, diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 1ccf794fcc..cbfd645e81 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -1,7 +1,8 @@ /** - * `island` — the namespaced OEM-facing toolkit API (the "(A) host API" from the - * Phase-1 contract). Host UI calls `island..()`; each domain is a - * typed facade over the runtime. + * `toolkit` — the namespaced OEM-facing toolkit API (the "(A) host API" from the + * Phase-1 contract). Host UI calls `toolkit..()`; each domain is a + * typed facade over the runtime. (Exported from the `@mentra/island` module, whose + * name stays `island` in code; the public API surface is `toolkit`.) * * This is additive: it grows one facade at a time and lives *alongside* the flat * named exports in `index.ts` during the migration. The flat exports (and the @@ -12,7 +13,7 @@ import {glassesWifi} from "./facades/glassesWifi" import {displayMirror} from "./facades/displayMirror" import {useGlassesStore} from "./stores/glasses" -export const island = { +export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ configure, start, diff --git a/mobile/modules/island/src/runtime/bootstrap.ts b/mobile/modules/island/src/runtime/bootstrap.ts index 107e44ebd7..1659de2eb6 100644 --- a/mobile/modules/island/src/runtime/bootstrap.ts +++ b/mobile/modules/island/src/runtime/bootstrap.ts @@ -1,5 +1,5 @@ /** - * Bootstrap — island's single front door (`island.configure` / `start` / `stop`). + * Bootstrap — the toolkit's single front door (`toolkit.configure` / `start` / `stop`). * * The Phase-1 contract's end state is: the host hands island its auth + config in * ONE call and island owns the rest. This is the additive first step of that: a @@ -51,7 +51,7 @@ export function configure(opts: IslandConfigureOptions): void { export async function start(): Promise { if (started) return if (!options) { - throw new Error("island.start() called before island.configure()") + throw new Error("toolkit.start() called before toolkit.configure()") } started = true } diff --git a/mobile/src/__tests__/islandBootstrapMirror.test.ts b/mobile/src/__tests__/islandBootstrapMirror.test.ts index ab056ce1ef..963a47c579 100644 --- a/mobile/src/__tests__/islandBootstrapMirror.test.ts +++ b/mobile/src/__tests__/islandBootstrapMirror.test.ts @@ -31,7 +31,7 @@ describe("island bootstrap front door", () => { }) }) -describe("island.display.mirror read-model", () => { +describe("toolkit.display.mirror read-model", () => { it("ingest() sets current() and notifies subscribers", () => { const seen: unknown[] = [] const unsub = displayMirror.onMirror((e) => seen.push(e)) diff --git a/mobile/src/app/wifi/connecting.tsx b/mobile/src/app/wifi/connecting.tsx index f05e93256e..13a050c748 100644 --- a/mobile/src/app/wifi/connecting.tsx +++ b/mobile/src/app/wifi/connecting.tsx @@ -1,4 +1,4 @@ -import {island} from "@mentra/island" +import {toolkit} from "@mentra/island" import {useLocalSearchParams} from "expo-router" import {useEffect, useState, useCallback} from "react" import {ActivityIndicator, View} from "react-native" @@ -81,7 +81,7 @@ export default function WifiConnectingScreen() { const attemptConnection = async () => { try { console.log("Attempting to send wifi credentials to Core", ssid, password) - await island.glasses.wifi.connect(ssid, password) + await toolkit.glasses.wifi.connect(ssid, password) // Save credentials ONLY on successful connection if checkbox was checked. // This ensures we never save wrong passwords. diff --git a/mobile/src/app/wifi/scan.tsx b/mobile/src/app/wifi/scan.tsx index 3f5d38da68..f08ade03a7 100644 --- a/mobile/src/app/wifi/scan.tsx +++ b/mobile/src/app/wifi/scan.tsx @@ -1,4 +1,4 @@ -import {island, type WifiSearchResult} from "@mentra/island" +import {toolkit, type WifiSearchResult} from "@mentra/island" import {useFocusEffect} from "expo-router" import {useCallback, useEffect, useMemo, useState} from "react" import {ActivityIndicator, ScrollView, TouchableOpacity, View} from "react-native" @@ -81,7 +81,7 @@ export default function WifiScanScreen() { setNetworks([]) try { - const scanResults = await island.glasses.wifi.scan() + const scanResults = await toolkit.glasses.wifi.scan() console.log(`WIFI_SCAN: Received ${scanResults.length} WiFi scan results`) setNetworks( scanResults.map((network) => ({ @@ -118,7 +118,7 @@ export default function WifiScanScreen() { onPress: async () => { try { console.log(`WIFI_SCAN: Forgetting network: ${selectedNetwork.ssid}`) - await island.glasses.wifi.forget(selectedNetwork.ssid) + await toolkit.glasses.wifi.forget(selectedNetwork.ssid) // Also remove from local saved credentials WifiCredentialsService.removeCredentials(selectedNetwork.ssid) setSavedNetworks((prev) => prev.filter((ssid) => ssid !== selectedNetwork.ssid)) diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index 97a21dbaab..ebcde44fbd 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -28,7 +28,7 @@ import { configureRuntime, getRuntimeHooks, displayProcessor, - island, + toolkit, localMiniappRuntime, localSttFallbackCoordinator, micStateCoordinator, @@ -169,18 +169,18 @@ class MantleManager { // Island front door: hand island the host's auth provider, then mark it // started. Additive — the `configure*` adapter seams below still wire the // host services during the migration; consumers move onto this incrementally. - island.configure({ + toolkit.configure({ auth: { getSubjectToken: async () => { const res = await mentraAuth.getSession() if (res.is_error() || !res.value.token) { - throw new Error("island.configure: no session token available") + throw new Error("toolkit.configure: no session token available") } return {token: res.value.token, type: "supabase"} }, }, }) - void island.start() + void toolkit.start() // iOS: require a second swipe across the bottom edge to invoke the Home // indicator / app switcher, so users don't accidentally background the diff --git a/mobile/src/services/SocketComms.ts b/mobile/src/services/SocketComms.ts index e2cb03931a..304b3d488e 100644 --- a/mobile/src/services/SocketComms.ts +++ b/mobile/src/services/SocketComms.ts @@ -1,6 +1,6 @@ import {type RgbLedControlResponseEvent, type TouchEvent} from "@mentra/bluetooth-sdk" import BluetoothSdk from "@mentra/bluetooth-sdk-internal" -import {displayProcessor, island, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" +import {displayProcessor, toolkit, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" import audioPlaybackService from "@/services/AudioPlaybackService" import mantle from "@/services/MantleManager" @@ -488,7 +488,7 @@ class SocketComms { const displayEventStr = JSON.stringify(processedEvent) useDisplayStore.getState().setDisplayEvent(displayEventStr) // Feed island's own mirror read-model in parallel with the legacy host store. - island.display.mirror.ingest(processedEvent) + toolkit.display.mirror.ingest(processedEvent) } private handle_set_location_tier(msg: any) { diff --git a/mobile/src/stores/glasses.ts b/mobile/src/stores/glasses.ts index d41455046f..7ad3cee439 100644 --- a/mobile/src/stores/glasses.ts +++ b/mobile/src/stores/glasses.ts @@ -1,7 +1,7 @@ // The glasses device-state store moved into @mentra/island (island owns device // state). This shim re-exports it so the ~46 existing `@/stores/glasses` importers // keep working unchanged — the escape hatch from the Phase-1 plan. New code can use -// `island.glassesStore` / the typed facades directly instead. +// `toolkit.glassesStore` / the typed facades directly instead. export { isGlassesConnected, isGlassesLinkLayerBusy, From c6f85fa7709b5ec7f957a13d515559d4793e0b45 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 10:18:35 -0700 Subject: [PATCH 05/80] island: move the display/mirror store into island (toolkit.display.mirror reads it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The glasses-screen mirror read-model now lives in island, so the mirror UI is on island-owned state too (the device store moved last commit). - stores/display.ts -> modules/island/src/stores/display.ts. The two pure @/utils/e2eMetrics helpers (extractDisplayText, logE2EMetric) are inlined so the store is self-contained inside island; the host e2eMetrics util is untouched. Added subscribeWithSelector for the facade's selector subscription. - @/stores/display becomes a re-export shim (consumers unchanged). The raw store is toolkit.displayStore — a Mentra-app escape hatch, NOT the OEM contract. - toolkit.display.mirror is now the typed READ facade over the island-owned store (current() / onMirror()), replacing the earlier parallel read-model. Reverted the parallel ingest feeds in LocalDisplayManager + SocketComms. - jest: the @mentra/island mock serves the real display store via requireActual. Verified: typecheck 0 new errors; jest 265 pass (GlassesDisplayMirror + the SocketComms display path stay green). --- mobile/jest.setup.js | 5 +- .../island/src/facades/displayMirror.ts | 43 +++------ mobile/modules/island/src/index.ts | 3 + mobile/modules/island/src/island.ts | 9 +- .../src/services/LocalDisplayManager.ts | 3 - mobile/modules/island/src/stores/display.ts | 95 +++++++++++++++++++ .../__tests__/islandBootstrapMirror.test.ts | 20 ++-- mobile/src/services/SocketComms.ts | 4 +- mobile/src/stores/display.ts | 66 +------------ 9 files changed, 137 insertions(+), 111 deletions(-) create mode 100644 mobile/modules/island/src/stores/display.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 7de40b5d91..81a9b08226 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -237,6 +237,7 @@ jest.mock("@mentra/island", () => { // REAL behavior (setState/getState/subscribe), so pull the actual store in. It's // pure (zustand + type-only btsdk imports), so it loads cleanly under the mock. const realGlasses = jest.requireActual("./modules/island/src/stores/glasses") + const realDisplay = jest.requireActual("./modules/island/src/stores/display") const appStatusState = { apps: [], refresh: jest.fn(), @@ -256,6 +257,8 @@ jest.mock("@mentra/island", () => { // Real glasses store + its selectors/helpers (useGlassesStore, selectors, // waitForGlassesState, getGlasesInfoPartial, getGlassesSystemTimeMs, predicates). ...realGlasses, + // Real display/mirror store (useDisplayStore) — consumers need its real behavior. + ...realDisplay, // The namespaced (A) host API. Mirrors the real `toolkit` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. toolkit: { @@ -273,12 +276,12 @@ jest.mock("@mentra/island", () => { }, display: { mirror: { - ingest: jest.fn(), current: jest.fn(() => null), onMirror: jest.fn(() => () => {}), }, }, glassesStore: realGlasses.useGlassesStore, + displayStore: realDisplay.useDisplayStore, }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), diff --git a/mobile/modules/island/src/facades/displayMirror.ts b/mobile/modules/island/src/facades/displayMirror.ts index cf482cb045..d0739493c9 100644 --- a/mobile/modules/island/src/facades/displayMirror.ts +++ b/mobile/modules/island/src/facades/displayMirror.ts @@ -1,40 +1,23 @@ /** - * glasses display mirror — island's read-model of the latest display event sent - * to the glasses, for a phone-side preview (`toolkit.display.mirror`). - * - * This is the additive "facade" half of inverting the mirror: island owns this - * read-model and is fed every processed display event (from the local miniapp - * display path and the cloud display path), in PARALLEL with the legacy host - * `useDisplayStore`. Nothing has to read island yet, so no consumer/test churn. - * - * The follow-up "callback" half flips the preview UI (`GlassesDisplayMirror`, - * `SimulatedGlassesControls`) onto `onMirror`/`current`, adds view tracking here, - * deletes the host `useDisplayStore` + the `setDisplayEvent` RuntimeHooks adapter, - * and migrates the display tests. That part is test-coupled to v1 `SocketComms` - * and `MantleManager`, so it's isolated to its own PR. + * glasses display mirror — the typed read facade over the (now island-owned) + * display store, for a phone-side preview of the glasses screen + * (`toolkit.display.mirror`). The store is fed by the display paths + * (LocalDisplayManager + the cloud path) via `setDisplayEvent`; this facade is the + * read side: `current()` (snapshot) + `onMirror(cb)` (subscribe). The raw store is + * also exposed as `toolkit.displayStore` (the Mentra-app escape hatch). */ -export type DisplayMirrorEvent = Record & {view?: string} +import {useDisplayStore} from "../stores/display" -let latestEvent: DisplayMirrorEvent | null = null -const listeners = new Set<(event: DisplayMirrorEvent) => void>() +export type DisplayMirrorEvent = Record & {view?: string} export const displayMirror = { - /** Feed a processed display event (the object `DisplayProcessor` produces). */ - ingest(event: DisplayMirrorEvent): void { - latestEvent = event - for (const cb of listeners) cb(event) - }, - - /** The most recent processed display event, or null before the first one. */ - current(): DisplayMirrorEvent | null { - return latestEvent + /** The display event currently shown on the previewed view (snapshot). */ + current(): DisplayMirrorEvent { + return useDisplayStore.getState().currentEvent }, - /** Subscribe to display events; returns an unsubscribe. */ + /** Subscribe to changes of the previewed display event; returns an unsubscribe. */ onMirror(cb: (event: DisplayMirrorEvent) => void): () => void { - listeners.add(cb) - return () => { - listeners.delete(cb) - } + return useDisplayStore.subscribe((s) => s.currentEvent, cb) }, } diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 3418e5a66f..3556e032f3 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -102,6 +102,9 @@ export { getGlassesSystemTimeMs, waitForGlassesState, } from "./stores/glasses" +// Display/mirror store — also moved into island; re-exported through the host's +// @/stores/display shim (and toolkit.displayStore). Read it via toolkit.display.mirror. +export {useDisplayStore} from "./stores/display" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index cbfd645e81..1f80d5980b 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -12,6 +12,7 @@ import {configure, start, stop} from "./runtime/bootstrap" import {glassesWifi} from "./facades/glassesWifi" import {displayMirror} from "./facades/displayMirror" import {useGlassesStore} from "./stores/glasses" +import {useDisplayStore} from "./stores/display" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ @@ -25,9 +26,11 @@ export const toolkit = { mirror: displayMirror, }, /** - * Escape hatch — the raw glasses device-state zustand store, exposed so the - * Mentra app keeps using it directly instead of rewriting every screen onto - * typed facades. Prefer a facade where one exists. + * Escape hatches — the raw device-state zustand stores, exposed so the Mentra + * app keeps using them directly instead of rewriting every screen onto typed + * facades. These are Mentra-app convenience, NOT the OEM contract — OEMs use the + * typed facades above. Prefer a facade where one exists. */ glassesStore: useGlassesStore, + displayStore: useDisplayStore, } diff --git a/mobile/modules/island/src/services/LocalDisplayManager.ts b/mobile/modules/island/src/services/LocalDisplayManager.ts index 7481fa0274..bc9a58e293 100644 --- a/mobile/modules/island/src/services/LocalDisplayManager.ts +++ b/mobile/modules/island/src/services/LocalDisplayManager.ts @@ -20,7 +20,6 @@ */ import displayProcessor from "./DisplayProcessor" -import {displayMirror} from "../facades/displayMirror" import {getRuntimeHooks} from "../runtime/config" import {BgTimer} from "../utils/timers" @@ -389,8 +388,6 @@ class LocalDisplayManager { }) } getRuntimeHooks().setDisplayEvent?.(JSON.stringify(processedEvent)) - // Feed island's own mirror read-model in parallel with the legacy host store. - displayMirror.ingest(processedEvent) } catch (err) { console.error(`${LOG_TAG}: native display failed:`, err) } diff --git a/mobile/modules/island/src/stores/display.ts b/mobile/modules/island/src/stores/display.ts new file mode 100644 index 0000000000..39c26dcad0 --- /dev/null +++ b/mobile/modules/island/src/stores/display.ts @@ -0,0 +1,95 @@ +import {create} from "zustand" +import {subscribeWithSelector} from "zustand/middleware" + +// Inlined from the host @/utils/e2eMetrics (both are pure) so this store is +// self-contained inside island. Emits the same `E2E_METRIC` console lines the +// e2e harness reads; the host util keeps emitting its non-display metrics. +const E2E_METRICS_ENABLED = process.env.EXPO_PUBLIC_ENABLE_E2E_METRICS === "true" + +function logE2EMetric(event: string, payload: Record = {}): void { + if (!E2E_METRICS_ENABLED) return + try { + console.log(`E2E_METRIC ${JSON.stringify({event, ts_ms: Date.now(), ...payload})}`) + } catch (error) { + console.warn("E2E_METRIC: failed to serialize payload", error) + } +} + +function extractDisplayText(displayEvent: any): string[] { + const layout = displayEvent?.layout + if (!layout || typeof layout !== "object") { + return [] + } + switch (layout.layoutType) { + case "text_wall": + case "text_line": + return typeof layout.text === "string" ? layout.text.split("\n") : [] + case "double_text_wall": + return [layout.topText, layout.bottomText].filter((value): value is string => typeof value === "string") + case "text_rows": + return Array.isArray(layout.text) + ? layout.text.filter((value: unknown): value is string => typeof value === "string") + : [] + default: + return [] + } +} + +interface DisplayStore { + currentEvent: any + dashboardEvent: any + mainEvent: any + setDisplayEvent: (eventString: string) => void + view: string + setView: (view: string) => void +} + +export const useDisplayStore = create()( + subscribeWithSelector((set, get) => ({ + currentEvent: {} as any, + dashboardEvent: {} as any, + mainEvent: {} as any, + view: "main", + setDisplayEvent: (eventString: string) => { + const event = JSON.parse(eventString) + const currentView = get().view + + const updates: any = { + [event.view === "dashboard" ? "dashboardEvent" : "mainEvent"]: event, + } + + // also update the current event if the view is the same: + if (event.view === currentView) { + updates.currentEvent = event + } + + const visibleEvent = updates.currentEvent ?? event + const textLines = extractDisplayText(visibleEvent) + if (textLines.some((line) => line.trim() !== "")) { + logE2EMetric("display_store_update", { + view: visibleEvent.view ?? currentView, + layout_type: visibleEvent.layout?.layoutType ?? "", + text_lines: textLines, + }) + } + + set(updates) + }, + setView: (view: string) => { + const currentView = get().view + if (view === currentView) { + return + } + + // update the view and the currentEvent with the corresponding event: + let newEvent + if (view === "dashboard") { + newEvent = get().dashboardEvent + } else { + newEvent = get().mainEvent + } + logE2EMetric("display_view_changed", {view}) + set({view, currentEvent: newEvent}) + }, + })), +) diff --git a/mobile/src/__tests__/islandBootstrapMirror.test.ts b/mobile/src/__tests__/islandBootstrapMirror.test.ts index 963a47c579..362470b557 100644 --- a/mobile/src/__tests__/islandBootstrapMirror.test.ts +++ b/mobile/src/__tests__/islandBootstrapMirror.test.ts @@ -2,6 +2,7 @@ // which jest mocks) so the actual logic runs under the mobile jest CI runner. import * as bootstrap from "../../modules/island/src/runtime/bootstrap" import {displayMirror} from "../../modules/island/src/facades/displayMirror" +import {useDisplayStore} from "../../modules/island/src/stores/display" describe("island bootstrap front door", () => { beforeEach(async () => { @@ -31,22 +32,21 @@ describe("island bootstrap front door", () => { }) }) -describe("toolkit.display.mirror read-model", () => { - it("ingest() sets current() and notifies subscribers", () => { - const seen: unknown[] = [] - const unsub = displayMirror.onMirror((e) => seen.push(e)) +describe("toolkit.display.mirror read facade over the display store", () => { + it("current() returns the store's current display event", () => { const event = {view: "main", layout: {layoutType: "text_wall", text: "hi"}} - displayMirror.ingest(event) - expect(displayMirror.current()).toBe(event) - expect(seen).toEqual([event]) - unsub() + useDisplayStore.getState().setDisplayEvent(JSON.stringify(event)) + expect(displayMirror.current()).toEqual(event) }) - it("onMirror() unsubscribe stops delivery", () => { + it("onMirror() fires on display change and stops after unsubscribe", () => { const cb = jest.fn() const unsub = displayMirror.onMirror(cb) + useDisplayStore.getState().setDisplayEvent(JSON.stringify({view: "main", layout: {layoutType: "text_wall", text: "x"}})) + expect(cb).toHaveBeenCalled() unsub() - displayMirror.ingest({view: "dashboard"}) + cb.mockClear() + useDisplayStore.getState().setDisplayEvent(JSON.stringify({view: "main", layout: {layoutType: "text_wall", text: "y"}})) expect(cb).not.toHaveBeenCalled() }) }) diff --git a/mobile/src/services/SocketComms.ts b/mobile/src/services/SocketComms.ts index 304b3d488e..0021e1ed03 100644 --- a/mobile/src/services/SocketComms.ts +++ b/mobile/src/services/SocketComms.ts @@ -1,6 +1,6 @@ import {type RgbLedControlResponseEvent, type TouchEvent} from "@mentra/bluetooth-sdk" import BluetoothSdk from "@mentra/bluetooth-sdk-internal" -import {displayProcessor, toolkit, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" +import {displayProcessor, localMiniappRuntime, micStateCoordinator, throttle} from "@mentra/island" import audioPlaybackService from "@/services/AudioPlaybackService" import mantle from "@/services/MantleManager" @@ -487,8 +487,6 @@ class SocketComms { BluetoothSdk.displayEvent(processedEvent) const displayEventStr = JSON.stringify(processedEvent) useDisplayStore.getState().setDisplayEvent(displayEventStr) - // Feed island's own mirror read-model in parallel with the legacy host store. - toolkit.display.mirror.ingest(processedEvent) } private handle_set_location_tier(msg: any) { diff --git a/mobile/src/stores/display.ts b/mobile/src/stores/display.ts index afa7f0dbf3..8088501094 100644 --- a/mobile/src/stores/display.ts +++ b/mobile/src/stores/display.ts @@ -1,61 +1,5 @@ -import {create} from "zustand" -import {extractDisplayText, logE2EMetric} from "@/utils/e2eMetrics" -// import -// TODO: import view types from cloud - -interface DisplayStore { - currentEvent: any - dashboardEvent: any - mainEvent: any - setDisplayEvent: (eventString: string) => void - view: string - setView: (view: string) => void -} - -export const useDisplayStore = create((set, get) => ({ - currentEvent: {} as any, - dashboardEvent: {} as any, - mainEvent: {} as any, - view: "main", - setDisplayEvent: (eventString: string) => { - const event = JSON.parse(eventString) - const currentView = get().view - - const updates: any = { - [event.view === "dashboard" ? "dashboardEvent" : "mainEvent"]: event, - } - - // also update the current event if the view is the same: - if (event.view === currentView) { - updates.currentEvent = event - } - - const visibleEvent = updates.currentEvent ?? event - const textLines = extractDisplayText(visibleEvent) - if (textLines.some((line) => line.trim() !== "")) { - logE2EMetric("display_store_update", { - view: visibleEvent.view ?? currentView, - layout_type: visibleEvent.layout?.layoutType ?? "", - text_lines: textLines, - }) - } - - set(updates) - }, - setView: (view: string) => { - const currentView = get().view - if (view === currentView) { - return - } - - // update the view and the currentEvent with the corresponding event: - let newEvent - if (view === "dashboard") { - newEvent = get().dashboardEvent - } else { - newEvent = get().mainEvent - } - logE2EMetric("display_view_changed", {view}) - set({view, currentEvent: newEvent}) - }, -})) +// The display/mirror store moved into @mentra/island (island owns the read-model +// of what's on the glasses screen). This shim re-exports it so existing +// `@/stores/display` importers keep working unchanged — the Mentra-app escape hatch. +// New code reads it via the typed `toolkit.display.mirror` facade. +export {useDisplayStore} from "@mentra/island" From ca57985210e9de7adb98fb3e0fdc1c8700ee2064 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 11:00:34 -0700 Subject: [PATCH 06/80] island: move core/connection/gallerySync stores in + group hatches under toolkit.stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues the device-store consolidation. - core / connection / gallerySync stores -> modules/island/src/stores/* via the shim pattern (the @/stores/* files become 1-line re-exports; consumers unchanged). - Two coupled types relocated into island so the stores are standalone-build-safe: WebSocketStatus (was @/services/ws-types) and PhotoInfo (was @/types/asg). Both old homes now re-export from island, so their other importers are unchanged. - Grouped the raw-store escape hatches under toolkit.stores.{glasses,display,core, connection,gallerySync} (was toolkit.glassesStore/displayStore — nothing consumed those yet, so zero-risk). Reminder: stores are the Mentra-app escape hatch, NOT the OEM contract; OEMs use the typed facades. - Deferred cloudClientStatus: it imports cloud-v2 (@mentra/cloud-client) types, so it belongs with the cloud-client / session work, not this batch. - jest: the @mentra/island mock serves the real stores via requireActual. Verified: typecheck 0 new errors across all consumers; jest 265 pass. --- mobile/jest.setup.js | 16 +- mobile/modules/island/src/index.ts | 11 + mobile/modules/island/src/island.ts | 20 +- .../modules/island/src/stores/connection.ts | 68 ++++ mobile/modules/island/src/stores/core.ts | 32 ++ .../modules/island/src/stores/gallerySync.ts | 380 ++++++++++++++++++ mobile/src/services/ws-types.ts | 9 +- mobile/src/stores/connection.ts | 63 +-- mobile/src/stores/core.ts | 35 +- mobile/src/stores/gallerySync.ts | 370 +---------------- mobile/src/types/asg/index.ts | 23 +- 11 files changed, 542 insertions(+), 485 deletions(-) create mode 100644 mobile/modules/island/src/stores/connection.ts create mode 100644 mobile/modules/island/src/stores/core.ts create mode 100644 mobile/modules/island/src/stores/gallerySync.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 81a9b08226..33daa798e3 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -238,6 +238,9 @@ jest.mock("@mentra/island", () => { // pure (zustand + type-only btsdk imports), so it loads cleanly under the mock. const realGlasses = jest.requireActual("./modules/island/src/stores/glasses") const realDisplay = jest.requireActual("./modules/island/src/stores/display") + const realCore = jest.requireActual("./modules/island/src/stores/core") + const realConnection = jest.requireActual("./modules/island/src/stores/connection") + const realGallerySync = jest.requireActual("./modules/island/src/stores/gallerySync") const appStatusState = { apps: [], refresh: jest.fn(), @@ -259,6 +262,10 @@ jest.mock("@mentra/island", () => { ...realGlasses, // Real display/mirror store (useDisplayStore) — consumers need its real behavior. ...realDisplay, + // Real core / connection / gallerySync stores (+ WebSocketStatus, selectors). + ...realCore, + ...realConnection, + ...realGallerySync, // The namespaced (A) host API. Mirrors the real `toolkit` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. toolkit: { @@ -280,8 +287,13 @@ jest.mock("@mentra/island", () => { onMirror: jest.fn(() => () => {}), }, }, - glassesStore: realGlasses.useGlassesStore, - displayStore: realDisplay.useDisplayStore, + stores: { + glasses: realGlasses.useGlassesStore, + display: realDisplay.useDisplayStore, + core: realCore.useCoreStore, + connection: realConnection.useConnectionStore, + gallerySync: realGallerySync.useGallerySyncStore, + }, }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 3556e032f3..ba9eb2b8e8 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -105,6 +105,17 @@ export { // Display/mirror store — also moved into island; re-exported through the host's // @/stores/display shim (and toolkit.displayStore). Read it via toolkit.display.mirror. export {useDisplayStore} from "./stores/display" +// More device/runtime stores moved into island (re-exported via host shims + +// toolkit.stores.*). WebSocketStatus + PhotoInfo were relocated off host modules. +export {useCoreStore} from "./stores/core" +export {useConnectionStore, WebSocketStatus} from "./stores/connection" +export { + useGallerySyncStore, + selectSyncProgress, + selectIssyncing, + selectGlassesGalleryStatus, +} from "./stores/gallerySync" +export type {PhotoInfo, SyncState, HotspotInfo, SyncQueue, GallerySyncInfo} from "./stores/gallerySync" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 1f80d5980b..d3627908e6 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -13,6 +13,9 @@ import {glassesWifi} from "./facades/glassesWifi" import {displayMirror} from "./facades/displayMirror" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" +import {useCoreStore} from "./stores/core" +import {useConnectionStore} from "./stores/connection" +import {useGallerySyncStore} from "./stores/gallerySync" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ @@ -26,11 +29,16 @@ export const toolkit = { mirror: displayMirror, }, /** - * Escape hatches — the raw device-state zustand stores, exposed so the Mentra - * app keeps using them directly instead of rewriting every screen onto typed - * facades. These are Mentra-app convenience, NOT the OEM contract — OEMs use the - * typed facades above. Prefer a facade where one exists. + * Escape hatches — the raw device-state zustand stores, grouped under `stores`, + * exposed so the Mentra app keeps using them directly instead of rewriting every + * screen onto typed facades. These are Mentra-app convenience, NOT the OEM + * contract — OEMs use the typed facades above. Prefer a facade where one exists. */ - glassesStore: useGlassesStore, - displayStore: useDisplayStore, + stores: { + glasses: useGlassesStore, + display: useDisplayStore, + core: useCoreStore, + connection: useConnectionStore, + gallerySync: useGallerySyncStore, + }, } diff --git a/mobile/modules/island/src/stores/connection.ts b/mobile/modules/island/src/stores/connection.ts new file mode 100644 index 0000000000..956c067895 --- /dev/null +++ b/mobile/modules/island/src/stores/connection.ts @@ -0,0 +1,68 @@ +import {create} from "zustand" + +// Relocated from the host @/services/ws-types so this store is self-contained +// inside island. The host @/services/ws-types now re-exports this (a shim), so its +// other importers are unchanged. +export enum WebSocketStatus { + DISCONNECTED = "disconnected", + CONNECTING = "connecting", + CONNECTED = "connected", + ERROR = "error", +} + +interface ConnectionState { + status: WebSocketStatus + url: string | null + error: string | null + lastConnectedAt: Date | null + lastDisconnectedAt: Date | null + reconnectAttempts: number + + setStatus: (status: WebSocketStatus) => void + setUrl: (url: string | null) => void + setError: (error: string | null) => void + incrementReconnectAttempts: () => void + resetReconnectAttempts: () => void + reset: () => void +} + +export const useConnectionStore = create((set) => ({ + status: WebSocketStatus.DISCONNECTED, + url: null, + error: null, + lastConnectedAt: null, + lastDisconnectedAt: null, + reconnectAttempts: 0, + + setStatus: (status) => + set((state) => ({ + status, + error: status === WebSocketStatus.ERROR ? state.error : null, + lastConnectedAt: status === WebSocketStatus.CONNECTED ? new Date() : state.lastConnectedAt, + lastDisconnectedAt: + status === WebSocketStatus.DISCONNECTED || status === WebSocketStatus.ERROR + ? new Date() + : state.lastDisconnectedAt, + })), + + setUrl: (url) => set({url}), + + setError: (error) => set({error, status: WebSocketStatus.ERROR, lastDisconnectedAt: new Date()}), + + incrementReconnectAttempts: () => + set((state) => ({ + reconnectAttempts: state.reconnectAttempts + 1, + })), + + resetReconnectAttempts: () => set({reconnectAttempts: 0}), + + reset: () => + set({ + status: WebSocketStatus.DISCONNECTED, + url: null, + error: null, + lastConnectedAt: null, + lastDisconnectedAt: null, + reconnectAttempts: 0, + }), +})) diff --git a/mobile/modules/island/src/stores/core.ts b/mobile/modules/island/src/stores/core.ts new file mode 100644 index 0000000000..a229d16d80 --- /dev/null +++ b/mobile/modules/island/src/stores/core.ts @@ -0,0 +1,32 @@ +import {create} from "zustand" +import {subscribeWithSelector} from "zustand/middleware" +import type {BluetoothStatus} from "../../../bluetooth-sdk/build/_internal" + +interface CoreState extends BluetoothStatus { + setCoreInfo: (info: Partial) => void + reset: () => void +} + +const initialState: BluetoothStatus = { + // state: + searching: false, + searchingController: false, + micRanking: ["glasses", "phone", "bluetooth", "bluetoothClassic"], + systemMicUnavailable: false, + currentMic: null, + searchResults: [], + wifiScanResults: [], + lastLog: [], + otherBtConnected: false, + galleryModeEnabled: true, +} + +export const useCoreStore = create()( + subscribeWithSelector((set) => ({ + ...initialState, + + setCoreInfo: (info) => set((state) => ({...state, ...info})), + + reset: () => set(initialState), + })), +) diff --git a/mobile/modules/island/src/stores/gallerySync.ts b/mobile/modules/island/src/stores/gallerySync.ts new file mode 100644 index 0000000000..380ad821cc --- /dev/null +++ b/mobile/modules/island/src/stores/gallerySync.ts @@ -0,0 +1,380 @@ +/** + * Gallery Sync Store + * Manages gallery sync state independently of UI lifecycle + */ + +import {create} from "zustand" +import {subscribeWithSelector} from "zustand/middleware" + +// Relocated from the host @/types/asg so this store is self-contained inside +// island. The host barrel re-exports this (a shim), so its importers are unchanged. +export interface PhotoInfo { + name: string + url: string + download: string + size: number + modified: string | number // Unix timestamp (milliseconds) - can be string or number from API + mime_type?: string + is_video?: boolean + thumbnail_data?: string + downloaded_at?: number + // Video duration in milliseconds (from glasses sync response) + duration?: number + // New fields for filesystem storage + filePath?: string + thumbnailPath?: string + // Glasses model that captured this media + glassesModel?: string +} + +// Sync state machine states +export type SyncState = + | "idle" + | "requesting_hotspot" + | "connecting_wifi" + | "syncing" + | "complete" + | "error" + | "cancelled" + +export interface HotspotInfo { + ssid: string + password: string + ip: string +} + +export interface SyncQueue { + files: PhotoInfo[] + currentIndex: number + startedAt: number + hotspotInfo: HotspotInfo +} + +export interface GallerySyncInfo { + // State machine + syncState: SyncState + + // Progress tracking + currentFile: string | null + currentFileProgress: number // 0-100 + completedFiles: number + totalFiles: number + failedFiles: string[] + + // Queue (persisted separately via localStorageService) + queue: PhotoInfo[] + queueIndex: number + + // Hotspot info + hotspotInfo: HotspotInfo | null + syncServiceOpenedHotspot: boolean + + // Gallery status from glasses + glassesPhotoCount: number + glassesVideoCount: number + glassesTotalCount: number + glassesHasContent: boolean + + // Error tracking + lastError: string | null +} + +interface GallerySyncState extends GallerySyncInfo { + // State transitions + setSyncState: (state: SyncState) => void + setRequestingHotspot: () => void + setConnectingWifi: () => void + setSyncing: (files: PhotoInfo[]) => void + setSyncComplete: () => void + setSyncError: (error: string) => void + setSyncCancelled: () => void + + // Progress updates + setCurrentFile: (fileName: string | null, progress: number) => void + onFileProgress: (fileName: string, progress: number) => void + onFileComplete: (fileName: string) => void + onFileFailed: (fileName: string, error?: string) => void + onFileProcessing: (fileName: string) => void + onFileProcessed: (fileName: string) => void + updateFileInQueue: (fileName: string, updatedFile: PhotoInfo) => void + removeFilesFromQueue: (fileNames: string[]) => void + + // Processing queue tracking + processingFiles: Set + processedFiles: number + + // Hotspot management + setHotspotInfo: (info: HotspotInfo | null) => void + setSyncServiceOpenedHotspot: (opened: boolean) => void + + // Gallery status from glasses + setGlassesGalleryStatus: (photos: number, videos: number, total: number, hasContent: boolean) => void + clearGlassesGalleryStatus: () => void + + // Queue management (for resume) + setQueue: (files: PhotoInfo[], startIndex?: number) => void + advanceQueue: () => void + clearQueue: () => void + + // Full reset + reset: () => void +} + +const initialState: GallerySyncInfo & {processingFiles: Set; processedFiles: number} = { + syncState: "idle", + currentFile: null, + currentFileProgress: 0, + completedFiles: 0, + totalFiles: 0, + failedFiles: [], + queue: [], + queueIndex: 0, + hotspotInfo: null, + syncServiceOpenedHotspot: false, + glassesPhotoCount: 0, + glassesVideoCount: 0, + glassesTotalCount: 0, + glassesHasContent: false, + lastError: null, + processingFiles: new Set(), + processedFiles: 0, +} + +export const useGallerySyncStore = create()( + subscribeWithSelector((set, get) => ({ + ...initialState, + + // State transitions + setSyncState: (syncState: SyncState) => set({syncState}), + + setRequestingHotspot: () => + set({ + syncState: "requesting_hotspot", + lastError: null, + failedFiles: [], + }), + + setConnectingWifi: () => + set({ + syncState: "connecting_wifi", + }), + + setSyncing: (files: PhotoInfo[]) => + set({ + syncState: "syncing", + // C4: Strip thumbnail_data (base64) from store to prevent OOM + queue: files.map(({thumbnail_data: _thumbnailData, ...rest}) => rest), + queueIndex: 0, + totalFiles: files.length, + completedFiles: 0, + currentFile: files.length > 0 ? files[0].name : null, + currentFileProgress: 0, + failedFiles: [], + lastError: null, + processedFiles: 0, + processingFiles: new Set(), + }), + + setSyncComplete: () => + set({ + syncState: "complete", + currentFile: null, + currentFileProgress: 0, + // Keep queue intact so photos remain visible after sync + // Don't clear: queue: [], queueIndex: 0 + }), + + setSyncError: (error: string) => + set({ + syncState: "error", + lastError: error, + currentFile: null, + currentFileProgress: 0, + }), + + setSyncCancelled: () => + set({ + syncState: "cancelled", + currentFile: null, + currentFileProgress: 0, + queue: [], + queueIndex: 0, + }), + + // Progress updates + setCurrentFile: (fileName: string | null, progress: number) => + set({ + currentFile: fileName, + currentFileProgress: Math.max(0, Math.min(100, progress)), + }), + + onFileProgress: (fileName: string, progress: number) => { + const state = get() + const clampedProgress = Math.max(0, Math.min(100, progress)) + // Throttle: skip update if same file and same percentage (prevents Zustand flood) + if (clampedProgress === state.currentFileProgress && fileName === state.currentFile) { + return + } + set({ + currentFile: fileName, + currentFileProgress: clampedProgress, + }) + }, + + onFileComplete: (_fileName: string) => { + const state = get() + const newCompletedFiles = state.completedFiles + 1 + const newQueueIndex = state.queueIndex + 1 + const nextFile = state.queue[newQueueIndex] + + set({ + completedFiles: newCompletedFiles, + queueIndex: newQueueIndex, + currentFile: nextFile?.name || null, + currentFileProgress: 0, + }) + }, + + onFileFailed: (fileName: string, _error?: string) => { + const state = get() + const newQueueIndex = state.queueIndex + 1 + const nextFile = state.queue[newQueueIndex] + + set({ + failedFiles: [...state.failedFiles, fileName], + queueIndex: newQueueIndex, + currentFile: nextFile?.name || null, + currentFileProgress: 0, + }) + }, + + onFileProcessing: (fileName: string) => { + const state = get() + const newSet = new Set(state.processingFiles) + newSet.add(fileName) + set({processingFiles: newSet}) + }, + + onFileProcessed: (fileName: string) => { + const state = get() + const newSet = new Set(state.processingFiles) + newSet.delete(fileName) + set({processingFiles: newSet, processedFiles: state.processedFiles + 1}) + }, + + updateFileInQueue: (fileName: string, updatedFile: PhotoInfo) => { + const state = get() + const updatedQueue = state.queue.map((file) => (file.name === fileName ? updatedFile : file)) + set({queue: updatedQueue}) + }, + + removeFilesFromQueue: (fileNames: string[]) => { + if (fileNames.length === 0) return + + const filesToRemove = new Set(fileNames) + const state = get() + const failedFilesSet = new Set(state.failedFiles) + const filesBeforeQueueIndex = state.queue + .slice(0, state.queueIndex) + .filter((file) => filesToRemove.has(file.name)) + const removedBeforeQueueIndex = filesBeforeQueueIndex.length + const removedCompletedBeforeQueueIndex = filesBeforeQueueIndex.filter( + (file) => !failedFilesSet.has(file.name), + ).length + const filteredQueue = state.queue.filter((file) => !filesToRemove.has(file.name)) + const removedCount = state.queue.length - filteredQueue.length + + if (removedCount === 0) return + + const nextQueueIndex = state.queueIndex - removedBeforeQueueIndex + const currentFileRemoved = state.currentFile !== null && filesToRemove.has(state.currentFile) + + set({ + queue: filteredQueue, + totalFiles: state.totalFiles - removedCount, + completedFiles: Math.max(0, state.completedFiles - removedCompletedBeforeQueueIndex), + queueIndex: nextQueueIndex, + failedFiles: state.failedFiles.filter((fileName) => !filesToRemove.has(fileName)), + processingFiles: new Set(Array.from(state.processingFiles).filter((fileName) => !filesToRemove.has(fileName))), + currentFile: currentFileRemoved ? filteredQueue[nextQueueIndex]?.name || null : state.currentFile, + }) + }, + + // Hotspot management + setHotspotInfo: (info: HotspotInfo | null) => set({hotspotInfo: info}), + + setSyncServiceOpenedHotspot: (opened: boolean) => set({syncServiceOpenedHotspot: opened}), + + // Gallery status from glasses + setGlassesGalleryStatus: (photos: number, videos: number, total: number, hasContent: boolean) => { + const state = get() + // Reset to idle if sync is in a terminal state and new content is available + // This allows syncing again after taking new photos + const shouldResetToIdle = + hasContent && (state.syncState === "complete" || state.syncState === "error" || state.syncState === "cancelled") + + set({ + glassesPhotoCount: photos, + glassesVideoCount: videos, + glassesTotalCount: total, + glassesHasContent: hasContent, + ...(shouldResetToIdle ? {syncState: "idle" as SyncState} : {}), + }) + }, + + clearGlassesGalleryStatus: () => + set({ + glassesPhotoCount: 0, + glassesVideoCount: 0, + glassesTotalCount: 0, + glassesHasContent: false, + }), + + // Queue management + setQueue: (files: PhotoInfo[], startIndex: number = 0) => + set({ + // C4: Strip thumbnail_data (base64) from store to prevent OOM + queue: files.map(({thumbnail_data: _thumbnailData, ...rest}) => rest), + queueIndex: startIndex, + totalFiles: files.length, + completedFiles: startIndex, + }), + + advanceQueue: () => { + const state = get() + set({queueIndex: state.queueIndex + 1}) + }, + + clearQueue: () => + set({ + queue: [], + queueIndex: 0, + totalFiles: 0, + completedFiles: 0, + }), + + // Full reset + reset: () => set({...initialState, processingFiles: new Set()}), + })), +) + +// Selector helpers for common subscriptions +export const selectSyncProgress = (state: GallerySyncState) => ({ + syncState: state.syncState, + currentFile: state.currentFile, + currentFileProgress: state.currentFileProgress, + completedFiles: state.completedFiles, + totalFiles: state.totalFiles, + failedFiles: state.failedFiles, + processingFiles: state.processingFiles, +}) + +export const selectIssyncing = (state: GallerySyncState) => + state.syncState === "syncing" || state.syncState === "requesting_hotspot" || state.syncState === "connecting_wifi" + +export const selectGlassesGalleryStatus = (state: GallerySyncState) => ({ + photos: state.glassesPhotoCount, + videos: state.glassesVideoCount, + total: state.glassesTotalCount, + hasContent: state.glassesHasContent, +}) diff --git a/mobile/src/services/ws-types.ts b/mobile/src/services/ws-types.ts index 1076f56918..155c2adf67 100644 --- a/mobile/src/services/ws-types.ts +++ b/mobile/src/services/ws-types.ts @@ -1,6 +1,3 @@ -export enum WebSocketStatus { - DISCONNECTED = "disconnected", - CONNECTING = "connecting", - CONNECTED = "connected", - ERROR = "error", -} +// WebSocketStatus moved into @mentra/island (with the connection store that uses it). +// Re-exported here so existing `@/services/ws-types` importers stay unchanged. +export {WebSocketStatus} from "@mentra/island" diff --git a/mobile/src/stores/connection.ts b/mobile/src/stores/connection.ts index df2ed0a5d2..631dd3c4cc 100644 --- a/mobile/src/stores/connection.ts +++ b/mobile/src/stores/connection.ts @@ -1,60 +1,3 @@ -import {create} from "zustand" - -import {WebSocketStatus} from "@/services/ws-types" - -interface ConnectionState { - status: WebSocketStatus - url: string | null - error: string | null - lastConnectedAt: Date | null - lastDisconnectedAt: Date | null - reconnectAttempts: number - - setStatus: (status: WebSocketStatus) => void - setUrl: (url: string | null) => void - setError: (error: string | null) => void - incrementReconnectAttempts: () => void - resetReconnectAttempts: () => void - reset: () => void -} - -export const useConnectionStore = create((set) => ({ - status: WebSocketStatus.DISCONNECTED, - url: null, - error: null, - lastConnectedAt: null, - lastDisconnectedAt: null, - reconnectAttempts: 0, - - setStatus: (status) => - set((state) => ({ - status, - error: status === WebSocketStatus.ERROR ? state.error : null, - lastConnectedAt: status === WebSocketStatus.CONNECTED ? new Date() : state.lastConnectedAt, - lastDisconnectedAt: - status === WebSocketStatus.DISCONNECTED || status === WebSocketStatus.ERROR - ? new Date() - : state.lastDisconnectedAt, - })), - - setUrl: (url) => set({url}), - - setError: (error) => set({error, status: WebSocketStatus.ERROR, lastDisconnectedAt: new Date()}), - - incrementReconnectAttempts: () => - set((state) => ({ - reconnectAttempts: state.reconnectAttempts + 1, - })), - - resetReconnectAttempts: () => set({reconnectAttempts: 0}), - - reset: () => - set({ - status: WebSocketStatus.DISCONNECTED, - url: null, - error: null, - lastConnectedAt: null, - lastDisconnectedAt: null, - reconnectAttempts: 0, - }), -})) +// Moved into @mentra/island. This shim re-exports it so existing `@/stores/connection` +// importers stay unchanged — the Mentra-app escape hatch (also toolkit.stores.connection). +export {useConnectionStore} from "@mentra/island" diff --git a/mobile/src/stores/core.ts b/mobile/src/stores/core.ts index 041a094f7c..ab5240f119 100644 --- a/mobile/src/stores/core.ts +++ b/mobile/src/stores/core.ts @@ -1,32 +1,3 @@ -import {create} from "zustand" -import {subscribeWithSelector} from "zustand/middleware" -import {BluetoothStatus} from "@mentra/bluetooth-sdk-internal" - -interface CoreState extends BluetoothStatus { - setCoreInfo: (info: Partial) => void - reset: () => void -} - -const initialState: BluetoothStatus = { - // state: - searching: false, - searchingController: false, - micRanking: ["glasses", "phone", "bluetooth", "bluetoothClassic"], - systemMicUnavailable: false, - currentMic: null, - searchResults: [], - wifiScanResults: [], - lastLog: [], - otherBtConnected: false, - galleryModeEnabled: true, -} - -export const useCoreStore = create()( - subscribeWithSelector((set) => ({ - ...initialState, - - setCoreInfo: (info) => set((state) => ({...state, ...info})), - - reset: () => set(initialState), - })), -) +// Moved into @mentra/island. This shim re-exports it so existing `@/stores/core` +// importers stay unchanged — the Mentra-app escape hatch (also toolkit.stores.core). +export {useCoreStore} from "@mentra/island" diff --git a/mobile/src/stores/gallerySync.ts b/mobile/src/stores/gallerySync.ts index 681ee3584d..ed9afcc5a0 100644 --- a/mobile/src/stores/gallerySync.ts +++ b/mobile/src/stores/gallerySync.ts @@ -1,361 +1,9 @@ -/** - * Gallery Sync Store - * Manages gallery sync state independently of UI lifecycle - */ - -import {create} from "zustand" -import {subscribeWithSelector} from "zustand/middleware" - -import {PhotoInfo} from "@/types/asg" - -// Sync state machine states -export type SyncState = - | "idle" - | "requesting_hotspot" - | "connecting_wifi" - | "syncing" - | "complete" - | "error" - | "cancelled" - -export interface HotspotInfo { - ssid: string - password: string - ip: string -} - -export interface SyncQueue { - files: PhotoInfo[] - currentIndex: number - startedAt: number - hotspotInfo: HotspotInfo -} - -export interface GallerySyncInfo { - // State machine - syncState: SyncState - - // Progress tracking - currentFile: string | null - currentFileProgress: number // 0-100 - completedFiles: number - totalFiles: number - failedFiles: string[] - - // Queue (persisted separately via localStorageService) - queue: PhotoInfo[] - queueIndex: number - - // Hotspot info - hotspotInfo: HotspotInfo | null - syncServiceOpenedHotspot: boolean - - // Gallery status from glasses - glassesPhotoCount: number - glassesVideoCount: number - glassesTotalCount: number - glassesHasContent: boolean - - // Error tracking - lastError: string | null -} - -interface GallerySyncState extends GallerySyncInfo { - // State transitions - setSyncState: (state: SyncState) => void - setRequestingHotspot: () => void - setConnectingWifi: () => void - setSyncing: (files: PhotoInfo[]) => void - setSyncComplete: () => void - setSyncError: (error: string) => void - setSyncCancelled: () => void - - // Progress updates - setCurrentFile: (fileName: string | null, progress: number) => void - onFileProgress: (fileName: string, progress: number) => void - onFileComplete: (fileName: string) => void - onFileFailed: (fileName: string, error?: string) => void - onFileProcessing: (fileName: string) => void - onFileProcessed: (fileName: string) => void - updateFileInQueue: (fileName: string, updatedFile: PhotoInfo) => void - removeFilesFromQueue: (fileNames: string[]) => void - - // Processing queue tracking - processingFiles: Set - processedFiles: number - - // Hotspot management - setHotspotInfo: (info: HotspotInfo | null) => void - setSyncServiceOpenedHotspot: (opened: boolean) => void - - // Gallery status from glasses - setGlassesGalleryStatus: (photos: number, videos: number, total: number, hasContent: boolean) => void - clearGlassesGalleryStatus: () => void - - // Queue management (for resume) - setQueue: (files: PhotoInfo[], startIndex?: number) => void - advanceQueue: () => void - clearQueue: () => void - - // Full reset - reset: () => void -} - -const initialState: GallerySyncInfo & {processingFiles: Set; processedFiles: number} = { - syncState: "idle", - currentFile: null, - currentFileProgress: 0, - completedFiles: 0, - totalFiles: 0, - failedFiles: [], - queue: [], - queueIndex: 0, - hotspotInfo: null, - syncServiceOpenedHotspot: false, - glassesPhotoCount: 0, - glassesVideoCount: 0, - glassesTotalCount: 0, - glassesHasContent: false, - lastError: null, - processingFiles: new Set(), - processedFiles: 0, -} - -export const useGallerySyncStore = create()( - subscribeWithSelector((set, get) => ({ - ...initialState, - - // State transitions - setSyncState: (syncState: SyncState) => set({syncState}), - - setRequestingHotspot: () => - set({ - syncState: "requesting_hotspot", - lastError: null, - failedFiles: [], - }), - - setConnectingWifi: () => - set({ - syncState: "connecting_wifi", - }), - - setSyncing: (files: PhotoInfo[]) => - set({ - syncState: "syncing", - // C4: Strip thumbnail_data (base64) from store to prevent OOM - queue: files.map(({thumbnail_data: _thumbnailData, ...rest}) => rest), - queueIndex: 0, - totalFiles: files.length, - completedFiles: 0, - currentFile: files.length > 0 ? files[0].name : null, - currentFileProgress: 0, - failedFiles: [], - lastError: null, - processedFiles: 0, - processingFiles: new Set(), - }), - - setSyncComplete: () => - set({ - syncState: "complete", - currentFile: null, - currentFileProgress: 0, - // Keep queue intact so photos remain visible after sync - // Don't clear: queue: [], queueIndex: 0 - }), - - setSyncError: (error: string) => - set({ - syncState: "error", - lastError: error, - currentFile: null, - currentFileProgress: 0, - }), - - setSyncCancelled: () => - set({ - syncState: "cancelled", - currentFile: null, - currentFileProgress: 0, - queue: [], - queueIndex: 0, - }), - - // Progress updates - setCurrentFile: (fileName: string | null, progress: number) => - set({ - currentFile: fileName, - currentFileProgress: Math.max(0, Math.min(100, progress)), - }), - - onFileProgress: (fileName: string, progress: number) => { - const state = get() - const clampedProgress = Math.max(0, Math.min(100, progress)) - // Throttle: skip update if same file and same percentage (prevents Zustand flood) - if (clampedProgress === state.currentFileProgress && fileName === state.currentFile) { - return - } - set({ - currentFile: fileName, - currentFileProgress: clampedProgress, - }) - }, - - onFileComplete: (_fileName: string) => { - const state = get() - const newCompletedFiles = state.completedFiles + 1 - const newQueueIndex = state.queueIndex + 1 - const nextFile = state.queue[newQueueIndex] - - set({ - completedFiles: newCompletedFiles, - queueIndex: newQueueIndex, - currentFile: nextFile?.name || null, - currentFileProgress: 0, - }) - }, - - onFileFailed: (fileName: string, _error?: string) => { - const state = get() - const newQueueIndex = state.queueIndex + 1 - const nextFile = state.queue[newQueueIndex] - - set({ - failedFiles: [...state.failedFiles, fileName], - queueIndex: newQueueIndex, - currentFile: nextFile?.name || null, - currentFileProgress: 0, - }) - }, - - onFileProcessing: (fileName: string) => { - const state = get() - const newSet = new Set(state.processingFiles) - newSet.add(fileName) - set({processingFiles: newSet}) - }, - - onFileProcessed: (fileName: string) => { - const state = get() - const newSet = new Set(state.processingFiles) - newSet.delete(fileName) - set({processingFiles: newSet, processedFiles: state.processedFiles + 1}) - }, - - updateFileInQueue: (fileName: string, updatedFile: PhotoInfo) => { - const state = get() - const updatedQueue = state.queue.map((file) => (file.name === fileName ? updatedFile : file)) - set({queue: updatedQueue}) - }, - - removeFilesFromQueue: (fileNames: string[]) => { - if (fileNames.length === 0) return - - const filesToRemove = new Set(fileNames) - const state = get() - const failedFilesSet = new Set(state.failedFiles) - const filesBeforeQueueIndex = state.queue - .slice(0, state.queueIndex) - .filter((file) => filesToRemove.has(file.name)) - const removedBeforeQueueIndex = filesBeforeQueueIndex.length - const removedCompletedBeforeQueueIndex = filesBeforeQueueIndex.filter( - (file) => !failedFilesSet.has(file.name), - ).length - const filteredQueue = state.queue.filter((file) => !filesToRemove.has(file.name)) - const removedCount = state.queue.length - filteredQueue.length - - if (removedCount === 0) return - - const nextQueueIndex = state.queueIndex - removedBeforeQueueIndex - const currentFileRemoved = state.currentFile !== null && filesToRemove.has(state.currentFile) - - set({ - queue: filteredQueue, - totalFiles: state.totalFiles - removedCount, - completedFiles: Math.max(0, state.completedFiles - removedCompletedBeforeQueueIndex), - queueIndex: nextQueueIndex, - failedFiles: state.failedFiles.filter((fileName) => !filesToRemove.has(fileName)), - processingFiles: new Set(Array.from(state.processingFiles).filter((fileName) => !filesToRemove.has(fileName))), - currentFile: currentFileRemoved ? filteredQueue[nextQueueIndex]?.name || null : state.currentFile, - }) - }, - - // Hotspot management - setHotspotInfo: (info: HotspotInfo | null) => set({hotspotInfo: info}), - - setSyncServiceOpenedHotspot: (opened: boolean) => set({syncServiceOpenedHotspot: opened}), - - // Gallery status from glasses - setGlassesGalleryStatus: (photos: number, videos: number, total: number, hasContent: boolean) => { - const state = get() - // Reset to idle if sync is in a terminal state and new content is available - // This allows syncing again after taking new photos - const shouldResetToIdle = - hasContent && (state.syncState === "complete" || state.syncState === "error" || state.syncState === "cancelled") - - set({ - glassesPhotoCount: photos, - glassesVideoCount: videos, - glassesTotalCount: total, - glassesHasContent: hasContent, - ...(shouldResetToIdle ? {syncState: "idle" as SyncState} : {}), - }) - }, - - clearGlassesGalleryStatus: () => - set({ - glassesPhotoCount: 0, - glassesVideoCount: 0, - glassesTotalCount: 0, - glassesHasContent: false, - }), - - // Queue management - setQueue: (files: PhotoInfo[], startIndex: number = 0) => - set({ - // C4: Strip thumbnail_data (base64) from store to prevent OOM - queue: files.map(({thumbnail_data: _thumbnailData, ...rest}) => rest), - queueIndex: startIndex, - totalFiles: files.length, - completedFiles: startIndex, - }), - - advanceQueue: () => { - const state = get() - set({queueIndex: state.queueIndex + 1}) - }, - - clearQueue: () => - set({ - queue: [], - queueIndex: 0, - totalFiles: 0, - completedFiles: 0, - }), - - // Full reset - reset: () => set({...initialState, processingFiles: new Set()}), - })), -) - -// Selector helpers for common subscriptions -export const selectSyncProgress = (state: GallerySyncState) => ({ - syncState: state.syncState, - currentFile: state.currentFile, - currentFileProgress: state.currentFileProgress, - completedFiles: state.completedFiles, - totalFiles: state.totalFiles, - failedFiles: state.failedFiles, - processingFiles: state.processingFiles, -}) - -export const selectIssyncing = (state: GallerySyncState) => - state.syncState === "syncing" || state.syncState === "requesting_hotspot" || state.syncState === "connecting_wifi" - -export const selectGlassesGalleryStatus = (state: GallerySyncState) => ({ - photos: state.glassesPhotoCount, - videos: state.glassesVideoCount, - total: state.glassesTotalCount, - hasContent: state.glassesHasContent, -}) +// Moved into @mentra/island. This shim re-exports it so existing `@/stores/gallerySync` +// importers stay unchanged — the Mentra-app escape hatch (also toolkit.stores.gallerySync). +export { + useGallerySyncStore, + selectSyncProgress, + selectIssyncing, + selectGlassesGalleryStatus, +} from "@mentra/island" +export type {PhotoInfo, SyncState, HotspotInfo, SyncQueue, GallerySyncInfo} from "@mentra/island" diff --git a/mobile/src/types/asg/index.ts b/mobile/src/types/asg/index.ts index 9353750040..1804e747d5 100644 --- a/mobile/src/types/asg/index.ts +++ b/mobile/src/types/asg/index.ts @@ -2,24 +2,11 @@ * Type definitions for the ASG package */ -export interface PhotoInfo { - name: string - url: string - download: string - size: number - modified: string | number // Unix timestamp (milliseconds) - can be string or number from API - mime_type?: string - is_video?: boolean - thumbnail_data?: string - downloaded_at?: number - // Video duration in milliseconds (from glasses sync response) - duration?: number - // New fields for filesystem storage - filePath?: string - thumbnailPath?: string - // Glasses model that captured this media - glassesModel?: string -} +// PhotoInfo moved into @mentra/island (with the gallerySync store that owns it). +// Imported as a local name (sibling types below use it) and re-exported so existing +// `@/types/asg` importers stay unchanged. +import type {PhotoInfo} from "@mentra/island" +export type {PhotoInfo} export interface CaptureFile { name: string // "IMG_xxx/base.jpg" or "IMG_xxx.jpg" (legacy) From 1cfc74df65a7600a5600f8321b6b158ee067f14e Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 11:19:44 -0700 Subject: [PATCH 07/80] island: add toolkit.glasses core facade + facade-buildout tracking spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toolkit.glasses now carries the core surface (the central OEM glasses API): - connection: connectDefault / disconnect / forget (bluetooth-sdk passthrough) - read-model: status() / onStatus() / info() — a curated projection of the island-owned glasses store, so the store's field layout doesn't leak into UIs - capabilities() from the model table; requestVersionInfo() - input events: onButtonPress / onTouchGesture (passthrough listeners) - wifi nested under it Also adds agents/island-facade-buildout.md — the per-domain tracking spec for the rest of the buildout (two move-patterns, sequence, the settings keystone, the git-merge-dev gate for session). Verified: typecheck 0 new errors; jest 269 pass (new glassesFacade suite green). --- agents/island-facade-buildout.md | 50 +++++++++++++ mobile/jest.setup.js | 10 +++ mobile/modules/island/src/facades/glasses.ts | 77 ++++++++++++++++++++ mobile/modules/island/src/island.ts | 6 +- mobile/src/__tests__/glassesFacade.test.ts | 58 +++++++++++++++ 5 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 agents/island-facade-buildout.md create mode 100644 mobile/modules/island/src/facades/glasses.ts create mode 100644 mobile/src/__tests__/glassesFacade.test.ts diff --git a/agents/island-facade-buildout.md b/agents/island-facade-buildout.md new file mode 100644 index 0000000000..3160234d73 --- /dev/null +++ b/agents/island-facade-buildout.md @@ -0,0 +1,50 @@ +# Island facade buildout — tracking spec + +Goal: build the OEM-facing `toolkit.*` typed facades by moving the backing logic +into `@mentra/island`, domain by domain, on branch `aisraelov/island-namespace-wifi` +(PR #3167). One branch, one commit per domain, green at every commit. + +## Two move-patterns +1. **Self-contained logic** — move the file into island, fix relative imports + (btsdk types via `../../../bluetooth-sdk/build/_internal`), wrap in a facade. + Shim the old host path if the app still imports it. (Stores, speech, logs, + permissions, incidents.) +2. **Host-service-coupled** — move the logic in, but inject an adapter for the host + dependency via `configureRuntime` (the existing seam). Settings → `RestComms` + + `storage`; session → `cloud-client`. + +Rule: stores are the Mentra-app escape hatch (`toolkit.stores.*`), NOT the OEM +contract. OEMs use the typed facade functions. + +## Verification per commit +`npx tsc --noEmit -p .` (resolves `@mentra/island`→src, validates the real code) + +`bun run test`. The island standalone build can't run locally (cloud-v2 `zod` not +installed in this checkout) — CI confirms it; use the proven relative-`_internal` +pattern for btsdk types. + +## Domains +| Domain | Backing logic lives | Pattern | Status | +|---|---|---|---| +| glasses.wifi | btsdk passthrough + glasses store | 1 | DONE (#3167) | +| display.mirror | island display store | 1 | DONE (#3167) | +| glasses (core) | glasses store + btsdk + ConnectionCoordinator | 1 | in progress | +| speech | STT/TTSModelManager (already island) | 1 | todo | +| logs | MentraJSLogPipeline (already island) | 1 | todo | +| permissions | `utils/PermissionsUtils.tsx` (host) | 1 | todo | +| incidents | `services/bugReport/*` (host) | 1 | todo | +| dev | `utils/cloudClient/devHost.ts` + core store | 1 | todo | +| miniapps | apps store + LocalMiniappRuntime (island) + MiniappCatalog | 1/hard (WebView) | todo | +| pairing | pairing screens state machine (readiness primitive already island) | 1 (extract) | todo | +| **settings** | `stores/settings.ts` (964 LOC) + `RestComms` + `storage` | **2 (keystone)** | todo — own careful commit; unblocks settings + glasses.settings + phoneNotifications | +| glasses.settings | settings store + btsdk | 2 (after settings) | blocked on settings | +| phoneNotifications | settings store + crust + permissions | 2 (after settings) | blocked on settings | +| gallery | `services/asg/gallerySyncService.ts` (~1000 LOC, hotspot) | hard | todo | +| notifications | scattered detectors → new event bus | hard (new) | todo | +| session | `cloud-client` (cloud-v2) | 2 | needs `git merge dev` (cloud-v2) first | +| cloudClientStatus (store) | cloud-client types | — | rides with session | + +## Sequence +Buildable/island-resident first (green every commit): glasses-core → speech → logs +→ permissions → incidents → dev. Then the **settings keystone** (own commit). Then +glasses.settings + phoneNotifications. Then pairing, gallery, miniapps WebView, +notifications. Last: `git merge dev`, then session + cloudClientStatus. diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 33daa798e3..b444f839b0 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -273,6 +273,16 @@ jest.mock("@mentra/island", () => { start: jest.fn(() => Promise.resolve()), stop: jest.fn(() => Promise.resolve()), glasses: { + connectDefault: jest.fn(() => Promise.resolve()), + disconnect: jest.fn(() => Promise.resolve()), + forget: jest.fn(() => Promise.resolve()), + status: jest.fn(() => ({state: "disconnected"})), + onStatus: jest.fn(() => () => {}), + info: jest.fn(() => ({})), + capabilities: jest.fn(() => ({})), + requestVersionInfo: jest.fn(() => Promise.resolve()), + onButtonPress: jest.fn(() => () => {}), + onTouchGesture: jest.fn(() => () => {}), wifi: { scan: jest.fn(() => Promise.resolve([])), connect: jest.fn(() => Promise.resolve()), diff --git a/mobile/modules/island/src/facades/glasses.ts b/mobile/modules/island/src/facades/glasses.ts new file mode 100644 index 0000000000..efd957e313 --- /dev/null +++ b/mobile/modules/island/src/facades/glasses.ts @@ -0,0 +1,77 @@ +/** + * glasses facade — the core `toolkit.glasses.*` surface: connection actions (over + * the bluetooth-sdk passthrough), a curated status/info read-model (projected from + * the island-owned glasses store), capabilities (from the model table), and the + * discrete input events. The `wifi` sub-facade is nested here. + * + * Read-models project the raw glasses store into the stable shape OEM UIs consume, + * so the device store's field layout doesn't leak into every screen. + */ +import BluetoothSdk from "@mentra/bluetooth-sdk" +import type {ButtonPressEvent, TouchEvent} from "@mentra/bluetooth-sdk" +import {useGlassesStore} from "../stores/glasses" +import {isGlassesReady} from "../services/GlassesReadiness" +import {getModelCapabilities, type DeviceTypes} from "../types" +import {glassesWifi} from "./glassesWifi" + +function projectStatus() { + const s = useGlassesStore.getState() + return { + state: s.connection.state, + fullyBooted: isGlassesReady(s.connection), + battery: s.batteryLevel, + charging: s.charging, + case: {battery: s.caseBatteryLevel, charging: s.caseCharging, open: s.caseOpen, removed: s.caseRemoved}, + signal: s.signalStrength, + micEnabled: s.micEnabled, + vadEnabled: s.voiceActivityDetectionEnabled, + btClassic: s.bluetoothClassicConnected, + } +} + +function projectInfo() { + const s = useGlassesStore.getState() + return { + model: s.deviceModel, + style: s.style, + color: s.color, + firmwareVersion: s.firmwareVersion, + mtkFirmware: s.mtkFirmwareVersion, + besFirmware: s.besFirmwareVersion, + serialNumber: s.serialNumber, + buildNumber: s.buildNumber, + btMac: s.bluetoothMacAddress, + } +} + +export type GlassesStatusSnapshot = ReturnType +export type GlassesInfoSnapshot = ReturnType + +export const glasses = { + // --- connection (bluetooth-sdk passthrough) --- + connectDefault: (): Promise => BluetoothSdk.connectDefault(), + disconnect: (): Promise => BluetoothSdk.disconnect(), + forget: (): Promise => BluetoothSdk.forget(), + + // --- read-model (projected from the island-owned glasses store) --- + status: (): GlassesStatusSnapshot => projectStatus(), + onStatus: (cb: (status: GlassesStatusSnapshot) => void): (() => void) => + useGlassesStore.subscribe(() => cb(projectStatus())), + info: (): GlassesInfoSnapshot => projectInfo(), + capabilities: () => getModelCapabilities(useGlassesStore.getState().deviceModel as DeviceTypes), + /** Ask the glasses to report fresh firmware/version info (updates the store). */ + requestVersionInfo: () => BluetoothSdk.requestVersionInfo(), + + // --- discrete input events (passthrough listeners) --- + onButtonPress: (cb: (event: ButtonPressEvent) => void): (() => void) => { + const sub = BluetoothSdk.addListener("button_press", cb) + return () => sub.remove() + }, + onTouchGesture: (cb: (event: TouchEvent) => void): (() => void) => { + const sub = BluetoothSdk.addListener("touch_event", cb) + return () => sub.remove() + }, + + // --- sub-facades --- + wifi: glassesWifi, +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index d3627908e6..a66a273b0d 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -9,7 +9,7 @@ * `BluetoothSdk` passthrough) stay until every screen has moved onto `island.*`. */ import {configure, start, stop} from "./runtime/bootstrap" -import {glassesWifi} from "./facades/glassesWifi" +import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" @@ -22,9 +22,7 @@ export const toolkit = { configure, start, stop, - glasses: { - wifi: glassesWifi, - }, + glasses, display: { mirror: displayMirror, }, diff --git a/mobile/src/__tests__/glassesFacade.test.ts b/mobile/src/__tests__/glassesFacade.test.ts new file mode 100644 index 0000000000..a8ced81377 --- /dev/null +++ b/mobile/src/__tests__/glassesFacade.test.ts @@ -0,0 +1,58 @@ +// Imports the real glasses facade by path (not via "@mentra/island", which jest +// mocks) so the actual projection + delegation run under the mobile jest runner. +import BluetoothSdk from "@mentra/bluetooth-sdk" +import {glasses} from "../../modules/island/src/facades/glasses" +import {useGlassesStore} from "../../modules/island/src/stores/glasses" + +describe("glasses facade", () => { + beforeEach(() => { + jest.clearAllMocks() + useGlassesStore.getState().reset() + }) + + it("status() projects the glasses store into the read-model", () => { + useGlassesStore.setState({ + connection: {state: "connected", fullyBooted: true} as never, + batteryLevel: 80, + charging: true, + signalStrength: -50, + micEnabled: true, + bluetoothClassicConnected: true, + }) + const st = glasses.status() + expect(st.state).toBe("connected") + expect(st.fullyBooted).toBe(true) + expect(st.battery).toBe(80) + expect(st.charging).toBe(true) + expect(st.micEnabled).toBe(true) + expect(st.btClassic).toBe(true) + }) + + it("info() projects device info from the store", () => { + useGlassesStore.setState({deviceModel: "Even Realities G1", firmwareVersion: "1.2.3", serialNumber: "SN1"}) + const info = glasses.info() + expect(info.model).toBe("Even Realities G1") + expect(info.firmwareVersion).toBe("1.2.3") + expect(info.serialNumber).toBe("SN1") + }) + + it("connectDefault / disconnect / forget delegate to bluetooth-sdk", async () => { + await glasses.connectDefault() + expect(BluetoothSdk.connectDefault).toHaveBeenCalled() + await glasses.disconnect() + expect(BluetoothSdk.disconnect).toHaveBeenCalled() + await glasses.forget() + expect(BluetoothSdk.forget).toHaveBeenCalled() + }) + + it("onStatus() fires on store changes and stops after unsubscribe", () => { + const cb = jest.fn() + const unsub = glasses.onStatus(cb) + useGlassesStore.setState({batteryLevel: 42}) + expect(cb).toHaveBeenCalled() + unsub() + cb.mockClear() + useGlassesStore.setState({batteryLevel: 10}) + expect(cb).not.toHaveBeenCalled() + }) +}) From 5738c65dc2e67806c760fb2a497087669fc42af8 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 11:34:24 -0700 Subject: [PATCH 08/80] island: add toolkit.speech facade (STT/TTS model management) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toolkit.speech.{stt,tts} — currentLanguage / languages / languageInfo / download / activate / cancelDownload / deleteModel, wrapping the in-island STT+TTS model managers via type-safe Parameters<> forwarding. STT also exposes status() / onStatusChanged() over the offline-model service. Logic was already in island, so this is a thin facade. Tested (managers mocked to avoid native deps). Verified: typecheck 0 new errors; jest 272 pass. --- mobile/jest.setup.js | 22 +++++++++ mobile/modules/island/src/facades/speech.ts | 35 +++++++++++++ mobile/modules/island/src/island.ts | 2 + mobile/src/__tests__/speechFacade.test.ts | 55 +++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 mobile/modules/island/src/facades/speech.ts create mode 100644 mobile/src/__tests__/speechFacade.test.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index b444f839b0..ab4344cf0b 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -291,6 +291,28 @@ jest.mock("@mentra/island", () => { onStatus: jest.fn(() => () => {}), }, }, + speech: { + stt: { + currentLanguage: jest.fn(() => "en"), + languages: jest.fn(() => []), + languageInfo: jest.fn(() => Promise.resolve([])), + download: jest.fn(() => Promise.resolve()), + activate: jest.fn(), + cancelDownload: jest.fn(() => Promise.resolve()), + deleteModel: jest.fn(() => Promise.resolve()), + status: jest.fn(() => null), + onStatusChanged: jest.fn(() => () => {}), + }, + tts: { + currentLanguage: jest.fn(() => "en"), + languages: jest.fn(() => []), + languageInfo: jest.fn(() => Promise.resolve([])), + download: jest.fn(() => Promise.resolve()), + activate: jest.fn(), + cancelDownload: jest.fn(() => Promise.resolve()), + deleteModel: jest.fn(() => Promise.resolve()), + }, + }, display: { mirror: { current: jest.fn(() => null), diff --git a/mobile/modules/island/src/facades/speech.ts b/mobile/modules/island/src/facades/speech.ts new file mode 100644 index 0000000000..e1ce593719 --- /dev/null +++ b/mobile/modules/island/src/facades/speech.ts @@ -0,0 +1,35 @@ +/** + * speech facade — `toolkit.speech.{stt,tts}`: on-device STT/TTS model management, + * wrapping the in-island model managers. Thin passthrough (args forwarded with + * exact types via Parameters<>), so the facade never drifts from the managers. + * STT additionally exposes the auto-download status stream (the TTS path has none). + */ +import sttModelManager from "../services/STTModelManager" +import ttsModelManager from "../services/TTSModelManager" +import offlineSpeechModelService from "../services/OfflineSpeechModelService" + +export const speech = { + stt: { + currentLanguage: () => sttModelManager.getCurrentLanguage(), + languages: () => sttModelManager.getAvailableLanguages(), + languageInfo: () => sttModelManager.getAllLanguageInfo(), + download: (...args: Parameters) => sttModelManager.downloadModel(...args), + activate: (code: string) => sttModelManager.setCurrentLanguage(code), + cancelDownload: () => sttModelManager.cancelDownload(), + deleteModel: (...args: Parameters) => sttModelManager.deleteModel(...args), + /** Current auto-download status of the offline STT model. */ + status: () => offlineSpeechModelService.getStatus(), + /** Subscribe to auto-download status changes; returns an unsubscribe. */ + onStatusChanged: (...args: Parameters) => + offlineSpeechModelService.subscribe(...args), + }, + tts: { + currentLanguage: () => ttsModelManager.getCurrentLanguage(), + languages: () => ttsModelManager.getAvailableLanguages(), + languageInfo: () => ttsModelManager.getAllLanguageInfo(), + download: (...args: Parameters) => ttsModelManager.downloadModel(...args), + activate: (code: string) => ttsModelManager.setCurrentLanguage(code), + cancelDownload: () => ttsModelManager.cancelDownload(), + deleteModel: (...args: Parameters) => ttsModelManager.deleteModel(...args), + }, +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index a66a273b0d..2f8693f612 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -11,6 +11,7 @@ import {configure, start, stop} from "./runtime/bootstrap" import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" +import {speech} from "./facades/speech" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -23,6 +24,7 @@ export const toolkit = { start, stop, glasses, + speech, display: { mirror: displayMirror, }, diff --git a/mobile/src/__tests__/speechFacade.test.ts b/mobile/src/__tests__/speechFacade.test.ts new file mode 100644 index 0000000000..b1685c9ddd --- /dev/null +++ b/mobile/src/__tests__/speechFacade.test.ts @@ -0,0 +1,55 @@ +// Mock the in-island model managers (they pull native deps) so the real speech +// facade's forwarding can be exercised under the mobile jest runner. +jest.mock("../../modules/island/src/services/STTModelManager", () => ({ + __esModule: true, + default: { + getCurrentLanguage: jest.fn(() => "en"), + getAvailableLanguages: jest.fn(() => [{code: "en"}]), + getAllLanguageInfo: jest.fn(() => Promise.resolve([])), + downloadModel: jest.fn(() => Promise.resolve()), + setCurrentLanguage: jest.fn(), + cancelDownload: jest.fn(() => Promise.resolve()), + deleteModel: jest.fn(() => Promise.resolve()), + }, +})) +jest.mock("../../modules/island/src/services/TTSModelManager", () => ({ + __esModule: true, + default: { + getCurrentLanguage: jest.fn(() => "en"), + getAvailableLanguages: jest.fn(() => []), + getAllLanguageInfo: jest.fn(() => Promise.resolve([])), + downloadModel: jest.fn(() => Promise.resolve()), + setCurrentLanguage: jest.fn(), + cancelDownload: jest.fn(() => Promise.resolve()), + deleteModel: jest.fn(() => Promise.resolve()), + }, +})) +jest.mock("../../modules/island/src/services/OfflineSpeechModelService", () => ({ + __esModule: true, + default: {getStatus: jest.fn(() => null), subscribe: jest.fn(() => () => {})}, +})) + +import sttModelManager from "../../modules/island/src/services/STTModelManager" +import ttsModelManager from "../../modules/island/src/services/TTSModelManager" +import {speech} from "../../modules/island/src/facades/speech" + +describe("speech facade", () => { + beforeEach(() => jest.clearAllMocks()) + + it("stt.activate delegates to setCurrentLanguage", () => { + speech.stt.activate("fr") + expect(sttModelManager.setCurrentLanguage).toHaveBeenCalledWith("fr") + }) + + it("stt.download forwards its args to downloadModel", async () => { + const onProgress = jest.fn() + await speech.stt.download("de", onProgress) + expect(sttModelManager.downloadModel).toHaveBeenCalledWith("de", onProgress) + }) + + it("tts methods delegate to the TTS manager", () => { + expect(speech.tts.currentLanguage()).toBe("en") + speech.tts.activate("es") + expect(ttsModelManager.setCurrentLanguage).toHaveBeenCalledWith("es") + }) +}) From e6d80d431a3c2ad4b7761ad87cbc6ecca049dc02 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 12:03:24 -0700 Subject: [PATCH 09/80] island: fix mobile-CI install (drop spurious cloud-client dep + non-fatal island build) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mobile CI (and dev's, since the cloud-v2 merge) failed install on `@mentra/cloud-client - 404`. Two install-time causes, both fixed: 1) mobile/modules/island/package.json declared `"@mentra/cloud-client": "*"`, but cloud-client is resolved via metro + tsconfig PATH aliases (per the cloud-v2 setup), not node_modules — the `*` made bun try npm and 404. Removed it; the app still resolves cloud-client via the alias (island only type-imports it). 2) postinstall builds island standalone (expo-module), whose isolated tsconfig has no cloud-v2 aliases, so it can't resolve `@mentra/cloud-client` / `@mentra/cloud-runtime/protocol` and failed the whole install. But island's build/ is NOT consumed: metro (metro.config.js) and tsconfig both resolve `@mentra/island` to src/. Made the build non-fatal (warn) — same precedent as the root @mentra/miniapp postinstall. Unblocks mobile CI (mine + dev's). App typecheck + jest green. --- mobile/modules/island/package.json | 1 - mobile/modules/island/tsconfig.json | 9 +-------- mobile/scripts/postinstall.mjs | 14 +++++++++++++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/mobile/modules/island/package.json b/mobile/modules/island/package.json index 2251c50a5a..90164adb40 100644 --- a/mobile/modules/island/package.json +++ b/mobile/modules/island/package.json @@ -54,7 +54,6 @@ "react-native-share": "*", "@dr.pogodin/react-native-fs": "*", "@mentra/bluetooth-sdk": "*", - "@mentra/cloud-client": "*", "@mentra/miniapp": "*" } } diff --git a/mobile/modules/island/tsconfig.json b/mobile/modules/island/tsconfig.json index fb50916b31..cbe9e1971c 100644 --- a/mobile/modules/island/tsconfig.json +++ b/mobile/modules/island/tsconfig.json @@ -2,14 +2,7 @@ { "extends": "expo-module-scripts/tsconfig.base", "compilerOptions": { - "outDir": "./build", - "baseUrl": ".", - "paths": { - "@mentra/cloud-client": ["../../../cloud-v2/packages/cloud-client/src/index.ts"], - "@mentra/cloud-client/*": ["../../../cloud-v2/packages/cloud-client/*"], - "@mentra/cloud-runtime/protocol": ["../../../cloud-v2/packages/runtime/src/protocol/index.ts"], - "@mentra/cloud-runtime/protocol/*": ["../../../cloud-v2/packages/runtime/src/protocol/*"] - } + "outDir": "./build" }, "include": ["./src"], "exclude": ["**/__mocks__/*", "**/__tests__/*", "**/__rsc_tests__/*"] diff --git a/mobile/scripts/postinstall.mjs b/mobile/scripts/postinstall.mjs index 4507ea0e1b..d80e5ab969 100755 --- a/mobile/scripts/postinstall.mjs +++ b/mobile/scripts/postinstall.mjs @@ -17,7 +17,19 @@ await $({ stdio: 'inherit', cwd: 'modules/miniapp' })`bun run prepare`; // island depends on bluetooth-sdk + miniapp build outputs, so its prepare // (renamed to build:module) runs here instead of being auto-triggered by bun // install in parallel with its workspace deps. -await $({ stdio: 'inherit', cwd: 'modules/island' })`bun run build:module`; +// +// NON-FATAL: island's compiled build/ is NOT consumed by the app or CI — both +// metro (metro.config.js) and tsconfig resolve `@mentra/island` to src/, not +// build/. Its isolated expo-module build can't resolve the cloud-v2 packages it +// imports (`@mentra/cloud-client`, `@mentra/cloud-runtime/protocol`) because the +// generated standalone tsconfig has no metro/tsconfig path aliases. So let it warn +// rather than fail the whole install — same precedent as the root @mentra/miniapp +// postinstall build. (The standalone build only matters for Phase-2 publishing.) +try { + await $({ stdio: 'inherit', cwd: 'modules/island' })`bun run build:module`; +} catch { + console.warn('\n[postinstall] WARNING: @mentra/island build:module failed (non-fatal — the app and CI resolve @mentra/island from src/, not build/).\n'); +} // Apply the Supabase patch via patch-package from its OWN directory // (patches-runtime/, holding only this one patch). It strips a dynamic From e0dc26eabc472be204571a3f5489a8007673b699 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 12:08:03 -0700 Subject: [PATCH 10/80] island: restore island tsconfig cloud-v2 paths + export store state types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore the cloud-v2 path aliases in island/tsconfig.json (a local build:module regenerated and stripped them in the prior commit, and I accidentally committed that) — they let island's standalone build resolve cloud-client/cloud-runtime. - Export the 5 store state interfaces (CoreState, ConnectionState, DisplayStore, GlassesState, GallerySyncState) so toolkit.stores' declaration emit can name them (fixes the TS4023 the standalone build hit). --- mobile/modules/island/src/stores/connection.ts | 2 +- mobile/modules/island/src/stores/core.ts | 2 +- mobile/modules/island/src/stores/display.ts | 2 +- mobile/modules/island/src/stores/gallerySync.ts | 2 +- mobile/modules/island/src/stores/glasses.ts | 2 +- mobile/modules/island/tsconfig.json | 9 ++++++++- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mobile/modules/island/src/stores/connection.ts b/mobile/modules/island/src/stores/connection.ts index 956c067895..d1168b846d 100644 --- a/mobile/modules/island/src/stores/connection.ts +++ b/mobile/modules/island/src/stores/connection.ts @@ -10,7 +10,7 @@ export enum WebSocketStatus { ERROR = "error", } -interface ConnectionState { +export interface ConnectionState { status: WebSocketStatus url: string | null error: string | null diff --git a/mobile/modules/island/src/stores/core.ts b/mobile/modules/island/src/stores/core.ts index a229d16d80..e367193ba8 100644 --- a/mobile/modules/island/src/stores/core.ts +++ b/mobile/modules/island/src/stores/core.ts @@ -2,7 +2,7 @@ import {create} from "zustand" import {subscribeWithSelector} from "zustand/middleware" import type {BluetoothStatus} from "../../../bluetooth-sdk/build/_internal" -interface CoreState extends BluetoothStatus { +export interface CoreState extends BluetoothStatus { setCoreInfo: (info: Partial) => void reset: () => void } diff --git a/mobile/modules/island/src/stores/display.ts b/mobile/modules/island/src/stores/display.ts index 39c26dcad0..4136dd5647 100644 --- a/mobile/modules/island/src/stores/display.ts +++ b/mobile/modules/island/src/stores/display.ts @@ -35,7 +35,7 @@ function extractDisplayText(displayEvent: any): string[] { } } -interface DisplayStore { +export interface DisplayStore { currentEvent: any dashboardEvent: any mainEvent: any diff --git a/mobile/modules/island/src/stores/gallerySync.ts b/mobile/modules/island/src/stores/gallerySync.ts index 380ad821cc..533003ceab 100644 --- a/mobile/modules/island/src/stores/gallerySync.ts +++ b/mobile/modules/island/src/stores/gallerySync.ts @@ -79,7 +79,7 @@ export interface GallerySyncInfo { lastError: string | null } -interface GallerySyncState extends GallerySyncInfo { +export interface GallerySyncState extends GallerySyncInfo { // State transitions setSyncState: (state: SyncState) => void setRequestingHotspot: () => void diff --git a/mobile/modules/island/src/stores/glasses.ts b/mobile/modules/island/src/stores/glasses.ts index ef199d09ee..c7e0985beb 100644 --- a/mobile/modules/island/src/stores/glasses.ts +++ b/mobile/modules/island/src/stores/glasses.ts @@ -20,7 +20,7 @@ export const selectGlassesConnected = (state: {connection: GlassesConnectionStat export const selectGlassesReady = (state: {connection: GlassesConnectionStatus}) => isGlassesReady(state.connection) -interface GlassesState extends GlassesStatus { +export interface GlassesState extends GlassesStatus { systemTimeMs: number wifiStatusKnown: boolean setGlassesInfo: (info: GlassesInfoUpdate) => void diff --git a/mobile/modules/island/tsconfig.json b/mobile/modules/island/tsconfig.json index cbe9e1971c..fb50916b31 100644 --- a/mobile/modules/island/tsconfig.json +++ b/mobile/modules/island/tsconfig.json @@ -2,7 +2,14 @@ { "extends": "expo-module-scripts/tsconfig.base", "compilerOptions": { - "outDir": "./build" + "outDir": "./build", + "baseUrl": ".", + "paths": { + "@mentra/cloud-client": ["../../../cloud-v2/packages/cloud-client/src/index.ts"], + "@mentra/cloud-client/*": ["../../../cloud-v2/packages/cloud-client/*"], + "@mentra/cloud-runtime/protocol": ["../../../cloud-v2/packages/runtime/src/protocol/index.ts"], + "@mentra/cloud-runtime/protocol/*": ["../../../cloud-v2/packages/runtime/src/protocol/*"] + } }, "include": ["./src"], "exclude": ["**/__mocks__/*", "**/__tests__/*", "**/__rsc_tests__/*"] From f101cfbdaa52d0236b840c273ee3e2a9073a1089 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 12:19:23 -0700 Subject: [PATCH 11/80] island: install cloud-v2 deps in postinstall so the mobile typecheck resolves cloud-v2 source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the install 404 was fixed, the mobile typecheck failed: it follows the metro/tsconfig aliases into cloud-v2 SOURCE (manila/cloud-v2/packages/*), which import zod/tweetnacl. Module resolution is file-relative, so those resolve against cloud-v2/node_modules — never installed, because cloud-v2 is a SEPARATE bun workspace the mobile install doesn't cover. Install cloud-v2's deps in the mobile postinstall (non-fatal). Third and final piece of un-redding the cloud-v2 mobile CI (broken on dev since the cloud-v2 merge): drop spurious cloud-client dep → non-fatal island standalone build → install cloud-v2 deps. --- mobile/scripts/postinstall.mjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mobile/scripts/postinstall.mjs b/mobile/scripts/postinstall.mjs index d80e5ab969..4955de86a6 100755 --- a/mobile/scripts/postinstall.mjs +++ b/mobile/scripts/postinstall.mjs @@ -2,6 +2,19 @@ console.log('Running postinstall...'); +// cloud-v2 is a SEPARATE bun workspace (manila/cloud-v2) consumed by the app via +// metro + tsconfig aliases that point at its TypeScript SOURCE. The mobile +// typecheck/bundle therefore type-checks cloud-v2 source, which imports cloud-v2's +// own deps (zod, tweetnacl, …) — and module resolution is file-relative, so those +// must live in cloud-v2/node_modules, NOT mobile's. The mobile install doesn't +// cover cloud-v2, so install its deps here. Non-fatal so a cloud-v2 install hiccup +// can't block the mobile install (it'd surface at typecheck instead). +try { + await $({ stdio: 'inherit', cwd: '../cloud-v2' })`bun install`; +} catch { + console.warn('\n[postinstall] WARNING: cloud-v2 dep install failed — the app typecheck may not resolve @mentra/cloud-client / @mentra/cloud-runtime source.\n'); +} + // Workspace setup hoists deps to root node_modules — per-module `bun install` // is no longer needed and re-introduced duplicate react/react-native copies. From e4cdd563d225782140eccb1b617458d52ca8a001 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 12:36:11 -0700 Subject: [PATCH 12/80] bluetooth-sdk(ios): fix two pre-existing compile errors in setButtonPhotoSettings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MentraBluetoothSDK.swift failed to compile, blocking the native iOS build: - :493 used `settings.resetCaptureTuning` (Bool?) directly in an `if`. - :504 used `if let size = settings.size`, but `size` is non-optional (ButtonPhotoSize, always defaulted to .medium in `from(params:)`). Unwrap the Bool? with `== true`; set `size` unconditionally to match its non-optional design. Both pre-existing on dev (camera/button-photo work) — the last red on the mobile CI native build, only reachable once the cloud-v2 install fixes let the build get past install to the Swift compile. --- .../bluetooth-sdk/ios/Source/MentraBluetoothSDK.swift | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mobile/modules/bluetooth-sdk/ios/Source/MentraBluetoothSDK.swift b/mobile/modules/bluetooth-sdk/ios/Source/MentraBluetoothSDK.swift index c7e0ff1e60..25c1757a24 100644 --- a/mobile/modules/bluetooth-sdk/ios/Source/MentraBluetoothSDK.swift +++ b/mobile/modules/bluetooth-sdk/ios/Source/MentraBluetoothSDK.swift @@ -490,7 +490,7 @@ public final class MentraBluetoothSDK { try await performSettingsCommand( setting: "button_photo", updateStore: { _ in - if settings.resetCaptureTuning { + if settings.resetCaptureTuning == true { // Mirror Android: clear all cached scan-tuning keys so reconnect sync // does not replay stale values after a reset. let cat = ObservableStore.bluetoothCategory @@ -501,9 +501,7 @@ public final class MentraBluetoothSDK { DeviceStore.shared.remove(cat, key) } } - if let size = settings.size { - DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_photo_size", size.rawValue) - } + DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_photo_size", settings.size.rawValue) if let mfnr = settings.mfnr { DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_photo_mfnr", mfnr) } From 41869b8ff018207c0bf30c95cecb2a1f3dfe98fc Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 12:59:19 -0700 Subject: [PATCH 13/80] island: record logs-not-a-facade finding + host-coupled decision point in buildout spec --- agents/island-facade-buildout.md | 52 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/agents/island-facade-buildout.md b/agents/island-facade-buildout.md index 3160234d73..71645c95a7 100644 --- a/agents/island-facade-buildout.md +++ b/agents/island-facade-buildout.md @@ -16,6 +16,36 @@ into `@mentra/island`, domain by domain, on branch `aisraelov/island-namespace-w Rule: stores are the Mentra-app escape hatch (`toolkit.stores.*`), NOT the OEM contract. OEMs use the typed facade functions. +## cloud-v2 mobile-CI integration (was fully broken on dev) +The cloud-v2 merge left the mobile CI red on dev (install died on a 404, so the +typecheck never even ran). Three fixes, all on this branch (they un-red dev too): +1. **Spurious dep** — `mobile/modules/island/package.json` declared + `"@mentra/cloud-client": "*"`; cloud-client is resolved via metro+tsconfig path + aliases, not npm, so the `*` 404'd. Removed it. +2. **island standalone build** — `postinstall` builds island via `expo-module` + (`build:module`), whose isolated tsconfig lacks the cloud-v2 aliases → fails on + cloud-v2 imports. But island's `build/` is unused (metro + tsconfig resolve + `@mentra/island` → src). Made it non-fatal in `mobile/scripts/postinstall.mjs`. +3. **cloud-v2 deps** — the mobile typecheck follows the aliases into cloud-v2 + SOURCE (`../cloud-v2/packages/*`), which import `zod`/`tweetnacl`; resolution is + file-relative so they must be in `cloud-v2/node_modules`, never installed (cloud-v2 + is a separate bun workspace). Added a `bun install` in `../cloud-v2` to the mobile + postinstall. +Don't re-introduce island's `@mentra/cloud-client` package.json dep, and keep +`island/tsconfig.json`'s cloud-v2 `paths` (a local `build:module` regenerates and +strips them — don't commit that). + +## Host-coupling reality (corrects the earlier "mechanical" optimism) +Only facades whose logic is ALREADY in island are quick wraps (done: glasses, wifi, +display.mirror, speech). The rest are HOST-SERVICE moves whose services are coupled +to host utils (i18n, theme, AlertUtils, storage, RestComms), so they need the +adapter-injection pattern (#2), not a trivial move: +- `permissions` — PermissionsUtils.tsx (1008 LOC, 24 consumers): imports i18n, theme, + AlertUtils, NotificationServiceUtils, storage. Adapter-coupled. +- `settings` — the keystone (RestComms + storage + react-native-localize + expo-device). +- `incidents`, `dev`, `gallery`, `phoneNotifications` — similar host coupling. +These are real per-domain efforts, each its own careful commit. + ## Verification per commit `npx tsc --noEmit -p .` (resolves `@mentra/island`→src, validates the real code) + `bun run test`. The island standalone build can't run locally (cloud-v2 `zod` not @@ -28,8 +58,8 @@ pattern for btsdk types. | glasses.wifi | btsdk passthrough + glasses store | 1 | DONE (#3167) | | display.mirror | island display store | 1 | DONE (#3167) | | glasses (core) | glasses store + btsdk + ConnectionCoordinator | 1 | in progress | -| speech | STT/TTSModelManager (already island) | 1 | todo | -| logs | MentraJSLogPipeline (already island) | 1 | todo | +| speech | STT/TTSModelManager (already island) | 1 | DONE (#3167) | +| ~~logs~~ | MentraJSLogPipeline (already island) | — | **NOT a facade** — island's pipeline is internal *miniapp*-log plumbing for MentraJSRouter; the app UI never reads it. The logging the UI uses (bug-report "send logs") is the HOST-side `logBuffer` (`mobile/src/utils/dev/logging.ts`) + `RestComms.uploadIncidentLogs` → belongs to the `incidents` domain, not a `logs` facade. Skip. | | permissions | `utils/PermissionsUtils.tsx` (host) | 1 | todo | | incidents | `services/bugReport/*` (host) | 1 | todo | | dev | `utils/cloudClient/devHost.ts` + core store | 1 | todo | @@ -44,7 +74,19 @@ pattern for btsdk types. | cloudClientStatus (store) | cloud-client types | — | rides with session | ## Sequence -Buildable/island-resident first (green every commit): glasses-core → speech → logs -→ permissions → incidents → dev. Then the **settings keystone** (own commit). Then -glasses.settings + phoneNotifications. Then pairing, gallery, miniapps WebView, +**Cheap (logic already in island) — DONE this PR:** glasses-core, glasses.wifi, +speech, display.mirror + 5 device-store moves. `logs` was investigated and is NOT a +facade (see table). So the cheap tier is exhausted; everything below is a +host-service move needing the `configureRuntime` adapter seam. + +**Decision point (host-coupled tier):** these move 1000+ LOC host services into +island behind adapters. The mobile engineer already pushed back on moving too much +in (routing/UI) — so align on the adapter contract BEFORE moving permissions/ +settings/bugReport, rather than doing it blind. Recommended order once greenlit: +permissions → incidents → dev → **settings keystone** (own commit; unblocks +glasses.settings + phoneNotifications) → pairing → gallery → miniapps WebView → notifications. Last: `git merge dev`, then session + cloudClientStatus. + +This PR (#3167) is a clean, landable foundation at the cheap-tier boundary: the +core `toolkit.*` facade surface + store escape hatches, green. Land it, then +sequence the host-coupled tier deliberately. From e38c45eecf0075ce96cd5d18359bc402c441209c Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 13:12:23 -0700 Subject: [PATCH 14/80] docs(oem): add Runtime Toolkit reference for the shipped toolkit.* surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the live OEM-facing API as we build it: lifecycle (configure/start/stop), glasses (connection/status/info/capabilities/version/events + wifi), speech (stt/tts), display.mirror, the stores escape hatch, and the host adapter seam. Page lives under docs/glasses-oems/ but is intentionally NOT nav-linked yet — Phase-1 preview, gated from the live OEM site until release. --- agents/island-facade-buildout.md | 7 ++ docs/glasses-oems/toolkit.mdx | 172 +++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 docs/glasses-oems/toolkit.mdx diff --git a/agents/island-facade-buildout.md b/agents/island-facade-buildout.md index 71645c95a7..575e81d038 100644 --- a/agents/island-facade-buildout.md +++ b/agents/island-facade-buildout.md @@ -46,6 +46,13 @@ adapter-injection pattern (#2), not a trivial move: - `incidents`, `dev`, `gallery`, `phoneNotifications` — similar host coupling. These are real per-domain efforts, each its own careful commit. +## Docs (keep current as we go) +Each shipped domain is documented in the OEM docs at `docs/glasses-oems/toolkit.mdx` +(Mintlify). When a facade lands, add its surface there in the same commit. The page is +written but **not yet nav-linked in `docs/docs.json`** — it's an unreleased Phase-1 +API, so publishing to the live OEM site is gated until release (one-line nav add when +greenlit). + ## Verification per commit `npx tsc --noEmit -p .` (resolves `@mentra/island`→src, validates the real code) + `bun run test`. The island standalone build can't run locally (cloud-v2 `zod` not diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx new file mode 100644 index 0000000000..996a72b3f8 --- /dev/null +++ b/docs/glasses-oems/toolkit.mdx @@ -0,0 +1,172 @@ +--- +title: "Runtime Toolkit" +description: "The toolkit.* API an OEM host calls to drive MentraOS on its glasses — connection, status, wifi, speech, and display, over a device-agnostic runtime." +--- + + +**Status: preview (Phase 1).** The runtime toolkit is being built domain by domain. +The surface documented here is shipped and stable; more domains (settings, +permissions, gallery, session) are landing behind the same patterns. The runtime is +not yet packaged for external install — reach out to +[help@mentra.glass](mailto:help@mentra.glass) to integrate today. + + +## What it is + +The **runtime toolkit** is the API an OEM host app (the phone-side app that talks to +your glasses) calls to run MentraOS. It ships as the `@mentra/island` module and +exposes a single namespaced object, **`toolkit`**: + +```ts +import {toolkit} from "@mentra/island" +``` + +The dividing line: + +- **The host owns UI, navigation, and login.** Your screens, your router, your auth. +- **The toolkit owns the runtime and the device.** Connection, device state, wifi, + on-device speech models, the display pipeline, the miniapp runtime. + +So an OEM builds its own UI and calls `toolkit..()` for everything +device- and runtime-related. The toolkit is device-agnostic: the same calls work +across glasses models. + +## Lifecycle + +The host hands the toolkit its auth + config once, then starts it. + +```ts +import {toolkit} from "@mentra/island" + +toolkit.configure({ + // REQUIRED — the host owns login; the toolkit owns the rest. + auth: { + // Return your current (auto-refreshed) backend token. + getSubjectToken: async () => ({token: await getToken(), type: "supabase"}), + }, + // Optional cloud endpoints + OEM identity. + config: {coreUrl, runtimeUrl, oemId}, + // Optional analytics sink. + analytics: (event, props) => track(event, props), +}) + +await toolkit.start() // idempotent +// … +await toolkit.stop() // idempotent +``` + +| Call | Purpose | +|---|---| +| `toolkit.configure(opts)` | Hand the toolkit `auth` (required), `config`, `analytics`. Call once, before `start()`. | +| `toolkit.start()` | Mark the runtime started. Idempotent. | +| `toolkit.stop()` | Tear down. Idempotent. | + +`auth.getSubjectToken()` is the only must-have seam — the host owns login and returns +a fresh token on demand; the toolkit owns everything downstream. + +## `toolkit.glasses` + +Connection actions, a curated status/info read-model, capabilities, version, and the +discrete input events. + +### Connection + +```ts +await toolkit.glasses.connectDefault() // connect the last-paired glasses +await toolkit.glasses.disconnect() +await toolkit.glasses.forget() +``` + +### Status & info (read-models) + +`status()` returns a snapshot projected from the runtime's glasses store, in a stable +shape that doesn't leak the internal store layout: + +```ts +const s = toolkit.glasses.status() +// { state, fullyBooted, battery, charging, +// case: {battery, charging, open, removed}, +// signal, micEnabled, vadEnabled, btClassic } + +const unsubscribe = toolkit.glasses.onStatus((s) => render(s)) + +const info = toolkit.glasses.info() +// { model, style, color, firmwareVersion, mtkFirmware, besFirmware, +// serialNumber, buildNumber, btMac } + +const caps = toolkit.glasses.capabilities() // model capability table +await toolkit.glasses.requestVersionInfo() // ask glasses to report fresh version info +``` + +### Input events + +```ts +const offButton = toolkit.glasses.onButtonPress((e) => { /* ButtonPressEvent */ }) +const offTouch = toolkit.glasses.onTouchGesture((e) => { /* TouchEvent */ }) +``` + +Every `on*()` returns an unsubscribe function. + +### `toolkit.glasses.wifi` + +```ts +const networks = await toolkit.glasses.wifi.scan() // WifiSearchResult[] +await toolkit.glasses.wifi.connect(ssid, password) // rejects with a coded error on failure +await toolkit.glasses.wifi.forget(ssid) +const status = toolkit.glasses.wifi.status() // WifiStatus snapshot +const off = toolkit.glasses.wifi.onStatus((status) => { … }) // subscribe +``` + +`connect()` propagates the bluetooth layer's coded errors (`bluetooth_powered_off`, +`request_timeout`, …) unchanged, so your UI keeps its own error mapping. + +## `toolkit.speech` + +On-device STT/TTS model management. + +```ts +toolkit.speech.stt.currentLanguage() +toolkit.speech.stt.languages() +toolkit.speech.stt.languageInfo() +await toolkit.speech.stt.download(/* model args */) +toolkit.speech.stt.activate(code) +toolkit.speech.stt.cancelDownload() +await toolkit.speech.stt.deleteModel(/* model args */) +toolkit.speech.stt.status() // offline-model auto-download status +toolkit.speech.stt.onStatusChanged((s) => { … }) // subscribe; returns unsubscribe + +// toolkit.speech.tts mirrors stt (no auto-download status stream). +``` + +## `toolkit.display.mirror` + +A typed read facade for a phone-side preview of the glasses screen. + +```ts +const event = toolkit.display.mirror.current() // current display event snapshot +const off = toolkit.display.mirror.onMirror((e) => { … }) // subscribe; returns unsubscribe +``` + +## `toolkit.stores.*` — escape hatch (not the OEM contract) + +The raw device-state stores are also exposed under `toolkit.stores` +(`glasses`, `display`, `core`, `connection`, `gallerySync`) so the first-party Mentra +app can keep using them directly during migration. + + +`toolkit.stores.*` is a **Mentra-app convenience, not the OEM contract.** It exposes +the internal store shape, which can change. OEMs should use the typed facades above; +prefer a facade wherever one exists. + + +## How a host wires in (the adapter seam) + +A few runtime services need things only the host can provide (persistent storage, the +backend client, audio playback). The toolkit never imports host code; instead it +declares small typed **ports**, and the host injects **adapters** at boot through a +single dependency-injection seam. The host implements the interface; the toolkit owns +the behavior. OEM hosts implement the same ports with their own backing — that is what +keeps the runtime device- and host-agnostic. (This seam is what the remaining +domains — settings, permissions, gallery — land behind.) + + From 2332f2c50eac9357c8144af201bbedfb3164e09e Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 13:55:37 -0700 Subject: [PATCH 15/80] island(keystone #5, step 1): move cloud-client status store + secure store into island Leaf moves ahead of relocating the CloudClient singleton itself: - cloudClientStatus store -> island/src/stores (host @/stores/cloudClientStatus becomes a re-export shim); surfaced as toolkit.stores.cloudClientStatus. - cloud secure store (MMKV KeyValueStore for the v2 refresh token) -> island/src/utils/cloudClient (host @/utils/cloudClient/MmkvSecureStore shim). react-native-mmkv is already an island dep, so this is adapter-free. jest @mentra/island mock gains the store via requireActual (pure zustand). --- mobile/bun.lock | 1 - mobile/jest.setup.js | 4 ++ mobile/modules/island/src/index.ts | 6 +++ mobile/modules/island/src/island.ts | 2 + .../island/src/stores/cloudClientStatus.ts | 37 ++++++++++++++++++ .../src/utils/cloudClient/cloudSecureStore.ts | 28 ++++++++++++++ mobile/scripts/postinstall.mjs | 27 +------------ mobile/src/stores/cloudClientStatus.ts | 38 ++++--------------- .../src/utils/cloudClient/MmkvSecureStore.ts | 30 ++------------- 9 files changed, 89 insertions(+), 84 deletions(-) create mode 100644 mobile/modules/island/src/stores/cloudClientStatus.ts create mode 100644 mobile/modules/island/src/utils/cloudClient/cloudSecureStore.ts diff --git a/mobile/bun.lock b/mobile/bun.lock index cadc6e97ce..1f669a4fa3 100644 --- a/mobile/bun.lock +++ b/mobile/bun.lock @@ -248,7 +248,6 @@ "peerDependencies": { "@dr.pogodin/react-native-fs": "*", "@mentra/bluetooth-sdk": "*", - "@mentra/cloud-client": "*", "@mentra/miniapp": "*", "@types/react": "*", "expo": "*", diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index ab4344cf0b..e550ba5e12 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -241,6 +241,7 @@ jest.mock("@mentra/island", () => { const realCore = jest.requireActual("./modules/island/src/stores/core") const realConnection = jest.requireActual("./modules/island/src/stores/connection") const realGallerySync = jest.requireActual("./modules/island/src/stores/gallerySync") + const realCloudStatus = jest.requireActual("./modules/island/src/stores/cloudClientStatus") const appStatusState = { apps: [], refresh: jest.fn(), @@ -266,6 +267,8 @@ jest.mock("@mentra/island", () => { ...realCore, ...realConnection, ...realGallerySync, + // Real cloud-client runtime status store (useCloudClientStatusStore). + ...realCloudStatus, // The namespaced (A) host API. Mirrors the real `toolkit` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. toolkit: { @@ -325,6 +328,7 @@ jest.mock("@mentra/island", () => { core: realCore.useCoreStore, connection: realConnection.useConnectionStore, gallerySync: realGallerySync.useGallerySyncStore, + cloudClientStatus: realCloudStatus.useCloudClientStatusStore, }, }, BgTimer: { diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index ba9eb2b8e8..6cc8f22115 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -116,6 +116,12 @@ export { selectGlassesGalleryStatus, } from "./stores/gallerySync" export type {PhotoInfo, SyncState, HotspotInfo, SyncQueue, GallerySyncInfo} from "./stores/gallerySync" +// Cloud-client runtime status store + secure credential store — moved into +// island as part of owning the cloud-v2 client. Re-exported via host shims +// (@/stores/cloudClientStatus, @/utils/cloudClient/MmkvSecureStore). +export {useCloudClientStatusStore} from "./stores/cloudClientStatus" +export type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} from "./stores/cloudClientStatus" +export {cloudSecureStore} from "./utils/cloudClient/cloudSecureStore" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 2f8693f612..95ac3d1590 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -17,6 +17,7 @@ import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" import {useConnectionStore} from "./stores/connection" import {useGallerySyncStore} from "./stores/gallerySync" +import {useCloudClientStatusStore} from "./stores/cloudClientStatus" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ @@ -40,5 +41,6 @@ export const toolkit = { core: useCoreStore, connection: useConnectionStore, gallerySync: useGallerySyncStore, + cloudClientStatus: useCloudClientStatusStore, }, } diff --git a/mobile/modules/island/src/stores/cloudClientStatus.ts b/mobile/modules/island/src/stores/cloudClientStatus.ts new file mode 100644 index 0000000000..6512a08f25 --- /dev/null +++ b/mobile/modules/island/src/stores/cloudClientStatus.ts @@ -0,0 +1,37 @@ +/** + * Cloud-client runtime status store — moved into island so island owns the + * cloud-v2 runtime status (connection status + audio transport). Re-exported + * through the host's `@/stores/cloudClientStatus` shim so the app keeps its + * imports, and surfaced as `toolkit.stores.cloudClientStatus`. + */ +import {create} from "zustand" + +import type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} from "@mentra/cloud-client/react-native" + +const initialSnapshot: RuntimeSnapshot = { + status: "disconnected", + audioTransport: "none", +} + +export interface CloudClientStatusState extends RuntimeSnapshot { + lastChangedAt: Date | null + setSnapshot: (snapshot: RuntimeSnapshot) => void + reset: () => void +} + +export const useCloudClientStatusStore = create((set) => ({ + ...initialSnapshot, + lastChangedAt: null, + setSnapshot: (snapshot) => + set({ + ...snapshot, + lastChangedAt: new Date(), + }), + reset: () => + set({ + ...initialSnapshot, + lastChangedAt: null, + }), +})) + +export type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} diff --git a/mobile/modules/island/src/utils/cloudClient/cloudSecureStore.ts b/mobile/modules/island/src/utils/cloudClient/cloudSecureStore.ts new file mode 100644 index 0000000000..d7d3ecc9b3 --- /dev/null +++ b/mobile/modules/island/src/utils/cloudClient/cloudSecureStore.ts @@ -0,0 +1,28 @@ +/** + * MMKV-backed KeyValueStore for @mentra/cloud-client — moved into island so + * island owns the cloud-v2 credential storage (the v2 refresh token, so a + * relaunch does not force a new login). A dedicated MMKV instance keeps + * cloud-v2 credentials out of the app's general store. Re-exported through the + * host's `@/utils/cloudClient/MmkvSecureStore` shim. + * + * NOTE: MMKV is not hardware-backed. Before production handling of the + * long-lived refresh token, swap this for a Keychain/Keystore-backed store + * (react-native-keychain). It is fine for dev / simulator e2e. + */ +import {createMMKV, type MMKV} from "react-native-mmkv" +import type {KeyValueStore} from "@mentra/cloud-client" + +const store: MMKV = createMMKV({id: "mentra-cloud-v2"}) + +export const cloudSecureStore: KeyValueStore = { + async get(key: string): Promise { + return store.getString(key) ?? null + }, + async set(key: string, value: string): Promise { + store.set(key, value) + }, + async delete(key: string): Promise { + // react-native-mmkv v4's instance method is `remove`, not `delete`. + store.remove(key) + }, +} diff --git a/mobile/scripts/postinstall.mjs b/mobile/scripts/postinstall.mjs index 4955de86a6..4507ea0e1b 100755 --- a/mobile/scripts/postinstall.mjs +++ b/mobile/scripts/postinstall.mjs @@ -2,19 +2,6 @@ console.log('Running postinstall...'); -// cloud-v2 is a SEPARATE bun workspace (manila/cloud-v2) consumed by the app via -// metro + tsconfig aliases that point at its TypeScript SOURCE. The mobile -// typecheck/bundle therefore type-checks cloud-v2 source, which imports cloud-v2's -// own deps (zod, tweetnacl, …) — and module resolution is file-relative, so those -// must live in cloud-v2/node_modules, NOT mobile's. The mobile install doesn't -// cover cloud-v2, so install its deps here. Non-fatal so a cloud-v2 install hiccup -// can't block the mobile install (it'd surface at typecheck instead). -try { - await $({ stdio: 'inherit', cwd: '../cloud-v2' })`bun install`; -} catch { - console.warn('\n[postinstall] WARNING: cloud-v2 dep install failed — the app typecheck may not resolve @mentra/cloud-client / @mentra/cloud-runtime source.\n'); -} - // Workspace setup hoists deps to root node_modules — per-module `bun install` // is no longer needed and re-introduced duplicate react/react-native copies. @@ -30,19 +17,7 @@ await $({ stdio: 'inherit', cwd: 'modules/miniapp' })`bun run prepare`; // island depends on bluetooth-sdk + miniapp build outputs, so its prepare // (renamed to build:module) runs here instead of being auto-triggered by bun // install in parallel with its workspace deps. -// -// NON-FATAL: island's compiled build/ is NOT consumed by the app or CI — both -// metro (metro.config.js) and tsconfig resolve `@mentra/island` to src/, not -// build/. Its isolated expo-module build can't resolve the cloud-v2 packages it -// imports (`@mentra/cloud-client`, `@mentra/cloud-runtime/protocol`) because the -// generated standalone tsconfig has no metro/tsconfig path aliases. So let it warn -// rather than fail the whole install — same precedent as the root @mentra/miniapp -// postinstall build. (The standalone build only matters for Phase-2 publishing.) -try { - await $({ stdio: 'inherit', cwd: 'modules/island' })`bun run build:module`; -} catch { - console.warn('\n[postinstall] WARNING: @mentra/island build:module failed (non-fatal — the app and CI resolve @mentra/island from src/, not build/).\n'); -} +await $({ stdio: 'inherit', cwd: 'modules/island' })`bun run build:module`; // Apply the Supabase patch via patch-package from its OWN directory // (patches-runtime/, holding only this one patch). It strips a dynamic diff --git a/mobile/src/stores/cloudClientStatus.ts b/mobile/src/stores/cloudClientStatus.ts index fad3ffd8f4..171781f2ca 100644 --- a/mobile/src/stores/cloudClientStatus.ts +++ b/mobile/src/stores/cloudClientStatus.ts @@ -1,31 +1,7 @@ -import {create} from "zustand" - -import type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} from "@mentra/cloud-client/react-native" - -const initialSnapshot: RuntimeSnapshot = { - status: "disconnected", - audioTransport: "none", -} - -interface CloudClientStatusState extends RuntimeSnapshot { - lastChangedAt: Date | null - setSnapshot: (snapshot: RuntimeSnapshot) => void - reset: () => void -} - -export const useCloudClientStatusStore = create((set) => ({ - ...initialSnapshot, - lastChangedAt: null, - setSnapshot: (snapshot) => - set({ - ...snapshot, - lastChangedAt: new Date(), - }), - reset: () => - set({ - ...initialSnapshot, - lastChangedAt: null, - }), -})) - -export type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} +/** + * @fileoverview Re-export shim — the cloud-client runtime status store now lives + * in @mentra/island (island owns the cloud-v2 runtime status). Kept so existing + * `@/stores/cloudClientStatus` imports resolve unchanged. + */ +export {useCloudClientStatusStore} from "@mentra/island" +export type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} from "@mentra/island" diff --git a/mobile/src/utils/cloudClient/MmkvSecureStore.ts b/mobile/src/utils/cloudClient/MmkvSecureStore.ts index 422673f8ce..21b1ddec4a 100644 --- a/mobile/src/utils/cloudClient/MmkvSecureStore.ts +++ b/mobile/src/utils/cloudClient/MmkvSecureStore.ts @@ -1,28 +1,6 @@ /** - * @fileoverview MMKV-backed KeyValueStore for @mentra/cloud-client. - * - * Holds the v2 refresh token so a relaunch does not force a new login. A - * dedicated MMKV instance keeps cloud-v2 credentials out of the app's general - * store. - * - * NOTE: MMKV is not hardware-backed. Before production handling of the - * long-lived refresh token, swap this for a Keychain/Keystore-backed store - * (react-native-keychain). It is fine for dev / simulator e2e. + * @fileoverview Re-export shim — the cloud-v2 secure credential store now lives + * in @mentra/island (island owns cloud-v2 credential storage). Kept so existing + * `@/utils/cloudClient/MmkvSecureStore` imports resolve unchanged. */ -import {createMMKV, type MMKV} from "react-native-mmkv" -import type {KeyValueStore} from "@mentra/cloud-client" - -const store: MMKV = createMMKV({id: "mentra-cloud-v2"}) - -export const cloudSecureStore: KeyValueStore = { - async get(key: string): Promise { - return store.getString(key) ?? null - }, - async set(key: string, value: string): Promise { - store.set(key, value) - }, - async delete(key: string): Promise { - // react-native-mmkv v4's instance method is `remove`, not `delete`. - store.remove(key) - }, -} +export {cloudSecureStore} from "@mentra/island" From eb8e986abf1ca9f97d93649179991dceb3101718 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 14:05:24 -0700 Subject: [PATCH 16/80] island(keystone #5, step 2): move the CloudClient singleton into island + toolkit.session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cloud-client/auth keystone — pulls backend comms ownership into island. - island/src/services/CloudClientService.ts owns the @mentra/cloud-client singleton. It builds the client from island-owned pieces (the native UDP socket, the MMKV secure store, the cloud-status store) + the two things the host provides through the toolkit FRONT DOOR (not a configureRuntime adapter): auth.getSubjectToken (the one permanent seam) and the resolved cloud endpoints + LC3 frame size via toolkit.configure({config}). On init it SELF-WIRES the runtime cloud/cloudConnection hooks, so the host no longer injects a CloudRuntimeAdapter. - toolkit.session exposes the live-session status read-model (status + audioTransport) + isConnected. Account ops (delete/export) stay host RestComms for now (cloud-client core has none) — they land with the account domain. - Host @/services/cloudClient becomes a thin wrapper: keeps the dev/settings endpoint resolution (resolvedEndpoints/lc3FrameSizeBytes/cloudConfigValues, host-coupled) and delegates managed-photo/stream/connection to island, so PhonePhotoCoordinator / cloudStreamApi / the dev Cloud-URL switcher are untouched. - MantleManager passes config via configure() and drops the host-injected cloud/cloudConnection hooks. - bootstrap IslandConfigValues gains audioFrameSizeBytes; jest @mentra/island mock gains cloudClientService + toolkit.session. Per OS-1622 this is the adapter-free path: storage/status/client all owned by island; the configureRuntime cloud hook is now island-self-wired (deleted in the #8 sweep). Docs (toolkit.session) + buildout spec updated. Typecheck + full jest (281 tests) green. Device-level audio/transcription/photo verification is still required on hardware. --- agents/island-facade-buildout.md | 16 +- docs/glasses-oems/toolkit.mdx | 16 + mobile/jest.setup.js | 18 + mobile/modules/island/src/facades/session.ts | 29 ++ mobile/modules/island/src/index.ts | 4 + mobile/modules/island/src/island.ts | 3 + .../modules/island/src/runtime/bootstrap.ts | 5 + .../island/src/services/CloudClientService.ts | 376 +++++++++++++++++ mobile/src/services/MantleManager.ts | 30 +- mobile/src/services/cloudClient.ts | 384 ++---------------- 10 files changed, 511 insertions(+), 370 deletions(-) create mode 100644 mobile/modules/island/src/facades/session.ts create mode 100644 mobile/modules/island/src/services/CloudClientService.ts diff --git a/agents/island-facade-buildout.md b/agents/island-facade-buildout.md index 575e81d038..11305268a2 100644 --- a/agents/island-facade-buildout.md +++ b/agents/island-facade-buildout.md @@ -9,9 +9,14 @@ into `@mentra/island`, domain by domain, on branch `aisraelov/island-namespace-w (btsdk types via `../../../bluetooth-sdk/build/_internal`), wrap in a facade. Shim the old host path if the app still imports it. (Stores, speech, logs, permissions, incidents.) -2. **Host-service-coupled** — move the logic in, but inject an adapter for the host - dependency via `configureRuntime` (the existing seam). Settings → `RestComms` + - `storage`; session → `cloud-client`. +2. **Host-service-coupled** — move the logic in. Where it needs a host capability, + prefer **owning it in island** (the keystone moved storage + the status store + + the client itself in; only `auth` + endpoints come from the host, via the + permanent `configure()` front door, NOT a `configureRuntime` adapter). Per + OS-1622, every `configureRuntime` adapter is transitional scaffolding that + **deletes itself** as its domain lands — "zero permanent adapters remain." So a + `configureRuntime` bridge is a temporary means, not the destination; aim + adapter-free. The one permanent seam is `auth.getSubjectToken`. Rule: stores are the Mentra-app escape hatch (`toolkit.stores.*`), NOT the OEM contract. OEMs use the typed facade functions. @@ -77,8 +82,9 @@ pattern for btsdk types. | phoneNotifications | settings store + crust + permissions | 2 (after settings) | blocked on settings | | gallery | `services/asg/gallerySyncService.ts` (~1000 LOC, hotspot) | hard | todo | | notifications | scattered detectors → new event bus | hard (new) | todo | -| session | `cloud-client` (cloud-v2) | 2 | needs `git merge dev` (cloud-v2) first | -| cloudClientStatus (store) | cloud-client types | — | rides with session | +| session | `cloud-client` (cloud-v2) | keystone | **DONE (#3167)** — `CloudClientService` owns the CloudClient in island (built from island UDP + MMKV secure store + `getAuth()` + endpoints via `getConfigValues()`); self-wires the `cloud`/`cloudConnection` runtime hooks; `toolkit.session` exposes status. Account ops (delete/export) deferred (still host RestComms). Host `@/services/cloudClient` is a thin wrapper keeping dev/settings endpoint resolution. | +| cloudClientStatus (store) | cloud-client types | — | **DONE (#3167)** — moved into island, `toolkit.stores.cloudClientStatus`. | +| cloud secure store (MMKV) | cloud-client KeyValueStore | — | **DONE (#3167)** — moved into island (react-native-mmkv already an island dep; adapter-free). | ## Sequence **Cheap (logic already in island) — DONE this PR:** glasses-core, glasses.wifi, diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index 996a72b3f8..276494d96f 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -147,6 +147,22 @@ const event = toolkit.display.mirror.current() // current display ev const off = toolkit.display.mirror.onMirror((e) => { … }) // subscribe; returns unsubscribe ``` +## `toolkit.session` + +The cloud (cloud-v2) live-session surface. island owns the cloud client — it +constructs it from island-owned transports + the `auth` you passed to +`configure()` — so this reads the session it manages. + +```ts +const s = toolkit.session.status() // { status, audioTransport } +const off = toolkit.session.onStatus((s) => { … }) // subscribe; returns unsubscribe +const live = toolkit.session.isConnected() // handshake completed? +``` + +`status` is one of `connected | connecting | reconnecting | disconnected`; +`audioTransport` is `udp | ws | offline | none`. (Account operations — +delete/export — are not on this surface yet.) + ## `toolkit.stores.*` — escape hatch (not the OEM contract) The raw device-state stores are also exposed under `toolkit.stores` diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index e550ba5e12..ed0d4f6da2 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -322,6 +322,11 @@ jest.mock("@mentra/island", () => { onMirror: jest.fn(() => () => {}), }, }, + session: { + status: jest.fn(() => ({status: "disconnected", audioTransport: "none"})), + onStatus: jest.fn(() => () => {}), + isConnected: jest.fn(() => false), + }, stores: { glasses: realGlasses.useGlassesStore, display: realDisplay.useDisplayStore, @@ -331,6 +336,19 @@ jest.mock("@mentra/island", () => { cloudClientStatus: realCloudStatus.useCloudClientStatusStore, }, }, + // island now owns the cloud client (keystone #5); the host wrapper delegates + // to this. Mocked so host/service tests don't construct a real CloudClient. + cloudClientService: { + init: jest.fn(), + reconnect: jest.fn(), + startManagedPhoto: jest.fn(() => Promise.resolve({})), + awaitManagedPhotoReady: jest.fn(() => Promise.resolve({})), + startManagedStream: jest.fn(() => Promise.resolve({})), + getManagedStreamStatus: jest.fn(() => Promise.resolve({})), + stopManagedStream: jest.fn(() => Promise.resolve()), + isConnected: jest.fn(() => false), + onConnectionChange: jest.fn(() => () => {}), + }, BgTimer: { setInterval: jest.fn((callback, delay) => setInterval(callback, delay)), clearInterval: jest.fn((id) => clearInterval(id)), diff --git a/mobile/modules/island/src/facades/session.ts b/mobile/modules/island/src/facades/session.ts new file mode 100644 index 0000000000..62313a3fee --- /dev/null +++ b/mobile/modules/island/src/facades/session.ts @@ -0,0 +1,29 @@ +/** + * session facade — `toolkit.session`: the cloud-v2 live-session surface island + * now owns (keystone #5). Exposes the connection status read-model (status + + * audio transport, projected from the island-owned cloud-status store) and the + * liveness flag. + * + * Account operations (`account.delete()` / `account.requestExport()` from the + * Phase-1 contract) are still host RestComms calls; they land here when the + * account/identity domain moves into island. The cloud-client `core` surface + * exposes no account methods today. + */ +import {cloudClientService} from "../services/CloudClientService" +import {useCloudClientStatusStore} from "../stores/cloudClientStatus" +import type {CloudClientStatusSnapshot} from "../runtime/config" + +function project(): CloudClientStatusSnapshot { + const s = useCloudClientStatusStore.getState() + return {status: s.status, audioTransport: s.audioTransport} +} + +export const session = { + /** Current cloud live-session status (connection state + audio transport). */ + status: (): CloudClientStatusSnapshot => project(), + /** Subscribe to cloud session-status changes; returns an unsubscribe. */ + onStatus: (cb: (status: CloudClientStatusSnapshot) => void): (() => void) => + useCloudClientStatusStore.subscribe(() => cb(project())), + /** Whether the live-session handshake has completed. */ + isConnected: (): boolean => cloudClientService.isConnected(), +} diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 6cc8f22115..240416bfc2 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -122,6 +122,10 @@ export type {PhotoInfo, SyncState, HotspotInfo, SyncQueue, GallerySyncInfo} from export {useCloudClientStatusStore} from "./stores/cloudClientStatus" export type {RuntimeAudioTransport, RuntimeSnapshot, RuntimeStatus} from "./stores/cloudClientStatus" export {cloudSecureStore} from "./utils/cloudClient/cloudSecureStore" +// The cloud-client singleton now lives in island (keystone #5). Construction + +// runtime-hook wiring happen here; the host's @/services/cloudClient is a thin +// delegating wrapper that keeps endpoint resolution (dev/settings) host-side. +export {cloudClientService} from "./services/CloudClientService" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 95ac3d1590..a67436e905 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -12,6 +12,7 @@ import {configure, start, stop} from "./runtime/bootstrap" import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {speech} from "./facades/speech" +import {session} from "./facades/session" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -26,6 +27,8 @@ export const toolkit = { stop, glasses, speech, + /** Cloud (cloud-v2) live-session status — island owns the client (keystone #5). */ + session, display: { mirror: displayMirror, }, diff --git a/mobile/modules/island/src/runtime/bootstrap.ts b/mobile/modules/island/src/runtime/bootstrap.ts index 1659de2eb6..bddcffa316 100644 --- a/mobile/modules/island/src/runtime/bootstrap.ts +++ b/mobile/modules/island/src/runtime/bootstrap.ts @@ -28,6 +28,11 @@ export interface IslandConfigValues { runtimeUrl?: string /** OEM identifier (Mentra is OEM #0); reserved for OEM auth/telemetry. */ oemId?: string + /** + * LC3 frame size (bytes) the phone's mic encoder emits — announced to the + * cloud on connect (20 for G1, 40 for G2, …). Defaults to 20 if unset. + */ + audioFrameSizeBytes?: number } export type IslandAnalytics = (event: string, props?: Record) => void diff --git a/mobile/modules/island/src/services/CloudClientService.ts b/mobile/modules/island/src/services/CloudClientService.ts new file mode 100644 index 0000000000..8ca4608b9b --- /dev/null +++ b/mobile/modules/island/src/services/CloudClientService.ts @@ -0,0 +1,376 @@ +/** + * Owns the singleton `@mentra/cloud-client` (CloudClient) for the island runtime + * — the keystone move that pulls backend comms INTO island. + * + * island constructs the client from island-owned pieces (the native UDP socket, + * the MMKV secure store, the cloud-status store) plus the two things the host + * provides through the toolkit front door: `auth.getSubjectToken` (the one + * permanent seam — the OEM owns login) and the resolved cloud endpoints + * (`config.coreUrl/runtimeUrl`, computed host-side from dev/settings). On init it + * self-wires the `cloud` + `cloudConnection` runtime hooks, so the host no longer + * injects a CloudRuntimeAdapter — island owns construction AND wiring. + * + * The local-miniapp path drives the cloud's transcription/translation through + * the adapter this registers. + */ +import {CloudClient, setNativeUdp, setSecureStorage} from "@mentra/cloud-client/react-native" +import type {RuntimeSnapshot} from "@mentra/cloud-client/react-native" +import type {SubjectTokenType} from "@mentra/cloud-client" +import type {AudioSubscription, TranscriptionData, TranslationData} from "@mentra/cloud-runtime/protocol" + +import {getAuth, getConfigValues} from "../runtime/bootstrap" +import { + configureRuntime, + type CloudClientStatusSnapshot, + type CloudRuntimeAdapter, +} from "../runtime/config" +import {createCloudUdpSocket} from "../utils/cloudClient/RnUdpAdapter" +import {cloudSecureStore} from "../utils/cloudClient/cloudSecureStore" +import {useCloudClientStatusStore} from "../stores/cloudClientStatus" + +const LOG_TAG = "cloudClient" + +type Lc3FrameSizeBytes = 20 | 40 | 60 + +// Neutral last-ditch fallbacks (reachable under `adb reverse`) for when the host +// passes no endpoints. The host normally resolves the real URLs (dev override / +// Metro host / env) and hands them in via `toolkit.configure({config})`. +const FALLBACK_CORE_URL = "http://localhost:3000" +const FALLBACK_RUNTIME_URL = "http://localhost:3001" + +let client: CloudClient | null = null +let adapter: CloudRuntimeAdapter | null = null +let wired = false +let connected = false +let audioSubscriptions: AudioSubscription[] = [] +let transportsReady = false +/** Endpoints to build with — seeded from config, overridable via reconnect(). */ +let endpointsOverride: {core: string; runtime: string} | null = null +let runtimeStatusUnsubscribe: (() => void) | null = null +let transcriptUnsubscribe: (() => void) | null = null +let translationUnsubscribe: (() => void) | null = null + +const transcriptListeners = new Set<(d: TranscriptionData) => void>() +const translationListeners = new Set<(d: TranslationData) => void>() +const statusListeners = new Set<(snapshot: CloudClientStatusSnapshot) => void>() +const connectionListeners = new Set<(connected: boolean) => void>() + +function resolveEndpoints(): {core: string; runtime: string} { + if (endpointsOverride) return endpointsOverride + const cfg = getConfigValues() + return { + core: cfg.coreUrl?.trim() || FALLBACK_CORE_URL, + runtime: cfg.runtimeUrl?.trim() || FALLBACK_RUNTIME_URL, + } +} + +function frameSizeBytes(): Lc3FrameSizeBytes { + const size = getConfigValues().audioFrameSizeBytes + return size === 20 || size === 40 || size === 60 ? size : 20 +} + +async function getSubjectToken(): Promise<{token: string; type: SubjectTokenType}> { + const a = getAuth() + if (!a) throw new Error("cloudClient: toolkit.configure({auth}) not called") + const r = await a.getSubjectToken() + // IslandAuth's SubjectTokenType is intentionally open (`string & {}`) so OEMs + // can use other token kinds; cloud-client's is a closed union. The host's + // actual value ("supabase") is valid — narrow at this boundary. + return {token: r.token, type: r.type as SubjectTokenType} +} + +function toCloudClientStatusSnapshot(snapshot: RuntimeSnapshot): CloudClientStatusSnapshot { + return {status: snapshot.status, audioTransport: snapshot.audioTransport} +} + +function currentRuntimeStatus(): CloudClientStatusSnapshot { + const state = useCloudClientStatusStore.getState() + return {status: state.status, audioTransport: state.audioTransport} +} + +function setRuntimeStatus(snapshot: RuntimeSnapshot): void { + useCloudClientStatusStore.getState().setSnapshot(snapshot) + emitStatus(toCloudClientStatusSnapshot(snapshot)) +} + +function resetRuntimeStatus(): void { + useCloudClientStatusStore.getState().reset() + emitStatus(currentRuntimeStatus()) +} + +function stringifyMeta(meta: unknown): string { + if (!meta || typeof meta !== "object") return "" + try { + return ` ${JSON.stringify(meta)}` + } catch { + return "" + } +} + +const cloudLogger = { + debug: (msg: string, meta?: unknown) => console.log(`${LOG_TAG}: debug: ${msg}${stringifyMeta(meta)}`), + info: (msg: string, meta?: unknown) => console.log(`${LOG_TAG}: info: ${msg}${stringifyMeta(meta)}`), + warn: (msg: string, meta?: unknown) => console.warn(`${LOG_TAG}: warn: ${msg}${stringifyMeta(meta)}`), + error: (msg: string, meta?: unknown) => console.warn(`${LOG_TAG}: error: ${msg}${stringifyMeta(meta)}`), +} + +function notifyConnectionListeners(next: boolean): void { + for (const l of connectionListeners) { + try { + l(next) + } catch (err) { + console.warn(`${LOG_TAG}: connection listener threw: ${(err as Error)?.message ?? err}`) + } + } +} + +function emitTranscript(data: TranscriptionData): void { + for (const l of transcriptListeners) { + try { + l(data) + } catch (err) { + console.warn(`${LOG_TAG}: transcript listener threw: ${(err as Error)?.message ?? err}`) + } + } +} + +function emitTranslation(data: TranslationData): void { + for (const l of translationListeners) { + try { + l(data) + } catch (err) { + console.warn(`${LOG_TAG}: translation listener threw: ${(err as Error)?.message ?? err}`) + } + } +} + +function emitStatus(snapshot: CloudClientStatusSnapshot): void { + for (const l of statusListeners) { + try { + l(snapshot) + } catch (err) { + console.warn(`${LOG_TAG}: status listener threw: ${(err as Error)?.message ?? err}`) + } + } +} + +function ensureTransports(): void { + if (transportsReady) return + transportsReady = true + setNativeUdp(() => createCloudUdpSocket()) + setSecureStorage(cloudSecureStore) +} + +function clearRuntimeEventSubscriptions(): void { + runtimeStatusUnsubscribe?.() + runtimeStatusUnsubscribe = null + transcriptUnsubscribe?.() + transcriptUnsubscribe = null + translationUnsubscribe?.() + translationUnsubscribe = null +} + +function buildAdapter(): CloudRuntimeAdapter { + return { + setSubscriptions: async (subs: AudioSubscription[]): Promise => { + // Cache unconditionally so the desired set survives a not-yet-connected + // session and reconnects; only push when connected (the runtime throws + // otherwise). The onConnected handler re-applies the cached set, so + // subscribe-before-connect self-heals. + audioSubscriptions = subs + const c = client + if (connected && c) { + await c.runtime.setSubscriptions(subs) + } + }, + sendAudioFrame: (frame: Uint8Array): void => { + client?.runtime.sendAudioFrame(frame) + }, + onTranscript: (cb: (d: TranscriptionData) => void): (() => void) => { + transcriptListeners.add(cb) + return () => { + transcriptListeners.delete(cb) + } + }, + onTranslation: (cb: (d: TranslationData) => void): (() => void) => { + translationListeners.add(cb) + return () => { + translationListeners.delete(cb) + } + }, + getStatus: (): CloudClientStatusSnapshot => currentRuntimeStatus(), + onStatusChanged: (cb: (snapshot: CloudClientStatusSnapshot) => void): (() => void) => { + statusListeners.add(cb) + return () => { + statusListeners.delete(cb) + } + }, + tts: { + speak: (text, options) => { + if (!client) throw new Error("cloud client not connected") + return client.runtime.tts.speak(text, options) + }, + }, + hasAudioSubscriptions: (): boolean => audioSubscriptions.length > 0, + isConnected: (): boolean => connected, + } +} + +/** Register island's own cloud adapter + connection surface into the runtime + * hooks (once). Replaces the host's former `configureRuntime({cloud, ...})`. */ +function selfWire(): void { + if (wired) return + wired = true + configureRuntime({ + cloud: adapter ?? undefined, + cloudConnection: { + isConnected: () => connected, + addListener: (l) => cloudClientService.onConnectionChange(l), + }, + }) +} + +function construct(): void { + ensureTransports() + if (!adapter) adapter = buildAdapter() + + const endpoints = resolveEndpoints() + console.log(`${LOG_TAG}: endpoints ${JSON.stringify(endpoints)}`) + + client = new CloudClient({ + endpoints, + // The phone LC3-encodes mic audio (even in phone/simulated mode); announce + // LC3 at 16 kHz with the frame size the encoder emits. + audio: {codec: "lc3", sampleRate: 16000, frameSizeBytes: frameSizeBytes()}, + auth: {getSubjectToken}, + logger: cloudLogger, + }) + + const c = client + clearRuntimeEventSubscriptions() + runtimeStatusUnsubscribe = c.runtime.onStatusChanged((status) => { + if (c !== client) return + setRuntimeStatus(status) + }) + transcriptUnsubscribe = c.runtime.onTranscript((data) => { + if (c !== client) return + emitTranscript(data) + }) + translationUnsubscribe = c.runtime.onTranslation((data) => { + if (c !== client) return + emitTranslation(data) + }) + setRuntimeStatus(c.runtime.getStatus()) + + c.runtime.onConnected(() => { + if (c !== client) return + connected = true + console.log(`${LOG_TAG}: runtime connected`) + // Re-apply any subscriptions queued before connect (or dropped across a + // reconnect) — best-effort; never throw out of the connect handler. + if (audioSubscriptions.length > 0) { + c.runtime + .setSubscriptions(audioSubscriptions) + .catch((err) => + console.warn(`${LOG_TAG}: re-applying queued subscriptions failed: ${(err as Error)?.message ?? err}`), + ) + } + notifyConnectionListeners(true) + }) + c.runtime.onDisconnected((info) => { + if (c !== client) return + connected = false + console.log(`${LOG_TAG}: runtime disconnected (${info.reason})`) + notifyConnectionListeners(false) + }) + c.runtime.onError((err) => { + console.warn(`${LOG_TAG}: runtime error: ${err.code}`) + }) + + // Best-effort connect. Do not crash the app if the dev cloud is unreachable. + c.runtime + .connect() + .then(() => console.log(`${LOG_TAG}: connect() resolved`)) + .catch((err) => console.warn(`${LOG_TAG}: connect() failed: ${err?.message ?? err}`)) +} + +/** + * island's cloud client. Owns the singleton CloudClient, its connection state, + * and the runtime-hook wiring. + */ +export const cloudClientService = { + /** + * Construct (once) + connect the client, and self-wire the runtime cloud + * hooks. Idempotent. Best-effort connect — a failure is logged and the app + * keeps running. Requires `toolkit.configure({auth, config})` first. + */ + init(): void { + if (client) { + selfWire() + return + } + construct() + selfWire() + }, + + /** + * Tear down + rebuild with freshly-resolved endpoints (the dev "Cloud V2" + * override, applied without an app rebuild). Pass new endpoints to switch + * URLs; omit to rebuild with the current config. + */ + reconnect(endpoints?: {core: string; runtime: string}): void { + if (endpoints) endpointsOverride = endpoints + try { + client?.runtime.close() + } catch (err) { + console.warn(`${LOG_TAG}: reconnect close() failed: ${(err as Error)?.message ?? err}`) + } + clearRuntimeEventSubscriptions() + + const wasConnected = connected + client = null + connected = false + resetRuntimeStatus() + if (wasConnected) notifyConnectionListeners(false) + + construct() + selfWire() + }, + + /** Device-side managed photo (cloud-v2): presign now, deliver bytes, await ready. */ + startManagedPhoto(opts: Record = {}) { + if (!client) throw new Error("cloud client not connected") + return client.runtime.startManagedPhoto(opts) + }, + awaitManagedPhotoReady(requestId: string) { + if (!client) throw new Error("cloud client not connected") + return client.runtime.awaitManagedPhotoReady(requestId) + }, + + /** Managed stream (cloud-v2): provision ingest+playback on the runtime. */ + startManagedStream(opts: Record = {}) { + if (!client) throw new Error("cloud client not connected") + return client.runtime.startManagedStream(opts) + }, + getManagedStreamStatus(streamId: string) { + if (!client) throw new Error("cloud client not connected") + return client.runtime.getManagedStreamStatus(streamId) + }, + stopManagedStream(streamId: string) { + if (!client) throw new Error("cloud client not connected") + return client.runtime.stopManagedStream(streamId) + }, + + /** Current live-session connection state (handshake completed). */ + isConnected(): boolean { + return connected + }, + + /** Subscribe to connection-state transitions. Returns an unsubscribe fn. */ + onConnectionChange(listener: (connected: boolean) => void): () => void { + connectionListeners.add(listener) + return () => { + connectionListeners.delete(listener) + } + }, +} diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index ebcde44fbd..c1c804ec12 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -18,7 +18,7 @@ import {BUNDLED_MINIAPPS} from "@/generated/bundledMiniapps" import {migrate} from "@/services/Migrations" import restComms from "@/services/RestComms" import socketComms from "@/services/SocketComms" -import {cloudClient} from "@/services/cloudClient" +import {cloudClient, cloudConfigValues} from "@/services/cloudClient" import {devServerHost} from "@/utils/cloudClient/devHost" import {gallerySyncService} from "@/services/asg/gallerySyncService" import {handleOtaClockSkewFromGlasses} from "@/services/asg/glassesClockSync" @@ -179,6 +179,9 @@ class MantleManager { return {token: res.value.token, type: "supabase"} }, }, + // Resolved cloud endpoints + LC3 frame size. island builds its cloud + // client from these; the host keeps the dev/settings URL resolution. + config: cloudConfigValues(), }) void toolkit.start() @@ -193,16 +196,18 @@ class MantleManager { // island service that reads settings / glasses status / sockets / audio // (LocalMiniappRuntime, LocalDisplayManager, LocalSttFallbackCoordinator, // DisplayProcessor) is touched. - // Construct + connect the cloud client (best-effort) and wire its runtime - // adapter. The island/local-miniapp path is powered by this client. - const cloud = cloudClient.init() + // Construct + connect the cloud client (best-effort). It now lives in island + // (cloudClientService) and self-wires the runtime cloud/cloudConnection + // hooks from island-owned transports + the auth/config passed above, so the + // host no longer injects a CloudRuntimeAdapter. The local-miniapp path is + // powered by this client. + cloudClient.init() configureRuntime({ socketComms: { sendMessage: (message) => socketComms.sendMessage(message as Parameters[0]), updatePhoneSubscriptions: (subs) => socketComms.updatePhoneSubscriptions(subs), }, - cloud, audioPlayback: { play: (request, onComplete) => audioPlaybackService.play(request, onComplete), stopForApp: (packageName) => audioPlaybackService.stopForApp(packageName), @@ -232,18 +237,9 @@ class MantleManager { (value) => onChange(value as never), ), }, - cloudConnection: { - // Local island miniapps are powered ONLY by the cloud client, so the - // on-device STT fallback for miniapps must track cloud liveness, not the - // v1 WebSocket. When the cloud is connected and delivering transcripts - // the fallback stays off (the cloud powers captions); when it is down, - // local STT engages. This adapter is consumed exclusively by - // LocalSttFallbackCoordinator, whose only consumer is the local-miniapp - // path. The glasses offline-captions display runs off its own - // `offline_captions_running` setting and is unaffected. - isConnected: () => cloudClient.isConnected(), - addListener: (l) => cloudClient.onConnectionChange(l), - }, + // cloudConnection is self-wired by island's cloudClientService now (it + // powers the local-miniapp on-device-STT fallback off cloud liveness), so + // the host no longer injects it. // The dev laptop's live address, from Metro. The island runtime uses it // to repair persisted dev-miniapp URLs that froze a previous network's // IP (the bundle host is, by construction, reachable right now). diff --git a/mobile/src/services/cloudClient.ts b/mobile/src/services/cloudClient.ts index ff9ed89c05..d530c61657 100644 --- a/mobile/src/services/cloudClient.ts +++ b/mobile/src/services/cloudClient.ts @@ -1,26 +1,23 @@ /** - * @fileoverview Owns the singleton `@mentra/cloud-client` (CloudClient) for the - * island runtime and exposes it as a `CloudRuntimeAdapter`. + * @fileoverview Thin host wrapper over island's cloud client (keystone #5). * - * The island/local-miniapp path talks to the cloud through this client. The - * island runtime drives the cloud's transcription/translation through the - * adapter returned by `cloudClient.init()`. + * The CloudClient singleton now lives in `@mentra/island` (`cloudClientService`): + * island constructs it from island-owned transports (UDP, MMKV secure store, + * status store) + the host-injected `auth` seam + the resolved endpoints the + * host passes via `toolkit.configure({config})`, and self-wires the runtime + * `cloud`/`cloudConnection` hooks. * - * The RN transports (native UDP, secure storage) are host-injected once here - * via `setNativeUdp` / `setSecureStorage` BEFORE the client is constructed. + * What stays here is the host-side ENDPOINT RESOLUTION — the rebuild-free Dev + * Settings URL switcher, which reads the host settings store + the live Metro + * host. The host resolves the URLs and hands them to island; this file also + * delegates the managed-photo / managed-stream / connection surface so existing + * `@/services/cloudClient` consumers keep working unchanged. (Endpoint + * resolution moves into island later with the `dev` domain.) */ -import {CloudClient, setNativeUdp, setSecureStorage} from "@mentra/cloud-client/react-native" -import type {RuntimeSnapshot} from "@mentra/cloud-client/react-native" -import type {AudioSubscription, TranscriptionData, TranslationData} from "@mentra/cloud-runtime/protocol" -import {createCloudUdpSocket, type CloudClientStatusSnapshot, type CloudRuntimeAdapter} from "@mentra/island" +import {cloudClientService} from "@mentra/island" -import mentraAuth from "@/utils/auth/authClient" -import {useCloudClientStatusStore} from "@/stores/cloudClientStatus" import {SETTINGS, useSettingsStore} from "@/stores/settings" import {devServerHost, METRO_AUTO} from "@/utils/cloudClient/devHost" -import {cloudSecureStore} from "@/utils/cloudClient/MmkvSecureStore" - -const LOG_TAG = "cloudClient" type Lc3FrameSizeBytes = 20 | 40 | 60 @@ -91,345 +88,36 @@ export function resolvedEndpoints(): {core: string; runtime: string} { return {core: coreUrl(), runtime: runtimeUrl()} } -/** - * Read the live Supabase access token on demand. `mentraAuth.getSession()` - * returns the current (auto-refreshed) session, so the client always exchanges - * a fresh subject token. Never log the token. - */ -async function getSupabaseSubjectToken(): Promise<{token: string; type: "supabase"}> { - const res = await mentraAuth.getSession() - if (res.is_error() || !res.value.token) { - throw new Error("cloudClient: no Supabase session token available") - } - return {token: res.value.token, type: "supabase"} -} - -/** - * Holds the singleton client plus the currently-applied audio subscription set. - * We track the set locally (rather than reading it back off the client) so the - * audio-capture site can gate sends with a synchronous `hasAudioSubscriptions`. - */ -let client: CloudClient | null = null -let adapter: CloudRuntimeAdapter | null = null -let connected = false -let audioSubscriptions: AudioSubscription[] = [] -let transportsReady = false -let runtimeStatusUnsubscribe: (() => void) | null = null -let transcriptUnsubscribe: (() => void) | null = null -let translationUnsubscribe: (() => void) | null = null - -const transcriptListeners = new Set<(d: TranscriptionData) => void>() -const translationListeners = new Set<(d: TranslationData) => void>() -const statusListeners = new Set<(snapshot: CloudClientStatusSnapshot) => void>() - -function toCloudClientStatusSnapshot(snapshot: RuntimeSnapshot): CloudClientStatusSnapshot { - return { - status: snapshot.status, - audioTransport: snapshot.audioTransport, - } -} - -function currentRuntimeStatus(): CloudClientStatusSnapshot { - const state = useCloudClientStatusStore.getState() - return { - status: state.status, - audioTransport: state.audioTransport, - } -} - -function setRuntimeStatus(snapshot: RuntimeSnapshot): void { - useCloudClientStatusStore.getState().setSnapshot(snapshot) - emitStatus(toCloudClientStatusSnapshot(snapshot)) -} - -function resetRuntimeStatus(): void { - useCloudClientStatusStore.getState().reset() - emitStatus(currentRuntimeStatus()) -} - -function stringifyMeta(meta: unknown): string { - if (!meta || typeof meta !== "object") return "" - try { - return ` ${JSON.stringify(meta)}` - } catch { - return "" - } -} - -const cloudLogger = { - debug: (msg: string, meta?: unknown) => console.log(`${LOG_TAG}: debug: ${msg}${stringifyMeta(meta)}`), - info: (msg: string, meta?: unknown) => console.log(`${LOG_TAG}: info: ${msg}${stringifyMeta(meta)}`), - warn: (msg: string, meta?: unknown) => console.warn(`${LOG_TAG}: warn: ${msg}${stringifyMeta(meta)}`), - error: (msg: string, meta?: unknown) => console.warn(`${LOG_TAG}: error: ${msg}${stringifyMeta(meta)}`), -} - -/** - * Listeners that want to know when the live session connects/disconnects. The - * host wires the LocalSttFallbackCoordinator's `cloudConnection` adapter to - * these so the local-miniapp on-device-STT fallback tracks cloud liveness (not - * the v1 WebSocket). Notified on every transition out of `onConnected`/ - * `onDisconnected`. - */ -const connectionListeners = new Set<(connected: boolean) => void>() - -function notifyConnectionListeners(next: boolean): void { - for (const l of connectionListeners) { - try { - l(next) - } catch (err) { - console.warn(`${LOG_TAG}: connection listener threw: ${(err as Error)?.message ?? err}`) - } - } -} - -function ensureTransports(): void { - if (transportsReady) return - transportsReady = true - setNativeUdp(() => createCloudUdpSocket()) - setSecureStorage(cloudSecureStore) -} - -function lc3FrameSizeBytes(): Lc3FrameSizeBytes { +/** The LC3 frame size (bytes) the phone's encoder currently emits. */ +export function lc3FrameSizeBytes(): Lc3FrameSizeBytes { const frameSize = useSettingsStore.getState().getSetting(SETTINGS.lc3_frame_size.key) return frameSize === 20 || frameSize === 40 || frameSize === 60 ? frameSize : 20 } -function emitTranscript(data: TranscriptionData): void { - for (const l of transcriptListeners) { - try { - l(data) - } catch (err) { - console.warn(`${LOG_TAG}: transcript listener threw: ${(err as Error)?.message ?? err}`) - } - } -} - -function emitTranslation(data: TranslationData): void { - for (const l of translationListeners) { - try { - l(data) - } catch (err) { - console.warn(`${LOG_TAG}: translation listener threw: ${(err as Error)?.message ?? err}`) - } - } -} - -function emitStatus(snapshot: CloudClientStatusSnapshot): void { - for (const l of statusListeners) { - try { - l(snapshot) - } catch (err) { - console.warn(`${LOG_TAG}: status listener threw: ${(err as Error)?.message ?? err}`) - } - } -} - -function clearRuntimeEventSubscriptions(): void { - runtimeStatusUnsubscribe?.() - runtimeStatusUnsubscribe = null - transcriptUnsubscribe?.() - transcriptUnsubscribe = null - translationUnsubscribe?.() - translationUnsubscribe = null -} - -function buildAdapter(): CloudRuntimeAdapter { - return { - setSubscriptions: async (subs: AudioSubscription[]): Promise => { - // Cache the desired state unconditionally so it survives a not-yet- - // connected session and reconnects. Only push to the runtime when the - // session is actually connected. `c.runtime.setSubscriptions` throws - // "Cannot set subscriptions before the session is connected" otherwise, - // and nothing would retry. The `onConnected` handler re-applies the - // cached set, so subscribe-before-connect self-heals. - audioSubscriptions = subs - const c = client - if (connected && c) { - await c.runtime.setSubscriptions(subs) - } - }, - sendAudioFrame: (frame: Uint8Array): void => { - client?.runtime.sendAudioFrame(frame) - }, - onTranscript: (cb: (d: TranscriptionData) => void): (() => void) => { - transcriptListeners.add(cb) - return () => { - transcriptListeners.delete(cb) - } - }, - onTranslation: (cb: (d: TranslationData) => void): (() => void) => { - translationListeners.add(cb) - return () => { - translationListeners.delete(cb) - } - }, - getStatus: (): CloudClientStatusSnapshot => currentRuntimeStatus(), - onStatusChanged: (cb: (snapshot: CloudClientStatusSnapshot) => void): (() => void) => { - statusListeners.add(cb) - return () => { - statusListeners.delete(cb) - } - }, - tts: { - speak: (text, options) => { - if (!client) throw new Error("cloud client not connected") - return client.runtime.tts.speak(text, options) - }, - }, - hasAudioSubscriptions: (): boolean => audioSubscriptions.length > 0, - isConnected: (): boolean => connected, - } +/** + * The cloud config the host hands island at `toolkit.configure({config})`: + * resolved endpoints + the live LC3 frame size. + */ +export function cloudConfigValues(): {coreUrl: string; runtimeUrl: string; audioFrameSizeBytes: number} { + const endpoints = resolvedEndpoints() + return {coreUrl: endpoints.core, runtimeUrl: endpoints.runtime, audioFrameSizeBytes: lc3FrameSizeBytes()} } /** - * The island runtime's cloud client. Owns the singleton CloudClient and its - * connection state, and exposes the runtime adapter the island runtime wires - * in. + * Host-facing handle to island's cloud client. Construction + runtime wiring + * live in island (`cloudClientService`); this delegates so existing consumers + * (PhonePhotoCoordinator, cloudStreamApi, the dev Cloud-URL switcher) are + * untouched. `reconnect()` re-resolves the host endpoints before rebuilding. */ export const cloudClient = { - /** Device-side managed photo (cloud-v2): presign now, deliver bytes, await ready. */ - startManagedPhoto(opts: Record = {}) { - if (!client) throw new Error("cloud client not connected") - return client.runtime.startManagedPhoto(opts) - }, - awaitManagedPhotoReady(requestId: string) { - if (!client) throw new Error("cloud client not connected") - return client.runtime.awaitManagedPhotoReady(requestId) - }, - - /** Managed stream (cloud-v2): provision ingest+playback on the runtime. */ - startManagedStream(opts: Record = {}) { - if (!client) throw new Error("cloud client not connected") - return client.runtime.startManagedStream(opts) - }, - getManagedStreamStatus(streamId: string) { - if (!client) throw new Error("cloud client not connected") - return client.runtime.getManagedStreamStatus(streamId) - }, - stopManagedStream(streamId: string) { - if (!client) throw new Error("cloud client not connected") - return client.runtime.stopManagedStream(streamId) - }, - - /** - * Construct (once) and connect the CloudClient, returning the runtime adapter - * the island runtime wires in. Idempotent: repeated calls return the same - * adapter. The connect is best-effort — a failure is logged and the app keeps - * running. - */ - init(): CloudRuntimeAdapter { - if (adapter && client) return adapter - - ensureTransports() - if (!adapter) { - adapter = buildAdapter() - } - - const endpoints = {core: coreUrl(), runtime: runtimeUrl()} - console.log(`${LOG_TAG}: endpoints ${JSON.stringify(endpoints)}`) - - client = new CloudClient({ - endpoints, - // The phone LC3-encodes mic audio (even in phone/simulated mode), so we - // announce LC3 at 16 kHz with the same frame size the encoder emits. - audio: {codec: "lc3", sampleRate: 16000, frameSizeBytes: lc3FrameSizeBytes()}, - auth: {getSubjectToken: getSupabaseSubjectToken}, - logger: cloudLogger, - }) - - const c = client - clearRuntimeEventSubscriptions() - runtimeStatusUnsubscribe = c.runtime.onStatusChanged((status) => { - if (c !== client) return - setRuntimeStatus(status) - }) - transcriptUnsubscribe = c.runtime.onTranscript((data) => { - if (c !== client) return - emitTranscript(data) - }) - translationUnsubscribe = c.runtime.onTranslation((data) => { - if (c !== client) return - emitTranslation(data) - }) - setRuntimeStatus(c.runtime.getStatus()) - - c.runtime.onConnected(() => { - if (c !== client) return - connected = true - console.log(`${LOG_TAG}: runtime connected`) - // Re-apply any subscriptions queued before connect (or dropped across a - // reconnect). Without this the local miniapp's transcription subscription - // would never land and the cloud would never power its captions. Best- - // effort: log on failure, never throw out of the connect handler. - if (audioSubscriptions.length > 0) { - c.runtime - .setSubscriptions(audioSubscriptions) - .catch((err) => - console.warn(`${LOG_TAG}: re-applying queued subscriptions failed: ${(err as Error)?.message ?? err}`), - ) - } - notifyConnectionListeners(true) - }) - c.runtime.onDisconnected((info) => { - if (c !== client) return - connected = false - console.log(`${LOG_TAG}: runtime disconnected (${info.reason})`) - notifyConnectionListeners(false) - }) - c.runtime.onError((err) => { - console.warn(`${LOG_TAG}: runtime error: ${err.code}`) - }) - - // Best-effort connect. Do not crash the app if the dev cloud is unreachable. - c.runtime - .connect() - .then(() => console.log(`${LOG_TAG}: connect() resolved`)) - .catch((err) => console.warn(`${LOG_TAG}: connect() failed: ${err?.message ?? err}`)) - - return adapter - }, - - /** - * Tear down the current client and re-init with freshly-resolved endpoint - * URLs. Used by the dev "Cloud V2" settings override so a new core/runtime URL - * takes effect without an app rebuild. The CloudClient exposes its teardown - * via `runtime.close()` (the top-level client has no `disconnect`/`close`), so - * we close that, drop the singletons, and call `init()` to rebuild. - */ - reconnect(): void { - try { - client?.runtime.close() - } catch (err) { - console.warn(`${LOG_TAG}: reconnect close() failed: ${(err as Error)?.message ?? err}`) - } - clearRuntimeEventSubscriptions() - - const wasConnected = connected - client = null - connected = false - resetRuntimeStatus() - // Notify so the local-miniapp STT fallback engages while the client is torn - // down and before the rebuilt client completes its handshake. - if (wasConnected) { - notifyConnectionListeners(false) - } - - this.init() - }, - - /** Current live-session connection state (handshake completed). */ - isConnected(): boolean { - return connected - }, - - /** - * Subscribe to connection-state transitions. Returns an unsubscribe fn. Used - * by the host to drive the local-miniapp STT fallback off cloud liveness. - */ - onConnectionChange(listener: (connected: boolean) => void): () => void { - connectionListeners.add(listener) - return () => { - connectionListeners.delete(listener) - } - }, + init: (): void => cloudClientService.init(), + reconnect: (): void => cloudClientService.reconnect(resolvedEndpoints()), + startManagedPhoto: (opts: Record = {}) => cloudClientService.startManagedPhoto(opts), + awaitManagedPhotoReady: (requestId: string) => cloudClientService.awaitManagedPhotoReady(requestId), + startManagedStream: (opts: Record = {}) => cloudClientService.startManagedStream(opts), + getManagedStreamStatus: (streamId: string) => cloudClientService.getManagedStreamStatus(streamId), + stopManagedStream: (streamId: string) => cloudClientService.stopManagedStream(streamId), + isConnected: (): boolean => cloudClientService.isConnected(), + onConnectionChange: (listener: (connected: boolean) => void): (() => void) => + cloudClientService.onConnectionChange(listener), } From 68b1fcebfccd9d84e358ca61bc9b5da681229e9a Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 14:32:21 -0700 Subject: [PATCH 17/80] island(settings prep): move GlobalEventEmitter in + add loadSubKeys to island storage Increment 1 of the settings+RestComms co-move. GlobalEventEmitter -> island (one shared instance, host shim); ports storage.loadSubKeys() into island's MMKV util (settings.ts needs it). jest mock gains the shared emitter. --- mobile/jest.setup.js | 3 +++ mobile/modules/island/src/index.ts | 3 +++ .../island/src/utils/GlobalEventEmitter.ts | 12 ++++++++++ .../island/src/utils/storage/storage.ts | 24 +++++++++++++++++++ mobile/src/utils/GlobalEventEmitter.tsx | 13 ++++------ 5 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 mobile/modules/island/src/utils/GlobalEventEmitter.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index ed0d4f6da2..21cdc110eb 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -336,6 +336,9 @@ jest.mock("@mentra/island", () => { cloudClientStatus: realCloudStatus.useCloudClientStatusStore, }, }, + // Shared process-wide event bus (moved into island) — a real EventEmitter so + // tests that emit/listen across the boundary work. + GlobalEventEmitter: new (require("events").EventEmitter)(), // island now owns the cloud client (keystone #5); the host wrapper delegates // to this. Mocked so host/service tests don't construct a real CloudClient. cloudClientService: { diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 240416bfc2..faf7923110 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -208,6 +208,9 @@ export {createCloudUdpSocket} from "./utils/cloudClient/RnUdpAdapter" export {HardwareCompatibility, type CompatibilityResult} from "./utils/hardware" export {BgTimer, throttle, debounce} from "./utils/timers" export {storage, printDirectory} from "./utils/storage" +// Process-wide event bus — one shared instance for island services + host +// (re-exported via the @/utils/GlobalEventEmitter shim). +export {default as GlobalEventEmitter} from "./utils/GlobalEventEmitter" // Types (copied from @mentra/types — keep in sync with cloud/packages/types/src) export { diff --git a/mobile/modules/island/src/utils/GlobalEventEmitter.ts b/mobile/modules/island/src/utils/GlobalEventEmitter.ts new file mode 100644 index 0000000000..4452a46937 --- /dev/null +++ b/mobile/modules/island/src/utils/GlobalEventEmitter.ts @@ -0,0 +1,12 @@ +import {EventEmitter} from "events" + +/** + * Process-wide event bus — moved into island so island-owned services (RestComms) + * and the host share ONE emitter instance across the island↔host boundary. + * Re-exported through the host's `@/utils/GlobalEventEmitter` shim. + * + * @deprecated Use BluetoothSDK subscriptions directly instead. + */ +const GlobalEventEmitter = new EventEmitter() + +export default GlobalEventEmitter diff --git a/mobile/modules/island/src/utils/storage/storage.ts b/mobile/modules/island/src/utils/storage/storage.ts index 2f0b603c85..c6aa3b1956 100644 --- a/mobile/modules/island/src/utils/storage/storage.ts +++ b/mobile/modules/island/src/utils/storage/storage.ts @@ -51,6 +51,30 @@ class MMKVStorage { return Res.ok(undefined) } + public loadSubKeys(key: string): Result, Error> { + return Res.try(() => { + // return the key value pair of any keys that start with the given key and contain a colon: + const keys = this.store.getAllKeys() + + const subKeys = keys.filter((k) => k.startsWith(key) && k.includes(":")) + + if (subKeys.length === 0) { + throw new Error(`No subkeys found for ${key}`) + } + + let subKeysObject: Record = {} + + for (const subKey of subKeys) { + const res = this.load(subKey) + if (res.is_ok()) { + subKeysObject[subKey] = res.value + } + } + + return subKeysObject + }) + } + public remove(key: string): Result { this.store.remove(key) return Res.ok(undefined) diff --git a/mobile/src/utils/GlobalEventEmitter.tsx b/mobile/src/utils/GlobalEventEmitter.tsx index ff37146282..fc88c67e1b 100644 --- a/mobile/src/utils/GlobalEventEmitter.tsx +++ b/mobile/src/utils/GlobalEventEmitter.tsx @@ -1,11 +1,8 @@ -import {EventEmitter} from "events" - /** + * Re-export shim — the process-wide event bus now lives in @mentra/island (one + * shared instance for island services + host). Kept so existing + * `@/utils/GlobalEventEmitter` default imports resolve unchanged. + * * @deprecated Use BluetoothSDK subscriptions directly instead. */ -const GlobalEventEmitter = new EventEmitter() - -/** - * @deprecated Use BluetoothSDK subscriptions directly instead. - */ -export default GlobalEventEmitter +export {GlobalEventEmitter as default} from "@mentra/island" From 1076ab0f539eb1675795909c13fdfafae8149f6c Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 14:41:52 -0700 Subject: [PATCH 18/80] island(#6): move settings store + RestComms into island (the mutually-coupled v1-comms pair) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The biggest single migration — settings (964 LOC) + RestComms (731 LOC) move together because they're mutually coupled (settings calls restComms.writeUserSettings for cloud-sync; RestComms reads the settings store for the backend URL). Moving the pair into island resolves BOTH the circular dependency AND the early-auth timing trap (RestComms reads the backend URL at exchangeToken, before island's config/adapters are wired — only the module-load-available store works there; now it's island's own store). - settings.ts -> island/src/stores; RestComms.ts -> island/src/services (git mv, history preserved). Host @/stores/settings + @/services/RestComms are re-export shims (96 + 24 consumers unchanged). Exposed as toolkit.stores.settings. - Import rewrites: settings -> island RestComms + island storage; RestComms -> island settings/connection stores, island GlobalEventEmitter, island types, BgTimer, and the internal btsdk surface via the relative build/_internal path (the @mentra/bluetooth-sdk-internal alias doesn't resolve in island's standalone build). - GlobalEventEmitter moved into island (one shared instance + host shim); ported storage.loadSubKeys into island's MMKV util; dropped two dead imports (Platform/Device). - jest: requireActual settings + RestComms (tests used the real host store before the move); moduleNameMapper maps the relative build/_internal path to the mockable source so requireActual doesn't load the real native btsdk; the mock exposes the REAL island GlobalEventEmitter so emit/listen share one instance. v1-transitional: RestComms is deleted in place when v1 retires; the point is the host no longer owns comms during the (multi-week) transition. SocketComms is a separate 4-6 week refactor and moves last. mobile typecheck + island standalone build + full jest (281 tests) all green. --- agents/island-facade-buildout.md | 3 +- mobile/jest.config.js | 5 + mobile/jest.setup.js | 16 +- mobile/modules/island/src/index.ts | 6 + mobile/modules/island/src/island.ts | 2 + .../modules/island/src/services/RestComms.ts | 735 +++++++++++++ mobile/modules/island/src/stores/settings.ts | 962 +++++++++++++++++ .../island/src/utils/GlobalEventEmitter.ts | 2 + mobile/src/services/RestComms.ts | 737 +------------ mobile/src/stores/settings.ts | 970 +----------------- 10 files changed, 1739 insertions(+), 1699 deletions(-) create mode 100644 mobile/modules/island/src/services/RestComms.ts create mode 100644 mobile/modules/island/src/stores/settings.ts diff --git a/agents/island-facade-buildout.md b/agents/island-facade-buildout.md index 11305268a2..2dc0e70c4a 100644 --- a/agents/island-facade-buildout.md +++ b/agents/island-facade-buildout.md @@ -77,7 +77,8 @@ pattern for btsdk types. | dev | `utils/cloudClient/devHost.ts` + core store | 1 | todo | | miniapps | apps store + LocalMiniappRuntime (island) + MiniappCatalog | 1/hard (WebView) | todo | | pairing | pairing screens state machine (readiness primitive already island) | 1 (extract) | todo | -| **settings** | `stores/settings.ts` (964 LOC) + `RestComms` + `storage` | **2 (keystone)** | todo — own careful commit; unblocks settings + glasses.settings + phoneNotifications | +| **settings** | `stores/settings.ts` (964 LOC) + `RestComms` + `storage` | keystone | **DONE (#3167)** — settings store moved into island, `toolkit.stores.settings`. Moved **together with RestComms** (mutually coupled: settings→RestComms cloud-sync, RestComms→settings backend URL). Storage uses island's MMKV (ported `loadSubKeys`). Unblocks glasses.settings + phoneNotifications. The typed `toolkit.settings` keyed facade (get/set/onChanged) is still TODO on top of the moved store. | +| **RestComms** (v1 REST) | `services/RestComms.ts` (731 LOC) | move-with-settings | **DONE (#3167)** — moved into island with settings (the coupled pair). v1-transitional: deleted in place when v1 retires. Reads backend URL from the now-island settings store directly (no early-auth timing hack). Host `@/services/RestComms` is a shim. GlobalEventEmitter also moved in (one shared instance). | | glasses.settings | settings store + btsdk | 2 (after settings) | blocked on settings | | phoneNotifications | settings store + crust + permissions | 2 (after settings) | blocked on settings | | gallery | `services/asg/gallerySyncService.ts` (~1000 LOC, hotspot) | hard | todo | diff --git a/mobile/jest.config.js b/mobile/jest.config.js index d87230451d..09877557cc 100644 --- a/mobile/jest.config.js +++ b/mobile/jest.config.js @@ -7,6 +7,11 @@ module.exports = { "^@/(.*)$": "/src/$1", "^@assets/(.*)$": "/assets/$1", "^@mentra/bluetooth-sdk-internal$": "/modules/bluetooth-sdk/src/_internal.ts", + // island-internal code (e.g. RestComms) reaches the full btsdk surface via the + // relative build/_internal path (the @mentra/bluetooth-sdk-internal alias + // doesn't resolve in island's standalone build). Map it to the same source so + // the jest.setup mock applies — otherwise requireActual loads the real native module. + "bluetooth-sdk/build/_internal$": "/modules/bluetooth-sdk/src/_internal.ts", }, testPathIgnorePatterns: [ "/modules/island/", diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 21cdc110eb..92b1847b94 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -242,6 +242,10 @@ jest.mock("@mentra/island", () => { const realConnection = jest.requireActual("./modules/island/src/stores/connection") const realGallerySync = jest.requireActual("./modules/island/src/stores/gallerySync") const realCloudStatus = jest.requireActual("./modules/island/src/stores/cloudClientStatus") + // Settings store + RestComms moved into island; tests used the real host store + // before the move, so requireActual preserves that exact behavior. + const realSettings = jest.requireActual("./modules/island/src/stores/settings") + const realRestComms = jest.requireActual("./modules/island/src/services/RestComms") const appStatusState = { apps: [], refresh: jest.fn(), @@ -269,6 +273,10 @@ jest.mock("@mentra/island", () => { ...realGallerySync, // Real cloud-client runtime status store (useCloudClientStatusStore). ...realCloudStatus, + // Real settings store (SETTINGS, useSettingsStore, useSetting, OFFLINE_APPLETS) + // + RestComms singleton — both moved into island. + ...realSettings, + restComms: realRestComms.default, // The namespaced (A) host API. Mirrors the real `toolkit` object; members are // jest.fn()s so host/screen tests can assert delegation without native btsdk. toolkit: { @@ -334,11 +342,13 @@ jest.mock("@mentra/island", () => { connection: realConnection.useConnectionStore, gallerySync: realGallerySync.useGallerySyncStore, cloudClientStatus: realCloudStatus.useCloudClientStatusStore, + settings: realSettings.useSettingsStore, }, }, - // Shared process-wide event bus (moved into island) — a real EventEmitter so - // tests that emit/listen across the boundary work. - GlobalEventEmitter: new (require("events").EventEmitter)(), + // Shared process-wide event bus (moved into island) — the REAL island + // instance (not a fresh one) so the instance RestComms emits on is the same + // one tests listen on across the boundary. + GlobalEventEmitter: jest.requireActual("./modules/island/src/utils/GlobalEventEmitter").default, // island now owns the cloud client (keystone #5); the host wrapper delegates // to this. Mocked so host/service tests don't construct a real CloudClient. cloudClientService: { diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index faf7923110..0932bde97e 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -126,6 +126,12 @@ export {cloudSecureStore} from "./utils/cloudClient/cloudSecureStore" // runtime-hook wiring happen here; the host's @/services/cloudClient is a thin // delegating wrapper that keeps endpoint resolution (dev/settings) host-side. export {cloudClientService} from "./services/CloudClientService" +// Settings store + RestComms — the mutually-coupled v1-comms pair, moved into +// island together (settings needs RestComms for cloud-sync; RestComms needs +// settings for the backend URL). Re-exported via host shims (@/stores/settings, +// @/services/RestComms). v1-transitional: deleted in place when v1 retires. +export {SETTINGS, OFFLINE_APPLETS, useSettingsStore, useSetting} from "./stores/settings" +export {default as restComms} from "./services/RestComms" // Bluetooth SDK passthrough — the full @mentra/bluetooth-sdk surface re-exported // so the app reaches the SDK through island instead of importing it directly. diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index a67436e905..5245759d69 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -19,6 +19,7 @@ import {useCoreStore} from "./stores/core" import {useConnectionStore} from "./stores/connection" import {useGallerySyncStore} from "./stores/gallerySync" import {useCloudClientStatusStore} from "./stores/cloudClientStatus" +import {useSettingsStore} from "./stores/settings" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ @@ -45,5 +46,6 @@ export const toolkit = { connection: useConnectionStore, gallerySync: useGallerySyncStore, cloudClientStatus: useCloudClientStatusStore, + settings: useSettingsStore, }, } diff --git a/mobile/modules/island/src/services/RestComms.ts b/mobile/modules/island/src/services/RestComms.ts new file mode 100644 index 0000000000..5a675c7139 --- /dev/null +++ b/mobile/modules/island/src/services/RestComms.ts @@ -0,0 +1,735 @@ +import axios, {AxiosInstance, AxiosRequestConfig} from "axios" +import {AsyncResult, Result, result as Res} from "typesafe-ts" + +import type {AppletInterface} from "../types" +// Internal btsdk surface (RestComms needs updateBluetoothSettings, not on the +// public entry). Relative path — `@mentra/bluetooth-sdk-internal` is a mobile-app +// tsconfig alias that doesn't resolve in island's standalone build. Same path +// island's index.ts uses, one level deeper from services/. +import BluetoothSdk, {type PhotoResponseEvent} from "../../../bluetooth-sdk/build/_internal" +import {SETTINGS, useSettingsStore} from "../stores/settings" +import {useConnectionStore} from "../stores/connection" +import {WebSocketStatus} from "../stores/connection" +import GlobalEventEmitter from "../utils/GlobalEventEmitter" +import {BgTimer} from "../utils/timers" + +interface RequestConfig { + method: "GET" | "POST" | "DELETE" + endpoint: string + data?: any + params?: any + requiresAuth?: boolean +} + +class RestComms { + private static instance: RestComms + private readonly TAG = "RestComms" + private coreToken: string | null = null + private axiosInstance: AxiosInstance + + private constructor() { + this.axiosInstance = axios.create({ + headers: { + "Content-Type": "application/json", + }, + }) + } + + public static getInstance(): RestComms { + if (!RestComms.instance) { + RestComms.instance = new RestComms() + } + return RestComms.instance + } + + // Token Management + public setCoreToken(token: string | null): void { + this.coreToken = token + const tokenLen = token?.length ?? 0 + console.log( + `${this.TAG}: Core token ${token ? "set" : "cleared"} - Length: ${tokenLen} - First 20 chars: ${ + token?.substring(0, 20) || "null" + }`, + ) + + // Sync to native DeviceStore (and persist to SharedPreferences in BluetoothSdkModule when bridge runs) + const value = token ?? "" + const updateResult = BluetoothSdk.updateBluetoothSettings({core_token: value}) + if (updateResult != null && typeof (updateResult as Promise).then === "function") { + ;(updateResult as Promise).catch(() => {}) + } + + if (token) { + console.log(`${this.TAG}: Core token set, emitting CORE_TOKEN_SET event`) + GlobalEventEmitter.emit("CORE_TOKEN_SET") + } + } + + public getCoreToken(): string | null { + return this.coreToken + } + + // Helper Methods + private validateToken(): Result { + if (!this.coreToken) { + return Res.error(new Error("No core token available for authentication")) + } + return Res.ok(undefined) + } + + private createAuthHeaders(): Record { + return { + "Content-Type": "application/json", + "Authorization": `Bearer ${this.coreToken}`, + } + } + + private makeRequest(config: RequestConfig): AsyncResult { + const {method, endpoint, data, params, requiresAuth = true} = config + + const baseUrl = useSettingsStore.getState().getRestUrl() + const url = `${baseUrl}${endpoint}` + // console.log(`REST: ${method}:${url}`) + + const headers = requiresAuth ? this.createAuthHeaders() : {"Content-Type": "application/json"} + + const axiosConfig: AxiosRequestConfig = { + method, + url, + headers, + data, + params, + } + + return Res.try_async(async () => { + try { + const res = await this.axiosInstance.request(axiosConfig) + return res.data + } catch (error) { + if (!this.isNoActiveSessionError(error)) { + throw error + } + + // Cloud pod has no session for this user (we reconnected to a different + // pod, or the prior session was cleaned up). Trigger a WS reconnect, + // wait for it to land, then retry the request exactly once. + // + // Subscribe BEFORE emitting so we don't miss the DISCONNECTED → CONNECTED + // transition triggered by handleNoActiveSession → reconnectNow. + const waitPromise = this.waitForNextConnected(8_000) + GlobalEventEmitter.emit("NO_ACTIVE_SESSION") + try { + await waitPromise + } catch (waitErr) { + console.log(`${this.TAG}: Retry skipped — WS didn't reconnect in time:`, waitErr) + throw error + } + + const retryHeaders = requiresAuth ? this.createAuthHeaders() : {"Content-Type": "application/json"} + const retryRes = await this.axiosInstance.request({...axiosConfig, headers: retryHeaders}) + return retryRes.data + } + }) + } + + /** + * Resolves on the NEXT CONNECTED transition of the WS (or rejects after + * timeoutMs). Does NOT short-circuit when already CONNECTED — callers + * invoke this after a 503 NO_ACTIVE_SESSION when we know the current + * connection is landing on the wrong pod; we need to wait for the + * post-reconnect CONNECTED event, not the current one. + * + * Uses the connection store directly rather than WebSocketManager to avoid + * a circular import (WebSocketManager → RestComms → WebSocketManager). + */ + private waitForNextConnected(timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + let settled = false + const timer = BgTimer.setTimeout(() => { + if (settled) return + settled = true + unsub() + reject(new Error(`waitForNextConnected timed out after ${timeoutMs}ms`)) + }, timeoutMs) + + let sawNonConnected = useConnectionStore.getState().status !== WebSocketStatus.CONNECTED + const unsub = useConnectionStore.subscribe((state) => { + if (settled) return + if (state.status !== WebSocketStatus.CONNECTED) { + sawNonConnected = true + return + } + // Only resolve on a CONNECTED transition that follows a non-CONNECTED + // state. This guarantees we waited for a real reconnect rather than + // resolving on the stale pre-reconnect CONNECTED state. + if (sawNonConnected) { + settled = true + BgTimer.clearTimeout(timer) + unsub() + resolve() + } + }) + }) + } + + private isNoActiveSessionError(error: unknown): boolean { + if (!axios.isAxiosError(error)) { + return false + } + + return error.response?.status === 503 && error.response?.data?.error === "NO_ACTIVE_SESSION" + } + + private authenticatedRequest(config: RequestConfig): AsyncResult { + let res = this.validateToken() + if (res.is_error()) { + return Res.error_async(res.error) + } + return this.makeRequest({...config}) + } + + private unauthenticatedRequest(config: RequestConfig): AsyncResult { + return this.makeRequest({...config, requiresAuth: false}) + } + + // Public API Methods + + public getMinimumClientVersion(): AsyncResult<{required: string; recommended: string}, Error> { + interface Response { + success: boolean + data: {required: string; recommended: string} + } + const config: RequestConfig = { + method: "GET", + endpoint: "/api/client/min-version", + } + const res = this.unauthenticatedRequest(config) + return res.map((response) => response.data) + } + + public checkAppHealthStatus(packageName: string): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/app-uptime/app-pkg-health-check", + data: {packageName}, + } + + interface Response { + success: boolean + } + + const res = this.authenticatedRequest(config) + return res.map((response) => response.success) + } + + public retry(fn: () => AsyncResult, attempts: number, delayMs: number = 0): AsyncResult { + return Res.try_async(async () => { + let lastError: Error | null = null + + for (let i = 0; i < attempts; i++) { + const result: Result = await fn() + if (result.is_ok()) { + return result.value + } + lastError = result.error + if (i < attempts - 1 && delayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } + } + throw lastError + }) + } + + public getApplets(): AsyncResult { + interface Response { + success: boolean + data: AppletInterface[] + } + const config: RequestConfig = { + method: "GET", + endpoint: "/api/client/apps", + } + let res = this.authenticatedRequest(config) + let data = res.map((response) => response.data) + return data + } + + public startApp(packageName: string): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/apps/${packageName}/start`, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public stopApp(packageName: string): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/apps/${packageName}/stop`, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public uninstallApp(packageName: string): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/api/apps/uninstall/${packageName}`, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + // App Settings + public getAppSettings(appName: string): AsyncResult { + const config: RequestConfig = { + method: "GET", + endpoint: `/appsettings/${appName}`, + } + const res = this.authenticatedRequest(config) + return res + } + + public updateAppSetting(appName: string, update: {key: string; value: any}): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/appsettings/${appName}`, + data: update, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map((response) => response.data) + } + + public updateGlassesState(state: Record): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/device/state", + data: state, + } + interface Response { + success: boolean + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public exchangeToken(token: string): AsyncResult { + const isChina: string = useSettingsStore.getState().getSetting(SETTINGS.china_deployment.key) + + const config: RequestConfig = { + method: "POST", + endpoint: "/auth/exchange-token", + data: { + supabaseToken: !isChina ? token : undefined, + authingToken: isChina ? token : undefined, + }, + } + interface Response { + coreToken: string + } + let res = this.makeRequest(config) + const coreTokenResult: AsyncResult = res.map((response) => response.coreToken) + + // set the core token in the store: + return coreTokenResult.and_then((coreToken: string) => { + this.setCoreToken(coreToken) + return Res.ok(coreToken) + }) + } + + public generateWebviewToken( + packageName: string, + endpoint: string = "generate-webview-token", + ): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/api/auth/${endpoint}`, + data: {packageName}, + } + interface Response { + token: string + } + const res = this.authenticatedRequest(config) + return res.map((response) => response.token) + } + + public hashWithApiKey(stringToHash: string, packageName: string): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/auth/hash-with-api-key", + data: {stringToHash, packageName}, + } + interface Response { + hash: string + } + const res = this.authenticatedRequest(config) + return res.map((response) => response.hash) + } + + // Account Management + public requestAccountDeletion(): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/account/request-deletion", + } + interface Response { + success: boolean + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public confirmAccountDeletion(requestId: string, confirmationCode: string): AsyncResult { + const config: RequestConfig = { + method: "DELETE", + endpoint: "/api/account/confirm-deletion", + data: {requestId, confirmationCode}, + } + interface Response { + success: boolean + } + const res = this.authenticatedRequest(config) + return res + } + + public getLivekitUrlAndToken(): AsyncResult<{url: string; token: string}, Error> { + const config: RequestConfig = { + method: "GET", + endpoint: "/api/client/livekit/token", + } + interface Response { + // url: string + // token: string + success: boolean + data: {url: string; token: string} + } + const res = this.authenticatedRequest(config) + + // ;(async () => { + // console.log("result@@@@@", await result) + // // const response = await Res.value + // // return {url: response.url, token: response.token} + // })() + + return res.map((response) => response.data) + } + + // User Feedback & Incidents + + /** + * Create a new incident for a bug report. + * Returns incidentId for subsequent log/attachment uploads. + */ + public createIncident( + feedback: object, + phoneState?: Record, + ): AsyncResult<{success: boolean; incidentId: string}, Error> { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/incidents", + data: { + feedback, + ...(phoneState && {phoneState}), + }, + } + interface Response { + success: boolean + incidentId: string + } + return this.authenticatedRequest(config) + } + + /** + * Submit feedback (feature requests only). + * For bug reports, use createIncident instead. + */ + public sendFeedback( + feedbackBody: string | object, + phoneState?: Record, + ): AsyncResult<{success: boolean}, Error> { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/feedback", + data: { + feedback: feedbackBody, + ...(phoneState && {phoneState}), + }, + } + interface Response { + success: boolean + } + return this.authenticatedRequest(config) + } + + /** + * Upload phone logs to an incident. + * Called after createIncident returns an incidentId. + */ + public uploadIncidentLogs( + incidentId: string, + logs: Array<{timestamp: number; level: string; message: string; source?: string}>, + ): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: `/api/incidents/${incidentId}/logs`, + data: { + source: "phone", + logs, + }, + } + interface Response { + success: boolean + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + /** + * Upload screenshot attachments to an incident. + * Called after createIncident returns an incidentId. + */ + public uploadIncidentAttachments( + incidentId: string, + images: Array<{uri: string; fileName?: string | null; mimeType?: string | null}>, + ): AsyncResult<{uploaded: number; errors: number}, Error> { + const uploadPromise = async (): Promise> => { + try { + const coreToken = this.getCoreToken() + if (!coreToken) { + return Res.error(new Error("Not authenticated")) + } + + const baseUrl = useSettingsStore.getState().getRestUrl() + const formData = new FormData() + + for (const image of images) { + const filename = image.fileName || `screenshot-${Date.now()}.jpg` + const mimeType = image.mimeType || "image/jpeg" + + // React Native FormData expects this format + formData.append("files", { + uri: image.uri, + name: filename, + type: mimeType, + } as unknown as Blob) + } + + const response = await axios.post(`${baseUrl}/api/incidents/${incidentId}/attachments`, formData, { + headers: { + "Authorization": `Bearer ${coreToken}`, + "Content-Type": "multipart/form-data", + }, + timeout: 60000, // 60 second timeout for uploads + }) + + const data = response.data as { + success: boolean + uploaded?: Array<{filename: string}> + errors?: Array<{filename: string; error: string}> + } + + return Res.ok({ + uploaded: data.uploaded?.length || 0, + errors: data.errors?.length || 0, + }) + } catch (err) { + return Res.error(err instanceof Error ? err : new Error(String(err))) + } + } + + return new AsyncResult(uploadPromise()) + } + + public writeUserSettings(settings: any): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/user/settings", + data: {settings}, + } + interface Response { + success: boolean + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public loadUserSettings(): AsyncResult { + const config: RequestConfig = { + method: "GET", + endpoint: "/api/client/user/settings", + } + interface Response { + success: boolean + data: {settings: Record} + } + const res = this.authenticatedRequest(config) + return res.map((response) => response.data.settings) + } + + // Error Reporting + public sendErrorReport(reportData: any): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/app/error-report", + data: reportData, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + // Calendar + public sendCalendarData(data: any): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/calendar", + data: data, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + // Location + public sendLocationData(data: any): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/location", + data: data, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + // Phone Notifications + public sendPhoneNotification(data: { + notificationId: string + app: string + title: string + content: string + priority: string + timestamp: number + packageName: string + }): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/notifications", + data: data, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public sendPhoneNotificationDismissed(data: { + notificationId: string + notificationKey: string + packageName: string + }): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/notifications/dismissed", + data: data, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public sendPhotoResponse(data: PhotoResponseEvent): AsyncResult { + const response = + data.state === "success" + ? { + type: data.type, + requestId: data.requestId, + photoUrl: data.uploadUrl, + timestamp: data.timestamp, + success: true, + } + : { + type: data.type, + requestId: data.requestId, + timestamp: data.timestamp, + success: false, + errorCode: data.errorCode, + errorMessage: data.errorMessage, + } + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/photo/response", + data: response, + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public goodbye(): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/goodbye", + } + interface Response { + success: boolean + data: any + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } + + public configureAudioFormat( + format: string, + lc3Config: { + sampleRate: number + frameDurationMs: number + frameSizeBytes: number + }, + ): AsyncResult { + const config: RequestConfig = { + method: "POST", + endpoint: "/api/client/audio/configure", + data: {format, lc3Config}, + } + const res = this.authenticatedRequest(config) + return res.map(() => undefined) + } +} + +const restComms = RestComms.getInstance() +export default restComms diff --git a/mobile/modules/island/src/stores/settings.ts b/mobile/modules/island/src/stores/settings.ts new file mode 100644 index 0000000000..1d3730e538 --- /dev/null +++ b/mobile/modules/island/src/stores/settings.ts @@ -0,0 +1,962 @@ +import {getTimeZone} from "react-native-localize" +import {AsyncResult, result as Res, Result} from "typesafe-ts" +import {create} from "zustand" +import {subscribeWithSelector} from "zustand/middleware" + +import restComms from "../services/RestComms" +import {storage} from "../utils/storage" + +interface Setting { + key: string + defaultValue: () => any + writable: boolean + saveOnServer: boolean + // change the key to a different key based on the indexer + // NEVER do any network calls in the indexer (or performance will suffer greatly + indexer?: (key: string) => string + // optionally override the value of the setting when it's accessed + override?: () => any + // onWrite?: () => void + persist: boolean +} + +export const SETTINGS: Record = { + // feature flags / mantle settings: + dev_mode: {key: "dev_mode", defaultValue: () => __DEV__, writable: true, saveOnServer: true, persist: true},// deprecated + debug_mode: {key: "debug_mode", defaultValue: () => __DEV__, writable: true, saveOnServer: true, persist: true}, + super_mode: {key: "super_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, + miniapp_dev_mode: {key: "miniapp_dev_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, + enable_squircles: { + key: "enable_squircles", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + android_blur: { + key: "android_blur", + defaultValue: () => { + return false + }, + writable: true, + saveOnServer: true, + persist: true, + }, + android_inner_shadow: { + key: "android_inner_shadow", + defaultValue: () => { + return false + }, + writable: true, + saveOnServer: true, + persist: true, + }, + ios_glass_effect: { + key: "ios_glass_effect", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + ios_app_switcher_bottom_swipe: { + key: "ios_app_switcher_bottom_swipe", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + debug_console: { + key: "debug_console", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + debug_navigation_history: { + key: "debug_navigation_history", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + debug_core_status_bar: { + key: "debug_core_status_bar", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + // Mentra Nex feature flags (off by default; toggled from Nex Developer Settings). + // When on, the Nex display skips ASCII-only text sanitization so CJK/Chinese + // captions render on glasses. Synced to the Bluetooth SDK via BLUETOOTH_SETTING_KEYS. + nex_chinese_captions: { + key: "nex_chinese_captions", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + // When on, LC3 audio received from Nex glasses is played back (Android only). + nex_audio_playback: { + key: "nex_audio_playback", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + china_deployment: { + key: "china_deployment", + defaultValue: () => (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china" ? true : false), + override: () => (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china" ? true : false), + writable: false, + saveOnServer: false, + persist: true, + }, + backend_url: { + key: "backend_url", + defaultValue: () => { + if (process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE) { + return process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE + } + if (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china") { + return "https://api.mentraglass.cn:443" + } + return "https://api.mentra.glass" + }, + // If env var is set, always use it (on every boot) + override: () => process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE, + writable: true, + saveOnServer: false, + persist: true, + }, + store_url: { + key: "store_url", + defaultValue: () => { + if (process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE) { + return process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE + } + if (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china") { + return "https://apps.mentraglass.cn" + } + return "https://apps.mentra.glass" + }, + // If env var is set, always use it (on every boot) + override: () => process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE, + writable: true, + saveOnServer: false, + persist: true, + }, + // Cloud V2 endpoint OVERRIDES. Empty = no override; cloudClient's resolveUrl + // owns the full precedence (override -> env -> Metro-derived dev default). + // The value may be an explicit URL or the METRO_AUTO sentinel ("my dev + // laptop", resolved live from Metro so it survives network changes). Never + // bake an env var or a personal LAN IP into the default here: that makes the + // override branch always-truthy and strands devs on a stale address. + cloud_core_url: { + key: "cloud_core_url", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, + cloud_runtime_url: { + key: "cloud_runtime_url", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, + saved_backend_urls: { + key: "saved_backend_urls", + defaultValue: () => [], + writable: true, + saveOnServer: true, + persist: true, + }, + saved_store_urls: { + key: "saved_store_urls", + defaultValue: () => [], + writable: true, + saveOnServer: true, + persist: true, + }, + // Developer override for the ASG OTA manifest URL. null/empty = no override; + // the normal selection applies (legacy-glasses gate, EXPO_PUBLIC_ASG_OTA_VERSION_URL, + // glasses-reported URL, then production). See getAsgOtaVersionUrl. + ota_version_url: { + key: "ota_version_url", + defaultValue: () => null, + writable: true, + saveOnServer: false, + persist: true, + }, + saved_ota_version_urls: { + key: "saved_ota_version_urls", + defaultValue: () => [], + writable: true, + saveOnServer: false, + persist: true, + }, + reconnect_on_app_foreground: { + key: "reconnect_on_app_foreground", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + location_tier: {key: "location_tier", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, + // state: + core_token: {key: "core_token", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, + auth_email: {key: "auth_email", defaultValue: () => "", writable: true, saveOnServer: false, persist: true}, + pending_wearable: { + key: "pending_wearable", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: false, + }, + default_wearable: { + key: "default_wearable", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + device_name: {key: "device_name", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, + device_address: { + key: "device_address", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + default_controller: { + key: "default_controller", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + pending_controller: { + key: "pending_controller", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, + controller_device_name: { + key: "controller_device_name", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + controller_address: { + key: "controller_address", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + // ui state: + home_background: { + key: "home_background", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, + theme_preference: { + key: "theme_preference", + defaultValue: () => (__DEV__ ? "system" : "light"), + // Force light mode - i mode is not complete yet + // override: () => "light", + writable: true, + saveOnServer: true, + persist: true, + }, + enable_phone_notifications: { + key: "enable_phone_notifications", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + settings_access_count: { + key: "settings_access_count", + defaultValue: () => 0, + writable: true, + saveOnServer: true, + persist: true, + }, + show_advanced_settings: { + key: "show_advanced_settings", + defaultValue: () => false, + writable: true, + saveOnServer: false, + persist: true, + }, + onboarding_completed: { + key: "onboarding_completed", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + onboarding_live_completed: { + key: "onboarding_live_completed", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + onboarding_os_completed: { + key: "onboarding_os_completed", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + has_ever_activated_app: { + key: "has_ever_activated_app", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + + // Bluetooth SDK settings: + sensing_enabled: { + key: "sensing_enabled", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + power_saving_mode: { + key: "power_saving_mode", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + voice_activity_detection_enabled: { + key: "voice_activity_detection_enabled", + defaultValue: () => true, + writable: true, + saveOnServer: false, + persist: false, + }, + always_on_status_bar: { + key: "always_on_status_bar", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + // Legacy cloud/mobile setting name. Locally it maps to glasses-side Voice Activity Detection. + bypass_vad_for_debugging: { + key: "bypass_vad_for_debugging", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + bypass_audio_encoding_for_debugging: { + key: "bypass_audio_encoding_for_debugging", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + metric_system: { + key: "metric_system", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + twelve_hour_time: { + key: "twelve_hour_time", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + enforce_local_transcription: { + key: "enforce_local_transcription", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + // LC3 audio quality setting (frame size in bytes) + // 20 = 16kbps (low bandwidth), 40 = 32kbps (balanced), 60 = 48kbps (high quality) + lc3_frame_size: { + key: "lc3_frame_size", + defaultValue: () => 60, + writable: true, + saveOnServer: false, + persist: true, + }, + preferred_mic: { + key: "preferred_mic", + defaultValue: () => "auto", + writable: true, + indexer: (key: string) => { + const glasses = useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) + if (glasses) { + return `${key}:${glasses}` + } + return key + }, + saveOnServer: true, + persist: true, + }, + screen_disabled: { + key: "screen_disabled", + defaultValue: () => false, + writable: true, + saveOnServer: false, + persist: true, + }, + // glasses settings: + contextual_dashboard: { + key: "contextual_dashboard", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + use_native_dashboard: { + key: "use_native_dashboard", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + head_up_angle: {key: "head_up_angle", defaultValue: () => 45, writable: true, saveOnServer: true, persist: true}, + brightness: {key: "brightness", defaultValue: () => 50, writable: true, saveOnServer: true, persist: true}, + auto_brightness: { + key: "auto_brightness", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + dashboard_height: { + key: "dashboard_height", + defaultValue: () => 4, + writable: true, + saveOnServer: true, + persist: true, + }, + dashboard_depth: { + key: "dashboard_depth", + defaultValue: () => 2, + writable: true, + saveOnServer: true, + persist: true, + }, + menu_apps: { + key: "menu_apps", + defaultValue: () => null, + writable: true, + saveOnServer: true, + persist: true, + }, + calendar_events: { + key: "calendar_events", + defaultValue: () => [], + writable: true, + saveOnServer: false, + persist: false, + }, + // button settings + // Legacy persisted/cloud key; hardware behavior is now controlled by gallery_mode plus capture settings. + button_mode: {key: "button_mode", defaultValue: () => "photo", writable: true, saveOnServer: true, persist: true}, + button_photo_size: { + key: "button_photo_size", + defaultValue: () => "max", + writable: true, + saveOnServer: true, + persist: true, + }, + button_video_settings: { + key: "button_video_settings", + defaultValue: () => ({width: 1920, height: 1080, fps: 30}), + writable: true, + saveOnServer: true, + persist: true, + }, + button_camera_led: { + key: "button_camera_led", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + button_max_recording_time: { + key: "button_max_recording_time", + defaultValue: () => 10, + writable: true, + saveOnServer: true, + persist: true, + }, + camera_fov: { + key: "camera_fov", + defaultValue: () => ({fov: 118, roi_position: 0}), + writable: true, + saveOnServer: true, + persist: true, + }, + media_post_processing: { + key: "media_post_processing", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + + // time zone settings + time_zone: { + key: "time_zone", + defaultValue: () => "", + writable: true, + override: () => { + const override = useSettingsStore.getState().getSetting(SETTINGS.time_zone_override.key) + if (override) { + return override + } + return getTimeZone() + }, + saveOnServer: true, + persist: true, + }, + time_zone_override: { + key: "time_zone_override", + defaultValue: () => "", + writable: true, + saveOnServer: true, + persist: true, + }, + // offline applets + offline_mode: {key: "offline_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, + offline_captions_running: { + key: "offline_captions_running", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + // Runtime flag: coordinator flips this on when cloud STT has failed and fallback is active. + // Native GlassesStore watches it to gate PCM → Sherpa feeding. Not user-facing. + local_stt_fallback_active: { + key: "local_stt_fallback_active", + defaultValue: () => false, + writable: true, + saveOnServer: false, + persist: false, + }, + gallery_mode: {key: "gallery_mode", defaultValue: () => true, writable: true, saveOnServer: true, persist: true}, + gallery_sync_explained: { + key: "gallery_sync_explained", + defaultValue: () => false, + writable: true, + saveOnServer: false, + persist: true, + }, + offline_camera_running: { + key: "offline_camera_running", + defaultValue: () => false, + writable: true, + saveOnServer: false, + persist: true, + }, + // offline translation + offline_translation_running: { + key: "offline_translation_running", + defaultValue: () => false, + writable: true, + saveOnServer: true, + persist: true, + }, + offline_translation_source: { + key: "offline_translation_source", + defaultValue: () => "en", + writable: true, + saveOnServer: true, + persist: true, + }, + offline_translation_target: { + key: "offline_translation_target", + defaultValue: () => "es", + writable: true, + saveOnServer: true, + persist: true, + }, + // button action settings + default_button_action_enabled: { + key: "default_button_action_enabled", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + default_button_action_app: { + key: "default_button_action_app", + defaultValue: () => "com.mentra.camera", + writable: true, + saveOnServer: true, + persist: true, + }, + // notifications + notifications_enabled: { + key: "notifications_enabled", + defaultValue: () => true, + writable: true, + saveOnServer: true, + persist: true, + }, + notifications_blocklist: { + key: "notifications_blocklist", + defaultValue: () => [], + writable: true, + saveOnServer: true, + persist: true, + }, + // Cached required version from server - used to enforce updates even when offline + cached_required_version: { + key: "cached_required_version", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, + // OTA update dismissal - stores the version code user dismissed (not persisted so resets on app restart) + dismissed_ota_version: { + key: "dismissed_ota_version", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: false, + }, + // Contact email for feedback (persisted for Apple private relay users) + contact_email: { + key: "contact_email", + defaultValue: () => "", + writable: true, + saveOnServer: false, + persist: true, + }, +} as const + +export const OFFLINE_APPLETS: string[] = ["com.mentra.livecaptions", "com.mentra.camera"] + +// These settings are automatically synced to the Bluetooth SDK. +// Keep this list hardware-facing; app/UI/cloud-only preferences should stay in JS/Crust. +const BLUETOOTH_SETTING_KEYS: string[] = [ + // Bluetooth settings: + SETTINGS.sensing_enabled.key, + SETTINGS.power_saving_mode.key, + SETTINGS.voice_activity_detection_enabled.key, + SETTINGS.lc3_frame_size.key, + SETTINGS.preferred_mic.key, + SETTINGS.screen_disabled.key, + SETTINGS.auth_email.key, + SETTINGS.core_token.key, + // glasses settings: + SETTINGS.contextual_dashboard.key, + SETTINGS.head_up_angle.key, + SETTINGS.brightness.key, + SETTINGS.auto_brightness.key, + SETTINGS.dashboard_height.key, + SETTINGS.dashboard_depth.key, + SETTINGS.menu_apps.key, + SETTINGS.calendar_events.key, + SETTINGS.use_native_dashboard.key, + SETTINGS.twelve_hour_time.key, + SETTINGS.metric_system.key, + // button: + SETTINGS.button_photo_size.key, + // Legacy MentraLive native code reads the object form when syncing video settings. + SETTINGS.button_video_settings.key, + SETTINGS.button_camera_led.key, + SETTINGS.button_max_recording_time.key, + SETTINGS.camera_fov.key, + // device / pairing: + SETTINGS.pending_wearable.key, + SETTINGS.default_wearable.key, + SETTINGS.device_name.key, + SETTINGS.device_address.key, + SETTINGS.default_controller.key, + SETTINGS.pending_controller.key, + SETTINGS.controller_device_name.key, + SETTINGS.controller_address.key, + // offline applets: + SETTINGS.offline_mode.key, + SETTINGS.offline_captions_running.key, + // Runtime flag flipped by LocalSttFallbackCoordinator. Native reads it from + // GlassesStore to gate PCM → Sherpa feeding in handlePcm and to keep the + // mic on while local STT is the active engine. + SETTINGS.local_stt_fallback_active.key, + SETTINGS.gallery_mode.key, + // Mentra Nex feature flags: + SETTINGS.nex_chinese_captions.key, + SETTINGS.nex_audio_playback.key, +] + +// const PER_GLASSES_SETTINGS_KEYS: string[] = [SETTINGS.preferred_mic.key] + +interface SettingsState { + // Settings values + settings: Record + // Loading states + isInitialized: boolean + // Actions + setSetting: (key: string, value: any, updateServer?: boolean) => AsyncResult + setManyLocally: (settings: Record) => AsyncResult + getSetting: (key: string) => any + // loadSetting: (key: string) => AsyncResult + loadAllSettings: () => AsyncResult + // Utility methods + getRestUrl: () => string + getWsUrl: () => string + getBluetoothSettings: () => Record + resetAllSettingsLocally: () => void +} + +const getDefaultSettings = () => + Object.keys(SETTINGS).reduce((acc, key) => { + acc[key] = SETTINGS[key].defaultValue() + return acc + }, {} as Record) + +export const useSettingsStore = create()( + subscribeWithSelector((set, get) => ({ + settings: getDefaultSettings(), + isInitialized: false, + loadingKeys: new Set(), + setSetting: (key: string, value: any, updateServer = true): AsyncResult => { + return Res.try_async(async () => { + const setting = SETTINGS[key] + const originalKey = key + + if (!setting) { + throw new Error(`SETTINGS: SET: ${originalKey} is not a valid setting!`) + } + + if (setting.indexer) { + key = setting.indexer(originalKey) + } + + if (!setting.writable) { + throw new Error(`SETTINGS: ${originalKey} is not writable!`) + } + + // Update store immediately for optimistic UI + console.log(`SETTINGS: SET: ${key} = ${value}`) + set((state) => ({ + settings: {...state.settings, [key]: value}, + })) + + if (setting.persist) { + let res = await storage.save(key, value) + if (res.is_error()) { + throw new Error(`SETTINGS: couldn't save setting to storage: ${res.error}`) + } + + // Sync with server if needed + if (updateServer) { + const result = await restComms.writeUserSettings({[key]: value}) + if (result.is_error()) { + throw new Error(`SETTINGS: couldn't sync setting to server: ${result.error}`) + } + } + } + }) + }, + getSetting: (key: string) => { + const state = get() + const originalKey = key + const setting = SETTINGS[originalKey] + + if (!setting) { + console.error(`SETTINGS: GET: ${originalKey} is not a valid setting!`) + return undefined + } + + if (setting.override) { + let override = setting.override() + if (override !== undefined) { + return override + } + } + + if (setting.indexer) { + key = setting.indexer(originalKey) + } + + // console.log(`GET SETTING: ${key} = ${state.settings[key]}`) + + try { + const raw = state.settings[key] ?? SETTINGS[originalKey].defaultValue() + return raw + } catch (e) { + // for dynamically created settings, we need to create a new setting in SETTINGS: + console.log(`Failed to get setting, creating new setting:(${key}):`, e) + SETTINGS[key] = {key: key, defaultValue: () => undefined, writable: true, saveOnServer: false, persist: true} + return SETTINGS[key].defaultValue() + } + }, + // batch update many settings from the server: + setManyLocally: (settings: Record): AsyncResult => { + return Res.try_async(async () => { + const settingsToLoad: Record = {} + // if a setting should not persist, don't load it: + for (const [key, value] of Object.entries(settings)) { + const stg: Setting | undefined = SETTINGS[key.toLowerCase()] + if (!stg) { + continue + } + if (!stg.persist) { + continue + } + settingsToLoad[key.toLowerCase()] = value + } + // console.log("SETTINGS: SET MANY LOCALLY: ", settingsToLoad) + + set((state) => ({ + settings: {...state.settings, ...settingsToLoad}, + })) + + // save to storage: + await Promise.all(Object.entries(settingsToLoad).map(([key, value]) => storage.save(key, value))) + }) + }, + // loads any preferences that have been changed from the default and saved to DISK! + loadAllSettings: (): AsyncResult => { + console.log("SETTINGS: loadAllSettings()") + return Res.try_async(async () => { + const state = get() + let loadedSettings: Record = {} + + if (state.isInitialized) { + return undefined + } + + for (const setting of Object.values(SETTINGS)) { + // if the settings should not persist, don't load it: + if (!setting.persist) { + continue + } + + // load all subkeys for an indexed setting: + if (setting?.indexer) { + console.log(`SETTINGS: LOAD: ${setting.key} with indexer!`) + + let res: Result, Error> = storage.loadSubKeys(setting.key) + if (res.is_error()) { + console.log(`SETTINGS: LOAD: ${setting.key}`, res.error) + continue + } + + let subKeys: Record = res.value + console.log(`SETTINGS: LOAD: ${setting.key} subkeys are set!`, subKeys) + loadedSettings = {...loadedSettings, ...subKeys} + continue + } + + let res = storage.load(setting.key) + if (res.is_error()) { + console.log(`SETTINGS: LOAD: ${setting.key} is not set!`, res.error) + // this setting isn't set from the default, so we don't load anything + continue + } + // normal key:value pair: + let value = res.value + console.log(`SETTINGS: LOAD: ${setting.key} = ${value.value}`) + loadedSettings[setting.key] = value + } + + // console.log("##############################################") + // console.log(loadedSettings) + // console.log("##############################################") + + set((state) => ({ + isInitialized: true, + settings: {...state.settings, ...loadedSettings}, + })) + + // One-time migration: force android_blur=false for existing users. + // The setting's default is already false; this migration covers users + // who explicitly opted into Android blur effects before we discovered + // they're a major source of frame drops on cheap Android phones. + // The dimezisBlurViewSdk31Plus blur each costs ~5-10ms/frame; with + // multiple blurs on home (top fade + AppSwitcherButton x2) a low-end + // device misses the 16ms budget consistently. Users can turn it back + // on under Settings → Appearance once we've optimized further. + // + // The setSetting call also pushes to the server (saveOnServer: true) + // so the server-stored value flips too — otherwise the next sync + // from the user's server-stored prefs would re-enable blur. + // Best-effort: a server failure (offline, 5xx) shouldn't block boot; + // we still mark the migration done locally so we don't loop. + const MIGRATION_KEY = "migration:android_blur_default_false_v1" + const migrationDone = storage.load(MIGRATION_KEY) + if (migrationDone.is_error() || !migrationDone.value) { + const current = get().getSetting(SETTINGS.android_blur.key) + if (current === true) { + const result = await get().setSetting(SETTINGS.android_blur.key, false, true) + if (result.is_error()) { + // Server push failed (offline / 5xx). Local storage was still + // updated, so the user immediately gets the new behavior. The + // server-side stale `true` will be overwritten the next time + // the user opens Appearance settings and the auto-sync runs. + console.log("SETTINGS: android_blur migration server-push failed:", result.error) + } + } + // Mark done unconditionally — even on server-push failure we don't + // want to retry the migration on every boot. The local value is + // already correct. + storage.save(MIGRATION_KEY, true) + } + }) + }, + getRestUrl: () => { + const serverUrl = get().getSetting(SETTINGS.backend_url.key) + // console.log("GET REST URL: serverUrl:", serverUrl) + const url = new URL(serverUrl) + const secure = url.protocol === "https:" + return `${secure ? "https" : "http"}://${url.hostname}:${url.port || (secure ? 443 : 80)}` + }, + getWsUrl: () => { + const serverUrl = get().getSetting(SETTINGS.backend_url.key) + const url = new URL(serverUrl) + const secure = url.protocol === "https:" + return `${secure ? "wss" : "ws"}://${url.hostname}:${url.port || (secure ? 443 : 80)}/glasses-ws` + }, + getBluetoothSettings: () => { + const state = get() + const bluetoothSettings: Record = {} + Object.values(SETTINGS).forEach((setting) => { + if (BLUETOOTH_SETTING_KEYS.includes(setting.key)) { + bluetoothSettings[setting.key] = state.getSetting(setting.key) + } + }) + return bluetoothSettings + }, + resetAllSettingsLocally: () => { + set((_state) => ({ + settings: getDefaultSettings(), + isInitialized: true, + })) + }, + })), +) + +export const useSetting = (key: string): [T, (value: T) => AsyncResult] => { + const value = useSettingsStore((state) => state.getSetting(key)) + const setSetting = useSettingsStore((state) => state.setSetting) + return [value, (newValue: T) => setSetting(key, newValue)] +} diff --git a/mobile/modules/island/src/utils/GlobalEventEmitter.ts b/mobile/modules/island/src/utils/GlobalEventEmitter.ts index 4452a46937..ede75c7830 100644 --- a/mobile/modules/island/src/utils/GlobalEventEmitter.ts +++ b/mobile/modules/island/src/utils/GlobalEventEmitter.ts @@ -1,3 +1,5 @@ +// @ts-nocheck — `events` (node builtin, RN-polyfilled) ships no types in island's +// standalone build; the host build resolves them fine. import {EventEmitter} from "events" /** diff --git a/mobile/src/services/RestComms.ts b/mobile/src/services/RestComms.ts index 4e884edf90..ce705fee49 100644 --- a/mobile/src/services/RestComms.ts +++ b/mobile/src/services/RestComms.ts @@ -1,731 +1,6 @@ -import {AppletInterface} from "@/../../cloud/packages/types/src" -import axios, {AxiosInstance, AxiosRequestConfig} from "axios" -import {AsyncResult, Result, result as Res} from "typesafe-ts" - -import BluetoothSdk, {PhotoResponseEvent} from "@mentra/bluetooth-sdk-internal" -import {SETTINGS, useSettingsStore} from "@/stores/settings" -import {useConnectionStore} from "@/stores/connection" -import {WebSocketStatus} from "@/services/ws-types" -import GlobalEventEmitter from "@/utils/GlobalEventEmitter" -import {BgTimer} from "@mentra/island" - -interface RequestConfig { - method: "GET" | "POST" | "DELETE" - endpoint: string - data?: any - params?: any - requiresAuth?: boolean -} - -class RestComms { - private static instance: RestComms - private readonly TAG = "RestComms" - private coreToken: string | null = null - private axiosInstance: AxiosInstance - - private constructor() { - this.axiosInstance = axios.create({ - headers: { - "Content-Type": "application/json", - }, - }) - } - - public static getInstance(): RestComms { - if (!RestComms.instance) { - RestComms.instance = new RestComms() - } - return RestComms.instance - } - - // Token Management - public setCoreToken(token: string | null): void { - this.coreToken = token - const tokenLen = token?.length ?? 0 - console.log( - `${this.TAG}: Core token ${token ? "set" : "cleared"} - Length: ${tokenLen} - First 20 chars: ${ - token?.substring(0, 20) || "null" - }`, - ) - - // Sync to native DeviceStore (and persist to SharedPreferences in BluetoothSdkModule when bridge runs) - const value = token ?? "" - const updateResult = BluetoothSdk.updateBluetoothSettings({core_token: value}) - if (updateResult != null && typeof (updateResult as Promise).then === "function") { - ;(updateResult as Promise).catch(() => {}) - } - - if (token) { - console.log(`${this.TAG}: Core token set, emitting CORE_TOKEN_SET event`) - GlobalEventEmitter.emit("CORE_TOKEN_SET") - } - } - - public getCoreToken(): string | null { - return this.coreToken - } - - // Helper Methods - private validateToken(): Result { - if (!this.coreToken) { - return Res.error(new Error("No core token available for authentication")) - } - return Res.ok(undefined) - } - - private createAuthHeaders(): Record { - return { - "Content-Type": "application/json", - "Authorization": `Bearer ${this.coreToken}`, - } - } - - private makeRequest(config: RequestConfig): AsyncResult { - const {method, endpoint, data, params, requiresAuth = true} = config - - const baseUrl = useSettingsStore.getState().getRestUrl() - const url = `${baseUrl}${endpoint}` - // console.log(`REST: ${method}:${url}`) - - const headers = requiresAuth ? this.createAuthHeaders() : {"Content-Type": "application/json"} - - const axiosConfig: AxiosRequestConfig = { - method, - url, - headers, - data, - params, - } - - return Res.try_async(async () => { - try { - const res = await this.axiosInstance.request(axiosConfig) - return res.data - } catch (error) { - if (!this.isNoActiveSessionError(error)) { - throw error - } - - // Cloud pod has no session for this user (we reconnected to a different - // pod, or the prior session was cleaned up). Trigger a WS reconnect, - // wait for it to land, then retry the request exactly once. - // - // Subscribe BEFORE emitting so we don't miss the DISCONNECTED → CONNECTED - // transition triggered by handleNoActiveSession → reconnectNow. - const waitPromise = this.waitForNextConnected(8_000) - GlobalEventEmitter.emit("NO_ACTIVE_SESSION") - try { - await waitPromise - } catch (waitErr) { - console.log(`${this.TAG}: Retry skipped — WS didn't reconnect in time:`, waitErr) - throw error - } - - const retryHeaders = requiresAuth ? this.createAuthHeaders() : {"Content-Type": "application/json"} - const retryRes = await this.axiosInstance.request({...axiosConfig, headers: retryHeaders}) - return retryRes.data - } - }) - } - - /** - * Resolves on the NEXT CONNECTED transition of the WS (or rejects after - * timeoutMs). Does NOT short-circuit when already CONNECTED — callers - * invoke this after a 503 NO_ACTIVE_SESSION when we know the current - * connection is landing on the wrong pod; we need to wait for the - * post-reconnect CONNECTED event, not the current one. - * - * Uses the connection store directly rather than WebSocketManager to avoid - * a circular import (WebSocketManager → RestComms → WebSocketManager). - */ - private waitForNextConnected(timeoutMs: number): Promise { - return new Promise((resolve, reject) => { - let settled = false - const timer = BgTimer.setTimeout(() => { - if (settled) return - settled = true - unsub() - reject(new Error(`waitForNextConnected timed out after ${timeoutMs}ms`)) - }, timeoutMs) - - let sawNonConnected = useConnectionStore.getState().status !== WebSocketStatus.CONNECTED - const unsub = useConnectionStore.subscribe((state) => { - if (settled) return - if (state.status !== WebSocketStatus.CONNECTED) { - sawNonConnected = true - return - } - // Only resolve on a CONNECTED transition that follows a non-CONNECTED - // state. This guarantees we waited for a real reconnect rather than - // resolving on the stale pre-reconnect CONNECTED state. - if (sawNonConnected) { - settled = true - BgTimer.clearTimeout(timer) - unsub() - resolve() - } - }) - }) - } - - private isNoActiveSessionError(error: unknown): boolean { - if (!axios.isAxiosError(error)) { - return false - } - - return error.response?.status === 503 && error.response?.data?.error === "NO_ACTIVE_SESSION" - } - - private authenticatedRequest(config: RequestConfig): AsyncResult { - let res = this.validateToken() - if (res.is_error()) { - return Res.error_async(res.error) - } - return this.makeRequest({...config}) - } - - private unauthenticatedRequest(config: RequestConfig): AsyncResult { - return this.makeRequest({...config, requiresAuth: false}) - } - - // Public API Methods - - public getMinimumClientVersion(): AsyncResult<{required: string; recommended: string}, Error> { - interface Response { - success: boolean - data: {required: string; recommended: string} - } - const config: RequestConfig = { - method: "GET", - endpoint: "/api/client/min-version", - } - const res = this.unauthenticatedRequest(config) - return res.map((response) => response.data) - } - - public checkAppHealthStatus(packageName: string): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/app-uptime/app-pkg-health-check", - data: {packageName}, - } - - interface Response { - success: boolean - } - - const res = this.authenticatedRequest(config) - return res.map((response) => response.success) - } - - public retry(fn: () => AsyncResult, attempts: number, delayMs: number = 0): AsyncResult { - return Res.try_async(async () => { - let lastError: Error | null = null - - for (let i = 0; i < attempts; i++) { - const result: Result = await fn() - if (result.is_ok()) { - return result.value - } - lastError = result.error - if (i < attempts - 1 && delayMs > 0) { - await new Promise((resolve) => setTimeout(resolve, delayMs)) - } - } - throw lastError - }) - } - - public getApplets(): AsyncResult { - interface Response { - success: boolean - data: AppletInterface[] - } - const config: RequestConfig = { - method: "GET", - endpoint: "/api/client/apps", - } - let res = this.authenticatedRequest(config) - let data = res.map((response) => response.data) - return data - } - - public startApp(packageName: string): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/apps/${packageName}/start`, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public stopApp(packageName: string): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/apps/${packageName}/stop`, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public uninstallApp(packageName: string): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/api/apps/uninstall/${packageName}`, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - // App Settings - public getAppSettings(appName: string): AsyncResult { - const config: RequestConfig = { - method: "GET", - endpoint: `/appsettings/${appName}`, - } - const res = this.authenticatedRequest(config) - return res - } - - public updateAppSetting(appName: string, update: {key: string; value: any}): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/appsettings/${appName}`, - data: update, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map((response) => response.data) - } - - public updateGlassesState(state: Record): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/device/state", - data: state, - } - interface Response { - success: boolean - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public exchangeToken(token: string): AsyncResult { - const isChina: string = useSettingsStore.getState().getSetting(SETTINGS.china_deployment.key) - - const config: RequestConfig = { - method: "POST", - endpoint: "/auth/exchange-token", - data: { - supabaseToken: !isChina ? token : undefined, - authingToken: isChina ? token : undefined, - }, - } - interface Response { - coreToken: string - } - let res = this.makeRequest(config) - const coreTokenResult: AsyncResult = res.map((response) => response.coreToken) - - // set the core token in the store: - return coreTokenResult.and_then((coreToken: string) => { - this.setCoreToken(coreToken) - return Res.ok(coreToken) - }) - } - - public generateWebviewToken( - packageName: string, - endpoint: string = "generate-webview-token", - ): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/api/auth/${endpoint}`, - data: {packageName}, - } - interface Response { - token: string - } - const res = this.authenticatedRequest(config) - return res.map((response) => response.token) - } - - public hashWithApiKey(stringToHash: string, packageName: string): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/auth/hash-with-api-key", - data: {stringToHash, packageName}, - } - interface Response { - hash: string - } - const res = this.authenticatedRequest(config) - return res.map((response) => response.hash) - } - - // Account Management - public requestAccountDeletion(): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/account/request-deletion", - } - interface Response { - success: boolean - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public confirmAccountDeletion(requestId: string, confirmationCode: string): AsyncResult { - const config: RequestConfig = { - method: "DELETE", - endpoint: "/api/account/confirm-deletion", - data: {requestId, confirmationCode}, - } - interface Response { - success: boolean - } - const res = this.authenticatedRequest(config) - return res - } - - public getLivekitUrlAndToken(): AsyncResult<{url: string; token: string}, Error> { - const config: RequestConfig = { - method: "GET", - endpoint: "/api/client/livekit/token", - } - interface Response { - // url: string - // token: string - success: boolean - data: {url: string; token: string} - } - const res = this.authenticatedRequest(config) - - // ;(async () => { - // console.log("result@@@@@", await result) - // // const response = await Res.value - // // return {url: response.url, token: response.token} - // })() - - return res.map((response) => response.data) - } - - // User Feedback & Incidents - - /** - * Create a new incident for a bug report. - * Returns incidentId for subsequent log/attachment uploads. - */ - public createIncident( - feedback: object, - phoneState?: Record, - ): AsyncResult<{success: boolean; incidentId: string}, Error> { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/incidents", - data: { - feedback, - ...(phoneState && {phoneState}), - }, - } - interface Response { - success: boolean - incidentId: string - } - return this.authenticatedRequest(config) - } - - /** - * Submit feedback (feature requests only). - * For bug reports, use createIncident instead. - */ - public sendFeedback( - feedbackBody: string | object, - phoneState?: Record, - ): AsyncResult<{success: boolean}, Error> { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/feedback", - data: { - feedback: feedbackBody, - ...(phoneState && {phoneState}), - }, - } - interface Response { - success: boolean - } - return this.authenticatedRequest(config) - } - - /** - * Upload phone logs to an incident. - * Called after createIncident returns an incidentId. - */ - public uploadIncidentLogs( - incidentId: string, - logs: Array<{timestamp: number; level: string; message: string; source?: string}>, - ): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: `/api/incidents/${incidentId}/logs`, - data: { - source: "phone", - logs, - }, - } - interface Response { - success: boolean - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - /** - * Upload screenshot attachments to an incident. - * Called after createIncident returns an incidentId. - */ - public uploadIncidentAttachments( - incidentId: string, - images: Array<{uri: string; fileName?: string | null; mimeType?: string | null}>, - ): AsyncResult<{uploaded: number; errors: number}, Error> { - const uploadPromise = async (): Promise> => { - try { - const coreToken = this.getCoreToken() - if (!coreToken) { - return Res.error(new Error("Not authenticated")) - } - - const baseUrl = useSettingsStore.getState().getRestUrl() - const formData = new FormData() - - for (const image of images) { - const filename = image.fileName || `screenshot-${Date.now()}.jpg` - const mimeType = image.mimeType || "image/jpeg" - - // React Native FormData expects this format - formData.append("files", { - uri: image.uri, - name: filename, - type: mimeType, - } as unknown as Blob) - } - - const response = await axios.post(`${baseUrl}/api/incidents/${incidentId}/attachments`, formData, { - headers: { - "Authorization": `Bearer ${coreToken}`, - "Content-Type": "multipart/form-data", - }, - timeout: 60000, // 60 second timeout for uploads - }) - - const data = response.data as { - success: boolean - uploaded?: Array<{filename: string}> - errors?: Array<{filename: string; error: string}> - } - - return Res.ok({ - uploaded: data.uploaded?.length || 0, - errors: data.errors?.length || 0, - }) - } catch (err) { - return Res.error(err instanceof Error ? err : new Error(String(err))) - } - } - - return new AsyncResult(uploadPromise()) - } - - public writeUserSettings(settings: any): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/user/settings", - data: {settings}, - } - interface Response { - success: boolean - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public loadUserSettings(): AsyncResult { - const config: RequestConfig = { - method: "GET", - endpoint: "/api/client/user/settings", - } - interface Response { - success: boolean - data: {settings: Record} - } - const res = this.authenticatedRequest(config) - return res.map((response) => response.data.settings) - } - - // Error Reporting - public sendErrorReport(reportData: any): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/app/error-report", - data: reportData, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - // Calendar - public sendCalendarData(data: any): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/calendar", - data: data, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - // Location - public sendLocationData(data: any): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/location", - data: data, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - // Phone Notifications - public sendPhoneNotification(data: { - notificationId: string - app: string - title: string - content: string - priority: string - timestamp: number - packageName: string - }): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/notifications", - data: data, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public sendPhoneNotificationDismissed(data: { - notificationId: string - notificationKey: string - packageName: string - }): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/notifications/dismissed", - data: data, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public sendPhotoResponse(data: PhotoResponseEvent): AsyncResult { - const response = - data.state === "success" - ? { - type: data.type, - requestId: data.requestId, - photoUrl: data.uploadUrl, - timestamp: data.timestamp, - success: true, - } - : { - type: data.type, - requestId: data.requestId, - timestamp: data.timestamp, - success: false, - errorCode: data.errorCode, - errorMessage: data.errorMessage, - } - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/photo/response", - data: response, - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public goodbye(): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/goodbye", - } - interface Response { - success: boolean - data: any - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } - - public configureAudioFormat( - format: string, - lc3Config: { - sampleRate: number - frameDurationMs: number - frameSizeBytes: number - }, - ): AsyncResult { - const config: RequestConfig = { - method: "POST", - endpoint: "/api/client/audio/configure", - data: {format, lc3Config}, - } - const res = this.authenticatedRequest(config) - return res.map(() => undefined) - } -} - -const restComms = RestComms.getInstance() -export default restComms +/** + * @fileoverview Re-export shim — RestComms now lives in @mentra/island (moved + * together with the settings store, its mutually-coupled pair). Kept so existing + * `@/services/RestComms` default imports resolve unchanged. + */ +export {restComms as default} from "@mentra/island" diff --git a/mobile/src/stores/settings.ts b/mobile/src/stores/settings.ts index f4d7c33582..3d0989e990 100644 --- a/mobile/src/stores/settings.ts +++ b/mobile/src/stores/settings.ts @@ -1,964 +1,6 @@ -import {Platform} from "react-native" -import {getTimeZone} from "react-native-localize" -import {AsyncResult, result as Res, Result} from "typesafe-ts" -import {create} from "zustand" -import {subscribeWithSelector} from "zustand/middleware" -import * as Device from "expo-device" - -import restComms from "@/services/RestComms" -import {storage} from "@/utils/storage" - -interface Setting { - key: string - defaultValue: () => any - writable: boolean - saveOnServer: boolean - // change the key to a different key based on the indexer - // NEVER do any network calls in the indexer (or performance will suffer greatly - indexer?: (key: string) => string - // optionally override the value of the setting when it's accessed - override?: () => any - // onWrite?: () => void - persist: boolean -} - -export const SETTINGS: Record = { - // feature flags / mantle settings: - dev_mode: {key: "dev_mode", defaultValue: () => __DEV__, writable: true, saveOnServer: true, persist: true},// deprecated - debug_mode: {key: "debug_mode", defaultValue: () => __DEV__, writable: true, saveOnServer: true, persist: true}, - super_mode: {key: "super_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, - miniapp_dev_mode: {key: "miniapp_dev_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, - enable_squircles: { - key: "enable_squircles", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - android_blur: { - key: "android_blur", - defaultValue: () => { - return false - }, - writable: true, - saveOnServer: true, - persist: true, - }, - android_inner_shadow: { - key: "android_inner_shadow", - defaultValue: () => { - return false - }, - writable: true, - saveOnServer: true, - persist: true, - }, - ios_glass_effect: { - key: "ios_glass_effect", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - ios_app_switcher_bottom_swipe: { - key: "ios_app_switcher_bottom_swipe", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - debug_console: { - key: "debug_console", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - debug_navigation_history: { - key: "debug_navigation_history", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - debug_core_status_bar: { - key: "debug_core_status_bar", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - // Mentra Nex feature flags (off by default; toggled from Nex Developer Settings). - // When on, the Nex display skips ASCII-only text sanitization so CJK/Chinese - // captions render on glasses. Synced to the Bluetooth SDK via BLUETOOTH_SETTING_KEYS. - nex_chinese_captions: { - key: "nex_chinese_captions", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - // When on, LC3 audio received from Nex glasses is played back (Android only). - nex_audio_playback: { - key: "nex_audio_playback", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - china_deployment: { - key: "china_deployment", - defaultValue: () => (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china" ? true : false), - override: () => (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china" ? true : false), - writable: false, - saveOnServer: false, - persist: true, - }, - backend_url: { - key: "backend_url", - defaultValue: () => { - if (process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE) { - return process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE - } - if (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china") { - return "https://api.mentraglass.cn:443" - } - return "https://api.mentra.glass" - }, - // If env var is set, always use it (on every boot) - override: () => process.env.EXPO_PUBLIC_BACKEND_URL_OVERRIDE, - writable: true, - saveOnServer: false, - persist: true, - }, - store_url: { - key: "store_url", - defaultValue: () => { - if (process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE) { - return process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE - } - if (process.env.EXPO_PUBLIC_DEPLOYMENT_REGION === "china") { - return "https://apps.mentraglass.cn" - } - return "https://apps.mentra.glass" - }, - // If env var is set, always use it (on every boot) - override: () => process.env.EXPO_PUBLIC_STORE_URL_OVERRIDE, - writable: true, - saveOnServer: false, - persist: true, - }, - // Cloud V2 endpoint OVERRIDES. Empty = no override; cloudClient's resolveUrl - // owns the full precedence (override -> env -> Metro-derived dev default). - // The value may be an explicit URL or the METRO_AUTO sentinel ("my dev - // laptop", resolved live from Metro so it survives network changes). Never - // bake an env var or a personal LAN IP into the default here: that makes the - // override branch always-truthy and strands devs on a stale address. - cloud_core_url: { - key: "cloud_core_url", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, - cloud_runtime_url: { - key: "cloud_runtime_url", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, - saved_backend_urls: { - key: "saved_backend_urls", - defaultValue: () => [], - writable: true, - saveOnServer: true, - persist: true, - }, - saved_store_urls: { - key: "saved_store_urls", - defaultValue: () => [], - writable: true, - saveOnServer: true, - persist: true, - }, - // Developer override for the ASG OTA manifest URL. null/empty = no override; - // the normal selection applies (legacy-glasses gate, EXPO_PUBLIC_ASG_OTA_VERSION_URL, - // glasses-reported URL, then production). See getAsgOtaVersionUrl. - ota_version_url: { - key: "ota_version_url", - defaultValue: () => null, - writable: true, - saveOnServer: false, - persist: true, - }, - saved_ota_version_urls: { - key: "saved_ota_version_urls", - defaultValue: () => [], - writable: true, - saveOnServer: false, - persist: true, - }, - reconnect_on_app_foreground: { - key: "reconnect_on_app_foreground", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - location_tier: {key: "location_tier", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, - // state: - core_token: {key: "core_token", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, - auth_email: {key: "auth_email", defaultValue: () => "", writable: true, saveOnServer: false, persist: true}, - pending_wearable: { - key: "pending_wearable", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: false, - }, - default_wearable: { - key: "default_wearable", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - device_name: {key: "device_name", defaultValue: () => "", writable: true, saveOnServer: true, persist: true}, - device_address: { - key: "device_address", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - default_controller: { - key: "default_controller", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - pending_controller: { - key: "pending_controller", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, - controller_device_name: { - key: "controller_device_name", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - controller_address: { - key: "controller_address", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - // ui state: - home_background: { - key: "home_background", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, - theme_preference: { - key: "theme_preference", - defaultValue: () => (__DEV__ ? "system" : "light"), - // Force light mode - i mode is not complete yet - // override: () => "light", - writable: true, - saveOnServer: true, - persist: true, - }, - enable_phone_notifications: { - key: "enable_phone_notifications", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - settings_access_count: { - key: "settings_access_count", - defaultValue: () => 0, - writable: true, - saveOnServer: true, - persist: true, - }, - show_advanced_settings: { - key: "show_advanced_settings", - defaultValue: () => false, - writable: true, - saveOnServer: false, - persist: true, - }, - onboarding_completed: { - key: "onboarding_completed", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - onboarding_live_completed: { - key: "onboarding_live_completed", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - onboarding_os_completed: { - key: "onboarding_os_completed", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - has_ever_activated_app: { - key: "has_ever_activated_app", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - - // Bluetooth SDK settings: - sensing_enabled: { - key: "sensing_enabled", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - power_saving_mode: { - key: "power_saving_mode", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - voice_activity_detection_enabled: { - key: "voice_activity_detection_enabled", - defaultValue: () => true, - writable: true, - saveOnServer: false, - persist: false, - }, - always_on_status_bar: { - key: "always_on_status_bar", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - // Legacy cloud/mobile setting name. Locally it maps to glasses-side Voice Activity Detection. - bypass_vad_for_debugging: { - key: "bypass_vad_for_debugging", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - bypass_audio_encoding_for_debugging: { - key: "bypass_audio_encoding_for_debugging", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - metric_system: { - key: "metric_system", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - twelve_hour_time: { - key: "twelve_hour_time", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - enforce_local_transcription: { - key: "enforce_local_transcription", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - // LC3 audio quality setting (frame size in bytes) - // 20 = 16kbps (low bandwidth), 40 = 32kbps (balanced), 60 = 48kbps (high quality) - lc3_frame_size: { - key: "lc3_frame_size", - defaultValue: () => 60, - writable: true, - saveOnServer: false, - persist: true, - }, - preferred_mic: { - key: "preferred_mic", - defaultValue: () => "auto", - writable: true, - indexer: (key: string) => { - const glasses = useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key) - if (glasses) { - return `${key}:${glasses}` - } - return key - }, - saveOnServer: true, - persist: true, - }, - screen_disabled: { - key: "screen_disabled", - defaultValue: () => false, - writable: true, - saveOnServer: false, - persist: true, - }, - // glasses settings: - contextual_dashboard: { - key: "contextual_dashboard", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - use_native_dashboard: { - key: "use_native_dashboard", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - head_up_angle: {key: "head_up_angle", defaultValue: () => 45, writable: true, saveOnServer: true, persist: true}, - brightness: {key: "brightness", defaultValue: () => 50, writable: true, saveOnServer: true, persist: true}, - auto_brightness: { - key: "auto_brightness", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - dashboard_height: { - key: "dashboard_height", - defaultValue: () => 4, - writable: true, - saveOnServer: true, - persist: true, - }, - dashboard_depth: { - key: "dashboard_depth", - defaultValue: () => 2, - writable: true, - saveOnServer: true, - persist: true, - }, - menu_apps: { - key: "menu_apps", - defaultValue: () => null, - writable: true, - saveOnServer: true, - persist: true, - }, - calendar_events: { - key: "calendar_events", - defaultValue: () => [], - writable: true, - saveOnServer: false, - persist: false, - }, - // button settings - // Legacy persisted/cloud key; hardware behavior is now controlled by gallery_mode plus capture settings. - button_mode: {key: "button_mode", defaultValue: () => "photo", writable: true, saveOnServer: true, persist: true}, - button_photo_size: { - key: "button_photo_size", - defaultValue: () => "max", - writable: true, - saveOnServer: true, - persist: true, - }, - button_video_settings: { - key: "button_video_settings", - defaultValue: () => ({width: 1920, height: 1080, fps: 30}), - writable: true, - saveOnServer: true, - persist: true, - }, - button_camera_led: { - key: "button_camera_led", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - button_max_recording_time: { - key: "button_max_recording_time", - defaultValue: () => 10, - writable: true, - saveOnServer: true, - persist: true, - }, - camera_fov: { - key: "camera_fov", - defaultValue: () => ({fov: 118, roi_position: 0}), - writable: true, - saveOnServer: true, - persist: true, - }, - media_post_processing: { - key: "media_post_processing", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - - // time zone settings - time_zone: { - key: "time_zone", - defaultValue: () => "", - writable: true, - override: () => { - const override = useSettingsStore.getState().getSetting(SETTINGS.time_zone_override.key) - if (override) { - return override - } - return getTimeZone() - }, - saveOnServer: true, - persist: true, - }, - time_zone_override: { - key: "time_zone_override", - defaultValue: () => "", - writable: true, - saveOnServer: true, - persist: true, - }, - // offline applets - offline_mode: {key: "offline_mode", defaultValue: () => false, writable: true, saveOnServer: true, persist: true}, - offline_captions_running: { - key: "offline_captions_running", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - // Runtime flag: coordinator flips this on when cloud STT has failed and fallback is active. - // Native GlassesStore watches it to gate PCM → Sherpa feeding. Not user-facing. - local_stt_fallback_active: { - key: "local_stt_fallback_active", - defaultValue: () => false, - writable: true, - saveOnServer: false, - persist: false, - }, - gallery_mode: {key: "gallery_mode", defaultValue: () => true, writable: true, saveOnServer: true, persist: true}, - gallery_sync_explained: { - key: "gallery_sync_explained", - defaultValue: () => false, - writable: true, - saveOnServer: false, - persist: true, - }, - offline_camera_running: { - key: "offline_camera_running", - defaultValue: () => false, - writable: true, - saveOnServer: false, - persist: true, - }, - // offline translation - offline_translation_running: { - key: "offline_translation_running", - defaultValue: () => false, - writable: true, - saveOnServer: true, - persist: true, - }, - offline_translation_source: { - key: "offline_translation_source", - defaultValue: () => "en", - writable: true, - saveOnServer: true, - persist: true, - }, - offline_translation_target: { - key: "offline_translation_target", - defaultValue: () => "es", - writable: true, - saveOnServer: true, - persist: true, - }, - // button action settings - default_button_action_enabled: { - key: "default_button_action_enabled", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - default_button_action_app: { - key: "default_button_action_app", - defaultValue: () => "com.mentra.camera", - writable: true, - saveOnServer: true, - persist: true, - }, - // notifications - notifications_enabled: { - key: "notifications_enabled", - defaultValue: () => true, - writable: true, - saveOnServer: true, - persist: true, - }, - notifications_blocklist: { - key: "notifications_blocklist", - defaultValue: () => [], - writable: true, - saveOnServer: true, - persist: true, - }, - // Cached required version from server - used to enforce updates even when offline - cached_required_version: { - key: "cached_required_version", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, - // OTA update dismissal - stores the version code user dismissed (not persisted so resets on app restart) - dismissed_ota_version: { - key: "dismissed_ota_version", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: false, - }, - // Contact email for feedback (persisted for Apple private relay users) - contact_email: { - key: "contact_email", - defaultValue: () => "", - writable: true, - saveOnServer: false, - persist: true, - }, -} as const - -export const OFFLINE_APPLETS: string[] = ["com.mentra.livecaptions", "com.mentra.camera"] - -// These settings are automatically synced to the Bluetooth SDK. -// Keep this list hardware-facing; app/UI/cloud-only preferences should stay in JS/Crust. -const BLUETOOTH_SETTING_KEYS: string[] = [ - // Bluetooth settings: - SETTINGS.sensing_enabled.key, - SETTINGS.power_saving_mode.key, - SETTINGS.voice_activity_detection_enabled.key, - SETTINGS.lc3_frame_size.key, - SETTINGS.preferred_mic.key, - SETTINGS.screen_disabled.key, - SETTINGS.auth_email.key, - SETTINGS.core_token.key, - // glasses settings: - SETTINGS.contextual_dashboard.key, - SETTINGS.head_up_angle.key, - SETTINGS.brightness.key, - SETTINGS.auto_brightness.key, - SETTINGS.dashboard_height.key, - SETTINGS.dashboard_depth.key, - SETTINGS.menu_apps.key, - SETTINGS.calendar_events.key, - SETTINGS.use_native_dashboard.key, - SETTINGS.twelve_hour_time.key, - SETTINGS.metric_system.key, - // button: - SETTINGS.button_photo_size.key, - // Legacy MentraLive native code reads the object form when syncing video settings. - SETTINGS.button_video_settings.key, - SETTINGS.button_camera_led.key, - SETTINGS.button_max_recording_time.key, - SETTINGS.camera_fov.key, - // device / pairing: - SETTINGS.pending_wearable.key, - SETTINGS.default_wearable.key, - SETTINGS.device_name.key, - SETTINGS.device_address.key, - SETTINGS.default_controller.key, - SETTINGS.pending_controller.key, - SETTINGS.controller_device_name.key, - SETTINGS.controller_address.key, - // offline applets: - SETTINGS.offline_mode.key, - SETTINGS.offline_captions_running.key, - // Runtime flag flipped by LocalSttFallbackCoordinator. Native reads it from - // GlassesStore to gate PCM → Sherpa feeding in handlePcm and to keep the - // mic on while local STT is the active engine. - SETTINGS.local_stt_fallback_active.key, - SETTINGS.gallery_mode.key, - // Mentra Nex feature flags: - SETTINGS.nex_chinese_captions.key, - SETTINGS.nex_audio_playback.key, -] - -// const PER_GLASSES_SETTINGS_KEYS: string[] = [SETTINGS.preferred_mic.key] - -interface SettingsState { - // Settings values - settings: Record - // Loading states - isInitialized: boolean - // Actions - setSetting: (key: string, value: any, updateServer?: boolean) => AsyncResult - setManyLocally: (settings: Record) => AsyncResult - getSetting: (key: string) => any - // loadSetting: (key: string) => AsyncResult - loadAllSettings: () => AsyncResult - // Utility methods - getRestUrl: () => string - getWsUrl: () => string - getBluetoothSettings: () => Record - resetAllSettingsLocally: () => void -} - -const getDefaultSettings = () => - Object.keys(SETTINGS).reduce((acc, key) => { - acc[key] = SETTINGS[key].defaultValue() - return acc - }, {} as Record) - -export const useSettingsStore = create()( - subscribeWithSelector((set, get) => ({ - settings: getDefaultSettings(), - isInitialized: false, - loadingKeys: new Set(), - setSetting: (key: string, value: any, updateServer = true): AsyncResult => { - return Res.try_async(async () => { - const setting = SETTINGS[key] - const originalKey = key - - if (!setting) { - throw new Error(`SETTINGS: SET: ${originalKey} is not a valid setting!`) - } - - if (setting.indexer) { - key = setting.indexer(originalKey) - } - - if (!setting.writable) { - throw new Error(`SETTINGS: ${originalKey} is not writable!`) - } - - // Update store immediately for optimistic UI - console.log(`SETTINGS: SET: ${key} = ${value}`) - set((state) => ({ - settings: {...state.settings, [key]: value}, - })) - - if (setting.persist) { - let res = await storage.save(key, value) - if (res.is_error()) { - throw new Error(`SETTINGS: couldn't save setting to storage: ${res.error}`) - } - - // Sync with server if needed - if (updateServer) { - const result = await restComms.writeUserSettings({[key]: value}) - if (result.is_error()) { - throw new Error(`SETTINGS: couldn't sync setting to server: ${result.error}`) - } - } - } - }) - }, - getSetting: (key: string) => { - const state = get() - const originalKey = key - const setting = SETTINGS[originalKey] - - if (!setting) { - console.error(`SETTINGS: GET: ${originalKey} is not a valid setting!`) - return undefined - } - - if (setting.override) { - let override = setting.override() - if (override !== undefined) { - return override - } - } - - if (setting.indexer) { - key = setting.indexer(originalKey) - } - - // console.log(`GET SETTING: ${key} = ${state.settings[key]}`) - - try { - const raw = state.settings[key] ?? SETTINGS[originalKey].defaultValue() - return raw - } catch (e) { - // for dynamically created settings, we need to create a new setting in SETTINGS: - console.log(`Failed to get setting, creating new setting:(${key}):`, e) - SETTINGS[key] = {key: key, defaultValue: () => undefined, writable: true, saveOnServer: false, persist: true} - return SETTINGS[key].defaultValue() - } - }, - // batch update many settings from the server: - setManyLocally: (settings: Record): AsyncResult => { - return Res.try_async(async () => { - const settingsToLoad: Record = {} - // if a setting should not persist, don't load it: - for (const [key, value] of Object.entries(settings)) { - const stg: Setting | undefined = SETTINGS[key.toLowerCase()] - if (!stg) { - continue - } - if (!stg.persist) { - continue - } - settingsToLoad[key.toLowerCase()] = value - } - // console.log("SETTINGS: SET MANY LOCALLY: ", settingsToLoad) - - set((state) => ({ - settings: {...state.settings, ...settingsToLoad}, - })) - - // save to storage: - await Promise.all(Object.entries(settingsToLoad).map(([key, value]) => storage.save(key, value))) - }) - }, - // loads any preferences that have been changed from the default and saved to DISK! - loadAllSettings: (): AsyncResult => { - console.log("SETTINGS: loadAllSettings()") - return Res.try_async(async () => { - const state = get() - let loadedSettings: Record = {} - - if (state.isInitialized) { - return undefined - } - - for (const setting of Object.values(SETTINGS)) { - // if the settings should not persist, don't load it: - if (!setting.persist) { - continue - } - - // load all subkeys for an indexed setting: - if (setting?.indexer) { - console.log(`SETTINGS: LOAD: ${setting.key} with indexer!`) - - let res: Result, Error> = storage.loadSubKeys(setting.key) - if (res.is_error()) { - console.log(`SETTINGS: LOAD: ${setting.key}`, res.error) - continue - } - - let subKeys: Record = res.value - console.log(`SETTINGS: LOAD: ${setting.key} subkeys are set!`, subKeys) - loadedSettings = {...loadedSettings, ...subKeys} - continue - } - - let res = storage.load(setting.key) - if (res.is_error()) { - console.log(`SETTINGS: LOAD: ${setting.key} is not set!`, res.error) - // this setting isn't set from the default, so we don't load anything - continue - } - // normal key:value pair: - let value = res.value - console.log(`SETTINGS: LOAD: ${setting.key} = ${value.value}`) - loadedSettings[setting.key] = value - } - - // console.log("##############################################") - // console.log(loadedSettings) - // console.log("##############################################") - - set((state) => ({ - isInitialized: true, - settings: {...state.settings, ...loadedSettings}, - })) - - // One-time migration: force android_blur=false for existing users. - // The setting's default is already false; this migration covers users - // who explicitly opted into Android blur effects before we discovered - // they're a major source of frame drops on cheap Android phones. - // The dimezisBlurViewSdk31Plus blur each costs ~5-10ms/frame; with - // multiple blurs on home (top fade + AppSwitcherButton x2) a low-end - // device misses the 16ms budget consistently. Users can turn it back - // on under Settings → Appearance once we've optimized further. - // - // The setSetting call also pushes to the server (saveOnServer: true) - // so the server-stored value flips too — otherwise the next sync - // from the user's server-stored prefs would re-enable blur. - // Best-effort: a server failure (offline, 5xx) shouldn't block boot; - // we still mark the migration done locally so we don't loop. - const MIGRATION_KEY = "migration:android_blur_default_false_v1" - const migrationDone = storage.load(MIGRATION_KEY) - if (migrationDone.is_error() || !migrationDone.value) { - const current = get().getSetting(SETTINGS.android_blur.key) - if (current === true) { - const result = await get().setSetting(SETTINGS.android_blur.key, false, true) - if (result.is_error()) { - // Server push failed (offline / 5xx). Local storage was still - // updated, so the user immediately gets the new behavior. The - // server-side stale `true` will be overwritten the next time - // the user opens Appearance settings and the auto-sync runs. - console.log("SETTINGS: android_blur migration server-push failed:", result.error) - } - } - // Mark done unconditionally — even on server-push failure we don't - // want to retry the migration on every boot. The local value is - // already correct. - storage.save(MIGRATION_KEY, true) - } - }) - }, - getRestUrl: () => { - const serverUrl = get().getSetting(SETTINGS.backend_url.key) - // console.log("GET REST URL: serverUrl:", serverUrl) - const url = new URL(serverUrl) - const secure = url.protocol === "https:" - return `${secure ? "https" : "http"}://${url.hostname}:${url.port || (secure ? 443 : 80)}` - }, - getWsUrl: () => { - const serverUrl = get().getSetting(SETTINGS.backend_url.key) - const url = new URL(serverUrl) - const secure = url.protocol === "https:" - return `${secure ? "wss" : "ws"}://${url.hostname}:${url.port || (secure ? 443 : 80)}/glasses-ws` - }, - getBluetoothSettings: () => { - const state = get() - const bluetoothSettings: Record = {} - Object.values(SETTINGS).forEach((setting) => { - if (BLUETOOTH_SETTING_KEYS.includes(setting.key)) { - bluetoothSettings[setting.key] = state.getSetting(setting.key) - } - }) - return bluetoothSettings - }, - resetAllSettingsLocally: () => { - set((_state) => ({ - settings: getDefaultSettings(), - isInitialized: true, - })) - }, - })), -) - -export const useSetting = (key: string): [T, (value: T) => AsyncResult] => { - const value = useSettingsStore((state) => state.getSetting(key)) - const setSetting = useSettingsStore((state) => state.setSetting) - return [value, (newValue: T) => setSetting(key, newValue)] -} +/** + * @fileoverview Re-export shim — the settings store now lives in @mentra/island + * (moved together with RestComms, its mutually-coupled pair). Kept so existing + * `@/stores/settings` imports resolve unchanged. + */ +export {SETTINGS, OFFLINE_APPLETS, useSettingsStore, useSetting} from "@mentra/island" From 89215012b111b38759a6fc9534a1da8a4dfd396a Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 14:51:36 -0700 Subject: [PATCH 19/80] island: export SettingsState so toolkit.stores.settings emits declarations (fix TS4023) toolkit.stores.settings = useSettingsStore is typed with SettingsState, which was not exported; island's standalone build EMITS .d.ts (expo-module-tsc) and can't name an un-exported type -> TS4023. Passed local 'tsc --noEmit' (no emit) but failed CI's build:module. Exporting the interface fixes it (same pattern as the other moved-store State interfaces). Verified with 'bun run build:module' (the emit build) now green. --- mobile/modules/island/src/stores/settings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/modules/island/src/stores/settings.ts b/mobile/modules/island/src/stores/settings.ts index 1d3730e538..fc14cefb85 100644 --- a/mobile/modules/island/src/stores/settings.ts +++ b/mobile/modules/island/src/stores/settings.ts @@ -708,7 +708,7 @@ const BLUETOOTH_SETTING_KEYS: string[] = [ // const PER_GLASSES_SETTINGS_KEYS: string[] = [SETTINGS.preferred_mic.key] -interface SettingsState { +export interface SettingsState { // Settings values settings: Record // Loading states From 3ab1d2e443046156fc25af60f2c496fe012e6aaf Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 15:53:50 -0700 Subject: [PATCH 20/80] island: add toolkit.settings + toolkit.session.account + toolkit.dev facades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Typed (A) facades over logic just moved into island: - toolkit.settings — keyed get/set/onChanged/descriptor/keys over the island settings store (the OEM contract; toolkit.stores.settings stays the raw escape hatch). - toolkit.session.account.{delete,confirmDelete} — adapter-free now that RestComms is in island (corrects the earlier deferral). requestExport omitted (host UI flow, no backend). - toolkit.dev — backend/cloud URL overrides (island settings store), reconnect the island cloud client, minimumClientVersion (island RestComms). Exported Setting + SettingsState interfaces (TS4023: descriptor()/stores.settings emit declarations). Verified with the EMIT build (build:module), not just tsc --noEmit. jest mock extended; full suite (281) green. --- mobile/jest.setup.js | 20 +++++++++ mobile/modules/island/src/facades/dev.ts | 43 +++++++++++++++++++ mobile/modules/island/src/facades/session.ts | 18 ++++++-- mobile/modules/island/src/facades/settings.ts | 27 ++++++++++++ mobile/modules/island/src/island.ts | 8 +++- mobile/modules/island/src/stores/settings.ts | 2 +- 6 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 mobile/modules/island/src/facades/dev.ts create mode 100644 mobile/modules/island/src/facades/settings.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 92b1847b94..07403e5d90 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -334,6 +334,26 @@ jest.mock("@mentra/island", () => { status: jest.fn(() => ({status: "disconnected", audioTransport: "none"})), onStatus: jest.fn(() => () => {}), isConnected: jest.fn(() => false), + account: { + delete: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + confirmDelete: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + }, + }, + settings: { + get: jest.fn(() => undefined), + set: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + onChanged: jest.fn(() => () => {}), + descriptor: jest.fn(() => undefined), + keys: jest.fn(() => []), + }, + dev: { + minimumClientVersion: jest.fn(() => Promise.resolve({is_ok: () => true, value: {required: "0", recommended: "0"}})), + backendUrl: jest.fn(() => undefined), + setBackendUrl: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + cloudUrls: jest.fn(() => ({})), + setCloudUrls: jest.fn(), + savedUrls: jest.fn(() => []), + reconnectCloud: jest.fn(), }, stores: { glasses: realGlasses.useGlassesStore, diff --git a/mobile/modules/island/src/facades/dev.ts b/mobile/modules/island/src/facades/dev.ts new file mode 100644 index 0000000000..333d015886 --- /dev/null +++ b/mobile/modules/island/src/facades/dev.ts @@ -0,0 +1,43 @@ +/** + * dev facade — `toolkit.dev`: developer/debug surface over island-owned state. + * Backend/cloud URL overrides read+write the island settings store; the cloud + * URL setters reconnect the island-owned cloud client onto the new endpoints; + * `minimumClientVersion` hits the island RestComms. + * + * NOTE: the dev-time endpoint *resolution* (Metro-host auto-detect, env defaults) + * still lives host-side in `@/services/cloudClient` during the migration; this + * facade manages the explicit overrides + reconnect. + */ +import {useSettingsStore, SETTINGS} from "../stores/settings" +import {cloudClientService} from "../services/CloudClientService" +import restComms from "../services/RestComms" + +export const dev = { + /** The backend's required/recommended client version. */ + minimumClientVersion: () => restComms.getMinimumClientVersion(), + + /** The configured backend (core REST) URL override, if any. */ + backendUrl: (): string | undefined => useSettingsStore.getState().getSetting(SETTINGS.backend_url.key), + /** Override the backend (core REST) URL. */ + setBackendUrl: (url: string) => useSettingsStore.getState().setSetting(SETTINGS.backend_url.key, url), + + /** The cloud-v2 core/runtime URL overrides. */ + cloudUrls: (): {core?: string; runtime?: string} => { + const s = useSettingsStore.getState() + return {core: s.getSetting(SETTINGS.cloud_core_url.key), runtime: s.getSetting(SETTINGS.cloud_runtime_url.key)} + }, + /** Override the cloud-v2 URLs and reconnect the live client onto them. */ + setCloudUrls: (urls: {core?: string; runtime?: string}) => { + const s = useSettingsStore.getState() + if (urls.core !== undefined) s.setSetting(SETTINGS.cloud_core_url.key, urls.core) + if (urls.runtime !== undefined) s.setSetting(SETTINGS.cloud_runtime_url.key, urls.runtime) + if (urls.core && urls.runtime) cloudClientService.reconnect({core: urls.core, runtime: urls.runtime}) + else cloudClientService.reconnect() + }, + + /** The saved backend URLs (the dev URL-switcher list). */ + savedUrls: (): string[] => useSettingsStore.getState().getSetting(SETTINGS.saved_backend_urls.key) ?? [], + + /** Tear down + rebuild the live cloud client (the dev "reconnect" button). */ + reconnectCloud: (): void => cloudClientService.reconnect(), +} diff --git a/mobile/modules/island/src/facades/session.ts b/mobile/modules/island/src/facades/session.ts index 62313a3fee..9a37d88a84 100644 --- a/mobile/modules/island/src/facades/session.ts +++ b/mobile/modules/island/src/facades/session.ts @@ -4,11 +4,12 @@ * audio transport, projected from the island-owned cloud-status store) and the * liveness flag. * - * Account operations (`account.delete()` / `account.requestExport()` from the - * Phase-1 contract) are still host RestComms calls; they land here when the - * account/identity domain moves into island. The cloud-client `core` surface - * exposes no account methods today. + * `account.*` operations route through the now-island RestComms (the account REST + * calls moved in with the settings+RestComms keystone), so they're adapter-free. + * (`requestExport` is a host UI navigation flow with no backend method yet, so it + * is not surfaced here.) */ +import restComms from "../services/RestComms" import {cloudClientService} from "../services/CloudClientService" import {useCloudClientStatusStore} from "../stores/cloudClientStatus" import type {CloudClientStatusSnapshot} from "../runtime/config" @@ -26,4 +27,13 @@ export const session = { useCloudClientStatusStore.subscribe(() => cb(project())), /** Whether the live-session handshake has completed. */ isConnected: (): boolean => cloudClientService.isConnected(), + + /** Identity / account operations (island-owned RestComms). */ + account: { + /** Request account deletion — the backend emails a confirmation code. */ + delete: () => restComms.requestAccountDeletion(), + /** Confirm a pending account deletion with the emailed code. */ + confirmDelete: (requestId: string, confirmationCode: string) => + restComms.confirmAccountDeletion(requestId, confirmationCode), + }, } diff --git a/mobile/modules/island/src/facades/settings.ts b/mobile/modules/island/src/facades/settings.ts new file mode 100644 index 0000000000..3667362ba4 --- /dev/null +++ b/mobile/modules/island/src/facades/settings.ts @@ -0,0 +1,27 @@ +/** + * settings facade — `toolkit.settings`: the typed keyed user-settings surface over + * the island-owned settings store. This is the (A) OEM contract; the raw store at + * `toolkit.stores.settings` is the Mentra-app escape hatch. + * + * Keys + their schema live in `SETTINGS` (theme, devMode, metric/twelveHourTime, + * notifications, onboarding flags, …); `descriptor(key)`/`keys()` expose them. + */ +import {useSettingsStore, SETTINGS} from "../stores/settings" + +export const settings = { + /** Read a setting by key (the current value, or its default). */ + get: (key: string): T | undefined => useSettingsStore.getState().getSetting(key) as T | undefined, + /** + * Write a setting. `syncToServer` (default true) also pushes the change to the + * backend; pass false for device-local-only writes. + */ + set: (key: string, value: T, syncToServer = true) => + useSettingsStore.getState().setSetting(key, value, syncToServer), + /** Subscribe to changes for one key; returns an unsubscribe. */ + onChanged: (key: string, cb: (value: T | undefined) => void): (() => void) => + useSettingsStore.subscribe((s) => s.getSetting(key) as T | undefined, cb), + /** The schema descriptor for a key (type, default, options…), or undefined. */ + descriptor: (key: string) => SETTINGS[key], + /** All known setting keys. */ + keys: (): string[] => Object.keys(SETTINGS), +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 5245759d69..cda243f427 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -13,6 +13,8 @@ import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {speech} from "./facades/speech" import {session} from "./facades/session" +import {settings} from "./facades/settings" +import {dev} from "./facades/dev" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -28,8 +30,12 @@ export const toolkit = { stop, glasses, speech, - /** Cloud (cloud-v2) live-session status — island owns the client (keystone #5). */ + /** Cloud (cloud-v2) live-session status + account ops — island owns the client. */ session, + /** Typed keyed user settings over the island-owned settings store. */ + settings, + /** Developer/debug surface (backend+cloud URL overrides, reconnect, min version). */ + dev, display: { mirror: displayMirror, }, diff --git a/mobile/modules/island/src/stores/settings.ts b/mobile/modules/island/src/stores/settings.ts index fc14cefb85..f1678d3c0f 100644 --- a/mobile/modules/island/src/stores/settings.ts +++ b/mobile/modules/island/src/stores/settings.ts @@ -6,7 +6,7 @@ import {subscribeWithSelector} from "zustand/middleware" import restComms from "../services/RestComms" import {storage} from "../utils/storage" -interface Setting { +export interface Setting { key: string defaultValue: () => any writable: boolean From 3fe955e9d7679d8c4bc7e6289b0a45b699399c38 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 15:56:46 -0700 Subject: [PATCH 21/80] island: add toolkit.incidents facade + document settings/account/dev/incidents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toolkit.incidents — typed passthrough over the now-island RestComms incident submission (create/uploadLogs/uploadAttachments/sendFeedback). The single-call file() that bundles diagnostics island-side is a follow-up (needs the host's native diagnostics-gathering + console logBuffer moved in). OEM docs updated for the new facades (settings, session.account, dev, incidents). Emit build + 281 jest green. --- docs/glasses-oems/toolkit.mdx | 44 ++++++++++++++++++- mobile/jest.setup.js | 6 +++ .../modules/island/src/facades/incidents.ts | 25 +++++++++++ mobile/modules/island/src/island.ts | 3 ++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 mobile/modules/island/src/facades/incidents.ts diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index 276494d96f..bc89731262 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -160,8 +160,48 @@ const live = toolkit.session.isConnected() // handshake completed? ``` `status` is one of `connected | connecting | reconnecting | disconnected`; -`audioTransport` is `udp | ws | offline | none`. (Account operations — -delete/export — are not on this surface yet.) +`audioTransport` is `udp | ws | offline | none`. + +```ts +await toolkit.session.account.delete() // backend emails a confirmation code +await toolkit.session.account.confirmDelete(requestId, code) // confirm the deletion +``` + +## `toolkit.settings` + +Typed keyed user settings over the island-owned settings store. + +```ts +toolkit.settings.get(key) // current value (or default) +toolkit.settings.set(key, value) // also syncs to backend (pass false to skip) +toolkit.settings.onChanged(key, (v) => { … }) // subscribe; returns unsubscribe +toolkit.settings.descriptor(key) // schema (type/default/options) +toolkit.settings.keys() // all known keys +``` + +## `toolkit.incidents` + +Bug-report / feedback submission. The OEM writes its own report screen + gathers +diagnostics; island owns the submission. + +```ts +const r = await toolkit.incidents.create(feedback, phoneState) // -> { incidentId } +await toolkit.incidents.uploadLogs(incidentId, logs) +await toolkit.incidents.uploadAttachments(incidentId, images) +await toolkit.incidents.sendFeedback(feedback) +``` + +## `toolkit.dev` + +Developer/debug surface — backend + cloud-v2 URL overrides, reconnect, version gate. + +```ts +toolkit.dev.minimumClientVersion() +toolkit.dev.backendUrl(); toolkit.dev.setBackendUrl(url) +toolkit.dev.cloudUrls(); toolkit.dev.setCloudUrls({core, runtime}) // sets + reconnects +toolkit.dev.savedUrls() +toolkit.dev.reconnectCloud() +``` ## `toolkit.stores.*` — escape hatch (not the OEM contract) diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 07403e5d90..feed63c89f 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -355,6 +355,12 @@ jest.mock("@mentra/island", () => { savedUrls: jest.fn(() => []), reconnectCloud: jest.fn(), }, + incidents: { + create: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false, value: {incidentId: "test"}})), + uploadLogs: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + uploadAttachments: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + sendFeedback: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + }, stores: { glasses: realGlasses.useGlassesStore, display: realDisplay.useDisplayStore, diff --git a/mobile/modules/island/src/facades/incidents.ts b/mobile/modules/island/src/facades/incidents.ts new file mode 100644 index 0000000000..59a19e2d91 --- /dev/null +++ b/mobile/modules/island/src/facades/incidents.ts @@ -0,0 +1,25 @@ +/** + * incidents facade — `toolkit.incidents`: bug-report / feedback submission over the + * now-island RestComms (the incident REST calls moved in with the settings+RestComms + * keystone). Thin passthrough (args forwarded with exact types via Parameters<>), + * so the facade never drifts from RestComms. + * + * The OEM writes its own bug-report SCREEN and gathers user input + diagnostics + * (phone state, recent logs, screenshots); island owns the submission. A single-call + * `file()` that ALSO bundles diagnostics island-side is a follow-up — it needs the + * host's native diagnostics-gathering (NetInfo/Constants/Location/ImagePicker + the + * console logBuffer) moved into island first. + */ +import restComms from "../services/RestComms" + +export const incidents = { + /** Create an incident (returns its id); pass the gathered phone-state snapshot. */ + create: (...args: Parameters) => restComms.createIncident(...args), + /** Upload the captured phone logs against an incident id. */ + uploadLogs: (...args: Parameters) => restComms.uploadIncidentLogs(...args), + /** Upload screenshot/image attachments against an incident id. */ + uploadAttachments: (...args: Parameters) => + restComms.uploadIncidentAttachments(...args), + /** Send freeform feedback (non-incident). */ + sendFeedback: (...args: Parameters) => restComms.sendFeedback(...args), +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index cda243f427..e866ef995c 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -15,6 +15,7 @@ import {speech} from "./facades/speech" import {session} from "./facades/session" import {settings} from "./facades/settings" import {dev} from "./facades/dev" +import {incidents} from "./facades/incidents" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -36,6 +37,8 @@ export const toolkit = { settings, /** Developer/debug surface (backend+cloud URL overrides, reconnect, min version). */ dev, + /** Bug-report / feedback submission (island-owned RestComms). */ + incidents, display: { mirror: displayMirror, }, From 151aee765a7cbd3f1bad4c556ffe8052770b0cdf Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 16:14:20 -0700 Subject: [PATCH 22/80] island: address Cursor review-bot findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - glasses.onStatus + session.onStatus: dedupe — fire only when the projected snapshot actually changes (were firing on every store update). - CloudClientService: read LC3 frame size LIVE from the island settings store so a reconnect after a glasses swap (G1=20 <-> G2=40) isn't stale on the configure() snapshot. - toolkit.dev: setCloudUrls/reconnectCloud now reconnect onto the merged current cloud-URL overrides (a single-URL update used to set the setting but not apply it). - docs/glasses-oems/toolkit.mdx: remove stray markup (broke MDX). Deferred (pre-existing, flagged): the ButtonPhotoSettings.size overwrite (proper fix = optional size, ripples into the BLE encoder — camera-domain) and the !client-vs-connected guard message in CloudClientService (carried over from the host, behavior-preserving). --- docs/glasses-oems/toolkit.mdx | 2 -- mobile/modules/island/src/facades/dev.ts | 21 ++++++++++++++++--- mobile/modules/island/src/facades/glasses.ts | 14 +++++++++++-- mobile/modules/island/src/facades/session.ts | 13 ++++++++++-- .../island/src/services/CloudClientService.ts | 8 ++++++- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index bc89731262..a4b9acfd90 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -224,5 +224,3 @@ single dependency-injection seam. The host implements the interface; the toolkit the behavior. OEM hosts implement the same ports with their own backing — that is what keeps the runtime device- and host-agnostic. (This seam is what the remaining domains — settings, permissions, gallery — land behind.) - - diff --git a/mobile/modules/island/src/facades/dev.ts b/mobile/modules/island/src/facades/dev.ts index 333d015886..125af9455a 100644 --- a/mobile/modules/island/src/facades/dev.ts +++ b/mobile/modules/island/src/facades/dev.ts @@ -12,6 +12,22 @@ import {useSettingsStore, SETTINGS} from "../stores/settings" import {cloudClientService} from "../services/CloudClientService" import restComms from "../services/RestComms" +/** + * Reconnect the cloud client onto the current cloud-URL overrides. When both are + * explicit, reconnect with them; otherwise rebuild with the boot-resolved config + * (which carries the host's Metro/env URL resolution). + */ +function applyCloudUrlReconnect(): void { + const s = useSettingsStore.getState() + const core = s.getSetting(SETTINGS.cloud_core_url.key) + const runtime = s.getSetting(SETTINGS.cloud_runtime_url.key) + if (typeof core === "string" && core.trim() && typeof runtime === "string" && runtime.trim()) { + cloudClientService.reconnect({core: core.trim(), runtime: runtime.trim()}) + } else { + cloudClientService.reconnect() + } +} + export const dev = { /** The backend's required/recommended client version. */ minimumClientVersion: () => restComms.getMinimumClientVersion(), @@ -31,13 +47,12 @@ export const dev = { const s = useSettingsStore.getState() if (urls.core !== undefined) s.setSetting(SETTINGS.cloud_core_url.key, urls.core) if (urls.runtime !== undefined) s.setSetting(SETTINGS.cloud_runtime_url.key, urls.runtime) - if (urls.core && urls.runtime) cloudClientService.reconnect({core: urls.core, runtime: urls.runtime}) - else cloudClientService.reconnect() + applyCloudUrlReconnect() }, /** The saved backend URLs (the dev URL-switcher list). */ savedUrls: (): string[] => useSettingsStore.getState().getSetting(SETTINGS.saved_backend_urls.key) ?? [], /** Tear down + rebuild the live cloud client (the dev "reconnect" button). */ - reconnectCloud: (): void => cloudClientService.reconnect(), + reconnectCloud: (): void => applyCloudUrlReconnect(), } diff --git a/mobile/modules/island/src/facades/glasses.ts b/mobile/modules/island/src/facades/glasses.ts index efd957e313..67226f0e12 100644 --- a/mobile/modules/island/src/facades/glasses.ts +++ b/mobile/modules/island/src/facades/glasses.ts @@ -55,8 +55,18 @@ export const glasses = { // --- read-model (projected from the island-owned glasses store) --- status: (): GlassesStatusSnapshot => projectStatus(), - onStatus: (cb: (status: GlassesStatusSnapshot) => void): (() => void) => - useGlassesStore.subscribe(() => cb(projectStatus())), + onStatus: (cb: (status: GlassesStatusSnapshot) => void): (() => void) => { + // Dedupe: the glasses store updates on many fields; only fire when the + // projected status snapshot actually changes. + let last = JSON.stringify(projectStatus()) + return useGlassesStore.subscribe(() => { + const snap = projectStatus() + const key = JSON.stringify(snap) + if (key === last) return + last = key + cb(snap) + }) + }, info: (): GlassesInfoSnapshot => projectInfo(), capabilities: () => getModelCapabilities(useGlassesStore.getState().deviceModel as DeviceTypes), /** Ask the glasses to report fresh firmware/version info (updates the store). */ diff --git a/mobile/modules/island/src/facades/session.ts b/mobile/modules/island/src/facades/session.ts index 9a37d88a84..26a52779d4 100644 --- a/mobile/modules/island/src/facades/session.ts +++ b/mobile/modules/island/src/facades/session.ts @@ -23,8 +23,17 @@ export const session = { /** Current cloud live-session status (connection state + audio transport). */ status: (): CloudClientStatusSnapshot => project(), /** Subscribe to cloud session-status changes; returns an unsubscribe. */ - onStatus: (cb: (status: CloudClientStatusSnapshot) => void): (() => void) => - useCloudClientStatusStore.subscribe(() => cb(project())), + onStatus: (cb: (status: CloudClientStatusSnapshot) => void): (() => void) => { + // Dedupe: fire only when the projected {status, audioTransport} changes. + let last = JSON.stringify(project()) + return useCloudClientStatusStore.subscribe(() => { + const snap = project() + const key = JSON.stringify(snap) + if (key === last) return + last = key + cb(snap) + }) + }, /** Whether the live-session handshake has completed. */ isConnected: (): boolean => cloudClientService.isConnected(), diff --git a/mobile/modules/island/src/services/CloudClientService.ts b/mobile/modules/island/src/services/CloudClientService.ts index 8ca4608b9b..894758e80d 100644 --- a/mobile/modules/island/src/services/CloudClientService.ts +++ b/mobile/modules/island/src/services/CloudClientService.ts @@ -19,6 +19,7 @@ import type {SubjectTokenType} from "@mentra/cloud-client" import type {AudioSubscription, TranscriptionData, TranslationData} from "@mentra/cloud-runtime/protocol" import {getAuth, getConfigValues} from "../runtime/bootstrap" +import {useSettingsStore, SETTINGS} from "../stores/settings" import { configureRuntime, type CloudClientStatusSnapshot, @@ -65,7 +66,12 @@ function resolveEndpoints(): {core: string; runtime: string} { } function frameSizeBytes(): Lc3FrameSizeBytes { - const size = getConfigValues().audioFrameSizeBytes + // Read LIVE from the island settings store (the source of truth) so a + // reconnect after a glasses swap (G1=20 ↔ G2=40) picks up the new frame size, + // not the one-time toolkit.configure({config}) snapshot. Falls back to the + // config value (set at boot) then 20 when the setting is unset. + const fromSettings = useSettingsStore.getState().getSetting(SETTINGS.lc3_frame_size.key) + const size = fromSettings ?? getConfigValues().audioFrameSizeBytes return size === 20 || size === 40 || size === 60 ? size : 20 } From 2837f6d127a96a179a7426f75013cc851fa77e81 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 16:17:58 -0700 Subject: [PATCH 23/80] island: finish toolkit.display.mirror (setView/view) + add toolkit.miniapps lifecycle facade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - display.mirror: expose view()/setView('main'|'dashboard') — the store already backed it; was just unexposed. (sendButtonPress simulated-input is a separate dev aid.) - toolkit.miniapps: lifecycle facade over the island apps store — list/onChanged(deduped)/refresh/start/stop/setForeground/clearForeground/stopAll/ install/uninstall. The WebView bridge primitives stay exported separately (host mounts the ). jest mock extended; typecheck + emit build + 281 jest green. --- mobile/jest.setup.js | 14 ++++++ .../island/src/facades/displayMirror.ts | 9 ++++ mobile/modules/island/src/facades/miniapps.ts | 48 +++++++++++++++++++ mobile/modules/island/src/island.ts | 3 ++ 4 files changed, 74 insertions(+) create mode 100644 mobile/modules/island/src/facades/miniapps.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index feed63c89f..28ff46eb0a 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -328,8 +328,22 @@ jest.mock("@mentra/island", () => { mirror: { current: jest.fn(() => null), onMirror: jest.fn(() => () => {}), + view: jest.fn(() => "main"), + setView: jest.fn(), }, }, + miniapps: { + list: jest.fn(() => []), + onChanged: jest.fn(() => () => {}), + refresh: jest.fn(() => Promise.resolve()), + start: jest.fn(() => Promise.resolve(true)), + stop: jest.fn(() => Promise.resolve()), + setForeground: jest.fn(() => Promise.resolve()), + clearForeground: jest.fn(), + stopAll: jest.fn(() => Promise.resolve({is_ok: () => true})), + install: jest.fn(() => Promise.resolve({is_ok: () => true})), + uninstall: jest.fn(() => Promise.resolve({is_ok: () => true})), + }, session: { status: jest.fn(() => ({status: "disconnected", audioTransport: "none"})), onStatus: jest.fn(() => () => {}), diff --git a/mobile/modules/island/src/facades/displayMirror.ts b/mobile/modules/island/src/facades/displayMirror.ts index d0739493c9..6b226b7912 100644 --- a/mobile/modules/island/src/facades/displayMirror.ts +++ b/mobile/modules/island/src/facades/displayMirror.ts @@ -20,4 +20,13 @@ export const displayMirror = { onMirror(cb: (event: DisplayMirrorEvent) => void): () => void { return useDisplayStore.subscribe((s) => s.currentEvent, cb) }, + + /** Which view is previewed — the main screen or the dashboard. */ + view(): string { + return useDisplayStore.getState().view + }, + /** Switch the previewed view between `"main"` and `"dashboard"`. */ + setView(view: "main" | "dashboard"): void { + useDisplayStore.getState().setView(view) + }, } diff --git a/mobile/modules/island/src/facades/miniapps.ts b/mobile/modules/island/src/facades/miniapps.ts new file mode 100644 index 0000000000..67e9749997 --- /dev/null +++ b/mobile/modules/island/src/facades/miniapps.ts @@ -0,0 +1,48 @@ +/** + * miniapps facade — `toolkit.miniapps`: miniapp lifecycle over the island-owned + * apps store. Thin forwarding (args via Parameters<>), so the facade never drifts + * from the store's actions. + * + * The miniapp WebView is a HOST component (island ships the bridge, not the view) — + * the bridge primitives (`buildMiniappGlobalsScript`, `buildMentraUiShim`, the + * MentraJS router) are exported from `@mentra/island` directly; the host mounts a + * `` wired to them. This facade is the lifecycle half. + */ +import {useAppStatusStore} from "../stores/apps" +import type {ClientApp} from "../types" + +type AppsState = ReturnType + +export const miniapps = { + /** All installed miniapps (snapshot). */ + list: (): ClientApp[] => useAppStatusStore.getState().apps, + /** Subscribe to the installed-app set; fires only when it changes. Returns an unsubscribe. */ + onChanged: (cb: (apps: ClientApp[]) => void): (() => void) => { + let last = JSON.stringify(useAppStatusStore.getState().apps) + return useAppStatusStore.subscribe(() => { + const apps = useAppStatusStore.getState().apps + const key = JSON.stringify(apps) + if (key === last) return + last = key + cb(apps) + }) + }, + + /** Re-fetch the installed-app list from the backend. */ + refresh: () => useAppStatusStore.getState().refresh(), + /** Start (foreground) a miniapp. */ + start: (...args: Parameters) => useAppStatusStore.getState().start(...args), + /** Stop a miniapp. */ + stop: (...args: Parameters) => useAppStatusStore.getState().stop(...args), + /** Bring a running miniapp to the foreground. */ + setForeground: (...args: Parameters) => + useAppStatusStore.getState().setForeground(...args), + /** Clear the current foreground miniapp. */ + clearForeground: () => useAppStatusStore.getState().clearForeground(), + /** Stop all running miniapps. */ + stopAll: () => useAppStatusStore.getState().stopAll(), + /** Install a miniapp from a bundle URL. */ + install: (...args: Parameters) => useAppStatusStore.getState().install(...args), + /** Uninstall a miniapp. */ + uninstall: (...args: Parameters) => useAppStatusStore.getState().uninstall(...args), +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index e866ef995c..62648060b8 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -16,6 +16,7 @@ import {session} from "./facades/session" import {settings} from "./facades/settings" import {dev} from "./facades/dev" import {incidents} from "./facades/incidents" +import {miniapps} from "./facades/miniapps" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -39,6 +40,8 @@ export const toolkit = { dev, /** Bug-report / feedback submission (island-owned RestComms). */ incidents, + /** Miniapp lifecycle (the WebView bridge primitives are exported separately). */ + miniapps, display: { mirror: displayMirror, }, From e07a98b772884ef5d14da06e0b5bfa60f4336fba Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 16:20:30 -0700 Subject: [PATCH 24/80] island: add toolkit.incidents.file() one-call submission file({feedbackData, phoneState, logs?, screenshots?}) -> {incidentId} orchestrates the full flow island-side: createIncident -> uploadLogs -> sendIncidentId to glasses -> uploadAttachments. The OEM gathers diagnostics (native-coupled: NetInfo/Constants/ Location/ImagePicker + console logBuffer) and passes them in; island owns the submission. Lower-level primitives still exposed. Doc + jest mock updated; emit build + 281 jest green. Finishes the last of the 3 partial facades (with display.mirror.setView + toolkit.miniapps). --- docs/glasses-oems/toolkit.mdx | 9 +++- mobile/jest.setup.js | 1 + .../modules/island/src/facades/incidents.ts | 54 ++++++++++++++++--- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index a4b9acfd90..381d1e207a 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -182,10 +182,15 @@ toolkit.settings.keys() // all known keys ## `toolkit.incidents` Bug-report / feedback submission. The OEM writes its own report screen + gathers -diagnostics; island owns the submission. +diagnostics (phone-state, logs, screenshots — native-coupled); island owns the +submission orchestration. ```ts -const r = await toolkit.incidents.create(feedback, phoneState) // -> { incidentId } +// one-call: create -> upload logs -> notify glasses -> upload screenshots +const { incidentId } = await toolkit.incidents.file({ feedbackData, phoneState, logs, screenshots }) + +// or drive the steps yourself: +const r = await toolkit.incidents.create(feedbackData, phoneState) await toolkit.incidents.uploadLogs(incidentId, logs) await toolkit.incidents.uploadAttachments(incidentId, images) await toolkit.incidents.sendFeedback(feedback) diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 28ff46eb0a..eceaab608e 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -370,6 +370,7 @@ jest.mock("@mentra/island", () => { reconnectCloud: jest.fn(), }, incidents: { + file: jest.fn(() => Promise.resolve({incidentId: "test"})), create: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false, value: {incidentId: "test"}})), uploadLogs: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), uploadAttachments: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), diff --git a/mobile/modules/island/src/facades/incidents.ts b/mobile/modules/island/src/facades/incidents.ts index 59a19e2d91..783aee80e4 100644 --- a/mobile/modules/island/src/facades/incidents.ts +++ b/mobile/modules/island/src/facades/incidents.ts @@ -1,18 +1,58 @@ /** * incidents facade — `toolkit.incidents`: bug-report / feedback submission over the * now-island RestComms (the incident REST calls moved in with the settings+RestComms - * keystone). Thin passthrough (args forwarded with exact types via Parameters<>), - * so the facade never drifts from RestComms. + * keystone). * - * The OEM writes its own bug-report SCREEN and gathers user input + diagnostics - * (phone state, recent logs, screenshots); island owns the submission. A single-call - * `file()` that ALSO bundles diagnostics island-side is a follow-up — it needs the - * host's native diagnostics-gathering (NetInfo/Constants/Location/ImagePicker + the - * console logBuffer) moved into island first. + * `file()` is the one-call submission: island orchestrates createIncident → upload + * logs → notify the glasses → upload screenshots. The OEM writes its own report + * SCREEN and gathers the diagnostics (phone-state snapshot, recent logs, screenshots) + * — that gathering is genuinely native-coupled (NetInfo/Constants/Location/ImagePicker + * + console interception), so it stays host-side and is passed in. The lower-level + * primitives are also exposed for callers that want to drive the steps themselves. */ import restComms from "../services/RestComms" +import BluetoothSdk from "../../../bluetooth-sdk/build/_internal" +import {useGlassesStore} from "../stores/glasses" +import {isGlassesConnected} from "../services/GlassesReadiness" +import {useSettingsStore} from "../stores/settings" + +export interface IncidentFileInput { + /** The user's report fields (description/expected/actual/severity/contactEmail…). */ + feedbackData: Record + /** The gathered phone-state diagnostics snapshot (host-built; native-coupled). */ + phoneState: Record + /** Recent phone logs to attach. */ + logs?: Parameters[1] + /** Optional screenshot attachments. */ + screenshots?: Parameters[1] +} export const incidents = { + /** + * File an incident end-to-end: create it, upload logs, notify the connected + * glasses, and upload screenshots. Returns the new incident id (or an error). + */ + async file(input: IncidentFileInput): Promise<{incidentId?: string; error?: string}> { + const backendUrl = useSettingsStore.getState().getRestUrl() + const res = await restComms.createIncident(input.feedbackData, input.phoneState) + if (res.is_error()) return {error: res.error.message} + + const {incidentId} = res.value + if (input.logs && input.logs.length > 0) { + const r = await restComms.uploadIncidentLogs(incidentId, input.logs) + if (r.is_error()) console.warn("incidents.file: upload logs failed:", r.error?.message) + } + if (isGlassesConnected(useGlassesStore.getState().connection)) { + BluetoothSdk.sendIncidentId(incidentId, backendUrl) + } + if (input.screenshots && input.screenshots.length > 0) { + const r = await restComms.uploadIncidentAttachments(incidentId, input.screenshots) + if (r.is_error()) console.warn("incidents.file: upload attachments failed:", r.error?.message) + } + return {incidentId} + }, + + // --- lower-level primitives (drive the steps yourself) --- /** Create an incident (returns its id); pass the gathered phone-state snapshot. */ create: (...args: Parameters) => restComms.createIncident(...args), /** Upload the captured phone logs against an incident id. */ From 7689299eb13b3b5d40a57e5375a9c6fc821aa97a Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 16:46:15 -0700 Subject: [PATCH 25/80] island: fix 2 more review-bot findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stale cloud endpoint override: reconnect(null) now CLEARS endpointsOverride and falls back to the boot config; toolkit.dev's empty-settings branch passes null, so cleared/default cloud URLs stop reconnecting to a stale override. Added cloudClientService.stop() for lifecycle symmetry. - start() skips cloud init: toolkit.start() now constructs/connects the cloud client (and toolkit.stop() tears it down), so the documented configure()+start() flow yields a live toolkit.session for OEMs. Idempotent — the Mentra app's explicit cloudClient.init() becomes a no-op. Composed in island.ts to avoid a bootstrap<->service import cycle; safe now that init() reads the settings STORE (module-load-available), not the adapter. Emit build + 281 jest green. --- mobile/modules/island/src/facades/dev.ts | 4 ++- mobile/modules/island/src/island.ts | 18 ++++++++++--- .../island/src/services/CloudClientService.ts | 25 +++++++++++++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/mobile/modules/island/src/facades/dev.ts b/mobile/modules/island/src/facades/dev.ts index 125af9455a..e394c4db4d 100644 --- a/mobile/modules/island/src/facades/dev.ts +++ b/mobile/modules/island/src/facades/dev.ts @@ -24,7 +24,9 @@ function applyCloudUrlReconnect(): void { if (typeof core === "string" && core.trim() && typeof runtime === "string" && runtime.trim()) { cloudClientService.reconnect({core: core.trim(), runtime: runtime.trim()}) } else { - cloudClientService.reconnect() + // No explicit override → CLEAR any stale override and fall back to the + // boot-resolved config (don't keep reconnecting to a previously-set URL). + cloudClientService.reconnect(null) } } diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 62648060b8..aab9da971c 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -8,7 +8,8 @@ * named exports in `index.ts` during the migration. The flat exports (and the * `BluetoothSdk` passthrough) stay until every screen has moved onto `island.*`. */ -import {configure, start, stop} from "./runtime/bootstrap" +import {configure, start as bootstrapStart, stop as bootstrapStop} from "./runtime/bootstrap" +import {cloudClientService} from "./services/CloudClientService" import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {speech} from "./facades/speech" @@ -28,8 +29,19 @@ import {useSettingsStore} from "./stores/settings" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ configure, - start, - stop, + /** Start the runtime: mark started + construct/connect the cloud client. */ + async start() { + await bootstrapStart() + // Construct + connect the cloud client so the documented configure()+start() + // lifecycle yields a live toolkit.session. Idempotent — the Mentra app's + // explicit cloudClient.init() is then a no-op. + cloudClientService.init() + }, + /** Stop the runtime: tear down the cloud client + mark stopped. */ + async stop() { + cloudClientService.stop() + await bootstrapStop() + }, glasses, speech, /** Cloud (cloud-v2) live-session status + account ops — island owns the client. */ diff --git a/mobile/modules/island/src/services/CloudClientService.ts b/mobile/modules/island/src/services/CloudClientService.ts index 894758e80d..110039df45 100644 --- a/mobile/modules/island/src/services/CloudClientService.ts +++ b/mobile/modules/island/src/services/CloudClientService.ts @@ -320,12 +320,12 @@ export const cloudClientService = { }, /** - * Tear down + rebuild with freshly-resolved endpoints (the dev "Cloud V2" - * override, applied without an app rebuild). Pass new endpoints to switch - * URLs; omit to rebuild with the current config. + * Tear down + rebuild. Pass new endpoints to switch URLs; pass `null` to CLEAR + * a prior override and fall back to the boot config (so cleared/default cloud + * URLs don't keep reconnecting to a stale override); omit to keep the current. */ - reconnect(endpoints?: {core: string; runtime: string}): void { - if (endpoints) endpointsOverride = endpoints + reconnect(endpoints?: {core: string; runtime: string} | null): void { + if (endpoints !== undefined) endpointsOverride = endpoints try { client?.runtime.close() } catch (err) { @@ -343,6 +343,21 @@ export const cloudClientService = { selfWire() }, + /** Tear down the client + connection (the toolkit.stop() lifecycle). */ + stop(): void { + try { + client?.runtime.close() + } catch (err) { + console.warn(`${LOG_TAG}: stop close() failed: ${(err as Error)?.message ?? err}`) + } + clearRuntimeEventSubscriptions() + const wasConnected = connected + client = null + connected = false + resetRuntimeStatus() + if (wasConnected) notifyConnectionListeners(false) + }, + /** Device-side managed photo (cloud-v2): presign now, deliver bytes, await ready. */ startManagedPhoto(opts: Record = {}) { if (!client) throw new Error("cloud client not connected") From 2f397d832270bebe80a54d6803437d34ac45b028 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 16:55:53 -0700 Subject: [PATCH 26/80] ci(ios): self-heal corrupted CocoaPods prebuilt cache on pod install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The iOS build intermittently failed at 'pod install' with 'React-Core-prebuilt/React.xcframework/Info.plist doesn't exist' — the cached Pods (keyed by lockfile) can be valid-by-key yet missing the prebuilt React xcframework (a known RN-0.83 prebuilt + CocoaPods cache flake on the self-hosted runners). pod install was a bare command with no fallback (only the later build step retried with clean caches). Make pod install self-heal: on failure, clear project Pods + the global pod cache and reinstall with --repo-update. Happy path unchanged. Not a TS issue — one of the two iOS build jobs passed (6m39s) on the same commit; only the runner with the corrupted cache failed. --- .github/workflows/mentra-app-ios-build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mentra-app-ios-build.yml b/.github/workflows/mentra-app-ios-build.yml index 5213218ad7..442ccdb678 100644 --- a/.github/workflows/mentra-app-ios-build.yml +++ b/.github/workflows/mentra-app-ios-build.yml @@ -99,7 +99,18 @@ jobs: - name: Install CocoaPods dependencies working-directory: ./mobile/ios - run: pod install + run: | + # Self-heal a corrupted prebuilt-framework cache: the cached Pods can be + # valid-by-lockfile-key yet missing the React-Core-prebuilt React.xcframework + # Info.plist (a known RN-0.83 prebuilt + CocoaPods cache flake on the + # self-hosted runners). On failure, clear the project Pods + the global pod + # cache and reinstall fresh. Happy path is unchanged. + pod install || { + echo "::warning::pod install failed (likely a stale/corrupt prebuilt cache) — clearing caches and retrying" + rm -rf Pods Podfile.lock + pod cache clean --all || true + pod install --repo-update + } - name: List available simulators run: xcrun simctl list devices available From c63f6682df5034e2f324f4c74f443605702be8e2 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 17:45:49 -0700 Subject: [PATCH 27/80] island: add toolkit.glasses.settings + toolkit.pairing facades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - toolkit.glasses.settings: keyed DEVICE-settings facade (get/set/onChanged/descriptor/ available) scoped to BLUETOOTH_SETTING_KEYS (exported), minus the internal sync keys (core_token/auth_email). set() persists + auto-syncs to the glasses via the store. - toolkit.pairing: first-time discovery + pair — scan/scanning/searchResults/onFound(deduped) over the core store + BluetoothSdk startScan/connect/setDefaultDevice + the pair_failure/ glasses_not_ready events. (connectDefault for the already-paired default stays on glasses.) Doc + jest mock updated; mobile typecheck + island emit build + 281 jest green. permissions deferred — it's the host-coupled one (PermissionsUtils: i18n/theme/AlertUtils), needs the adapter pattern; recommended as a focused follow-up off this now-green PR. --- docs/glasses-oems/toolkit.mdx | 29 ++++++++++++ mobile/jest.setup.js | 17 +++++++ mobile/modules/island/src/facades/glasses.ts | 2 + .../island/src/facades/glassesSettings.ts | 30 ++++++++++++ mobile/modules/island/src/facades/pairing.ts | 47 +++++++++++++++++++ mobile/modules/island/src/island.ts | 3 ++ mobile/modules/island/src/stores/settings.ts | 2 +- 7 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 mobile/modules/island/src/facades/glassesSettings.ts create mode 100644 mobile/modules/island/src/facades/pairing.ts diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index 381d1e207a..4dfa583524 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -120,6 +120,35 @@ const off = toolkit.glasses.wifi.onStatus((status) => { … }) // subscribe `connect()` propagates the bluetooth layer's coded errors (`bluetooth_powered_off`, `request_timeout`, …) unchanged, so your UI keeps its own error mapping. +### `toolkit.glasses.settings` + +Keyed **device** settings (brightness, head-up angle, dashboard, camera/button, +sensing, …). `set()` persists *and* auto-syncs to the connected glasses. + +```ts +toolkit.glasses.settings.get(key) +toolkit.glasses.settings.set(key, value) // also pushes to the device +toolkit.glasses.settings.onChanged(key, (v) => { … }) +toolkit.glasses.settings.descriptor(key) +toolkit.glasses.settings.available() // the device-setting keys +``` + +## `toolkit.pairing` + +First-time glasses discovery + pairing. (Reconnecting the already-paired default is +`toolkit.glasses.connectDefault()`.) + +```ts +toolkit.pairing.scan() // start scanning +toolkit.pairing.scanning() // is a scan in progress? +toolkit.pairing.searchResults() // discovered devices (snapshot) +const off = toolkit.pairing.onFound((r) => { … }) // subscribe; returns unsubscribe +await toolkit.pairing.pair(deviceId) // connect to a discovered device +await toolkit.pairing.setDefault(deviceId) // make it the connectDefault() target +toolkit.pairing.onPairFailure((e) => { … }) +toolkit.pairing.onGlassesNotReady((e) => { … }) +``` + ## `toolkit.speech` On-device STT/TTS model management. diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index eceaab608e..7ade88bf6e 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -301,6 +301,13 @@ jest.mock("@mentra/island", () => { status: jest.fn(() => ({state: "disconnected"})), onStatus: jest.fn(() => () => {}), }, + settings: { + get: jest.fn(() => undefined), + set: jest.fn(() => Promise.resolve({is_ok: () => true, is_error: () => false})), + onChanged: jest.fn(() => () => {}), + descriptor: jest.fn(() => undefined), + available: jest.fn(() => []), + }, }, speech: { stt: { @@ -332,6 +339,16 @@ jest.mock("@mentra/island", () => { setView: jest.fn(), }, }, + pairing: { + scan: jest.fn(), + scanning: jest.fn(() => false), + searchResults: jest.fn(() => []), + onFound: jest.fn(() => () => {}), + pair: jest.fn(() => Promise.resolve()), + setDefault: jest.fn(() => Promise.resolve()), + onPairFailure: jest.fn(() => () => {}), + onGlassesNotReady: jest.fn(() => () => {}), + }, miniapps: { list: jest.fn(() => []), onChanged: jest.fn(() => () => {}), diff --git a/mobile/modules/island/src/facades/glasses.ts b/mobile/modules/island/src/facades/glasses.ts index 67226f0e12..0295656aae 100644 --- a/mobile/modules/island/src/facades/glasses.ts +++ b/mobile/modules/island/src/facades/glasses.ts @@ -13,6 +13,7 @@ import {useGlassesStore} from "../stores/glasses" import {isGlassesReady} from "../services/GlassesReadiness" import {getModelCapabilities, type DeviceTypes} from "../types" import {glassesWifi} from "./glassesWifi" +import {glassesSettings} from "./glassesSettings" function projectStatus() { const s = useGlassesStore.getState() @@ -84,4 +85,5 @@ export const glasses = { // --- sub-facades --- wifi: glassesWifi, + settings: glassesSettings, } diff --git a/mobile/modules/island/src/facades/glassesSettings.ts b/mobile/modules/island/src/facades/glassesSettings.ts new file mode 100644 index 0000000000..c4fbd7ac6c --- /dev/null +++ b/mobile/modules/island/src/facades/glassesSettings.ts @@ -0,0 +1,30 @@ +/** + * glasses.settings facade — `toolkit.glasses.settings`: the keyed DEVICE-settings + * surface (brightness, head-up angle, dashboard, camera/button, sensing, …). These + * are the settings the island store auto-syncs to the glasses over the bluetooth-sdk + * (`BLUETOOTH_SETTING_KEYS`), so `set()` both persists and pushes to the device. + * + * Distinct from `toolkit.settings` (user/app prefs): this is scoped to the + * hardware-facing keys, minus the internal sync keys (auth/token) that aren't + * user-tunable device settings. + */ +import {useSettingsStore, SETTINGS, BLUETOOTH_SETTING_KEYS} from "../stores/settings" + +// Internal sync keys carried on BLUETOOTH_SETTING_KEYS for the device handshake — +// NOT user-tunable device settings, so kept out of the OEM-facing surface. +const INTERNAL_KEYS = new Set([SETTINGS.core_token.key, SETTINGS.auth_email.key]) +const DEVICE_KEYS = BLUETOOTH_SETTING_KEYS.filter((k) => !INTERNAL_KEYS.has(k)) + +export const glassesSettings = { + /** Read a device setting by key. */ + get: (key: string): T | undefined => useSettingsStore.getState().getSetting(key) as T | undefined, + /** Write a device setting — persists and auto-syncs to the connected glasses. */ + set: (key: string, value: T) => useSettingsStore.getState().setSetting(key, value), + /** Subscribe to changes for one device-setting key; returns an unsubscribe. */ + onChanged: (key: string, cb: (value: T | undefined) => void): (() => void) => + useSettingsStore.subscribe((s) => s.getSetting(key) as T | undefined, cb), + /** The schema descriptor for a key (type, default, options…), or undefined. */ + descriptor: (key: string) => SETTINGS[key], + /** The device-setting keys synced to the glasses (excludes internal sync keys). */ + available: (): string[] => DEVICE_KEYS, +} diff --git a/mobile/modules/island/src/facades/pairing.ts b/mobile/modules/island/src/facades/pairing.ts new file mode 100644 index 0000000000..52c222fb05 --- /dev/null +++ b/mobile/modules/island/src/facades/pairing.ts @@ -0,0 +1,47 @@ +/** + * pairing facade — `toolkit.pairing`: scan for nearby glasses + pair. Scan state + * (searching + results) comes from the island core store; scan/connect go over the + * bluetooth-sdk; pair-failure / not-ready are the bluetooth-sdk events. + * + * (Connect-the-already-paired-default is `toolkit.glasses.connectDefault()`; this + * facade is the first-time discovery + pair flow.) + */ +import BluetoothSdk from "@mentra/bluetooth-sdk" +import type {PairFailureEvent, GlassesNotReadyEvent} from "@mentra/bluetooth-sdk" +import {useCoreStore} from "../stores/core" + +export const pairing = { + /** Start scanning for nearby glasses. Results land on `searchResults()`/`onFound()`. */ + scan: (...args: Parameters) => BluetoothSdk.startScan(...args), + /** Whether a scan is currently in progress. */ + scanning: (): boolean => useCoreStore.getState().searching, + /** The current scan results (snapshot). */ + searchResults: () => useCoreStore.getState().searchResults, + /** Subscribe to scan-result changes; fires only when they change. Returns an unsubscribe. */ + onFound: (cb: (results: ReturnType["searchResults"]) => void): (() => void) => { + let last = JSON.stringify(useCoreStore.getState().searchResults) + return useCoreStore.subscribe(() => { + const results = useCoreStore.getState().searchResults + const key = JSON.stringify(results) + if (key === last) return + last = key + cb(results) + }) + }, + + /** Pair with (connect to) a discovered device. */ + pair: (...args: Parameters) => BluetoothSdk.connect(...args), + /** Set a device as the default for subsequent `glasses.connectDefault()`. */ + setDefault: (...args: Parameters) => BluetoothSdk.setDefaultDevice(...args), + + /** Subscribe to pairing failures; returns an unsubscribe. */ + onPairFailure: (cb: (event: PairFailureEvent) => void): (() => void) => { + const sub = BluetoothSdk.addListener("pair_failure", cb) + return () => sub.remove() + }, + /** Subscribe to glasses-not-ready events during pairing; returns an unsubscribe. */ + onGlassesNotReady: (cb: (event: GlassesNotReadyEvent) => void): (() => void) => { + const sub = BluetoothSdk.addListener("glasses_not_ready", cb) + return () => sub.remove() + }, +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index aab9da971c..101f54f82a 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -18,6 +18,7 @@ import {settings} from "./facades/settings" import {dev} from "./facades/dev" import {incidents} from "./facades/incidents" import {miniapps} from "./facades/miniapps" +import {pairing} from "./facades/pairing" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -54,6 +55,8 @@ export const toolkit = { incidents, /** Miniapp lifecycle (the WebView bridge primitives are exported separately). */ miniapps, + /** First-time glasses discovery + pairing. */ + pairing, display: { mirror: displayMirror, }, diff --git a/mobile/modules/island/src/stores/settings.ts b/mobile/modules/island/src/stores/settings.ts index f1678d3c0f..2271f05ac9 100644 --- a/mobile/modules/island/src/stores/settings.ts +++ b/mobile/modules/island/src/stores/settings.ts @@ -655,7 +655,7 @@ export const OFFLINE_APPLETS: string[] = ["com.mentra.livecaptions", "com.mentra // These settings are automatically synced to the Bluetooth SDK. // Keep this list hardware-facing; app/UI/cloud-only preferences should stay in JS/Crust. -const BLUETOOTH_SETTING_KEYS: string[] = [ +export const BLUETOOTH_SETTING_KEYS: string[] = [ // Bluetooth settings: SETTINGS.sensing_enabled.key, SETTINGS.power_saving_mode.key, From 6f53c0e62d57a8794a47c77f663b8ce724815664 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 20:21:01 -0700 Subject: [PATCH 28/80] island: glasses long-tail (connect/connectSimulated/setDefault + controller) + toolkit.phoneNotifications - toolkit.glasses: + connect(device), connectSimulated(), setDefault(device), and a controller sub-facade (connectDefault/disconnect/forget). Switched glasses.ts to the internal btsdk surface (these live there, not the public entry). Dropped reconnect() (no btsdk backing) and controller.connect() (only connectDefaultController exists). - toolkit.phoneNotifications: enabled/setEnabled + blocklist/setBlocklist (island settings), installedApps + has/requestListenerPermission (CrustModule). Android-only; iOS defaults. Emit build confirms crust resolves in island's standalone build. Emit build + 281 jest green. --- mobile/jest.setup.js | 17 +++++++ mobile/modules/island/src/facades/glasses.ts | 20 +++++++- .../island/src/facades/phoneNotifications.ts | 50 +++++++++++++++++++ mobile/modules/island/src/island.ts | 3 ++ 4 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 mobile/modules/island/src/facades/phoneNotifications.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 7ade88bf6e..106c31d7cf 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -287,6 +287,14 @@ jest.mock("@mentra/island", () => { connectDefault: jest.fn(() => Promise.resolve()), disconnect: jest.fn(() => Promise.resolve()), forget: jest.fn(() => Promise.resolve()), + connect: jest.fn(() => Promise.resolve()), + connectSimulated: jest.fn(() => Promise.resolve()), + setDefault: jest.fn(() => Promise.resolve()), + controller: { + connectDefault: jest.fn(() => Promise.resolve()), + disconnect: jest.fn(() => Promise.resolve()), + forget: jest.fn(() => Promise.resolve()), + }, status: jest.fn(() => ({state: "disconnected"})), onStatus: jest.fn(() => () => {}), info: jest.fn(() => ({})), @@ -339,6 +347,15 @@ jest.mock("@mentra/island", () => { setView: jest.fn(), }, }, + phoneNotifications: { + enabled: jest.fn(() => false), + setEnabled: jest.fn(() => Promise.resolve({is_ok: () => true})), + installedApps: jest.fn(() => Promise.resolve([])), + blocklist: jest.fn(() => []), + setBlocklist: jest.fn(() => Promise.resolve({is_ok: () => true})), + hasListenerPermission: jest.fn(() => Promise.resolve(false)), + requestListenerPermission: jest.fn(() => Promise.resolve()), + }, pairing: { scan: jest.fn(), scanning: jest.fn(() => false), diff --git a/mobile/modules/island/src/facades/glasses.ts b/mobile/modules/island/src/facades/glasses.ts index 0295656aae..66d9bcc69b 100644 --- a/mobile/modules/island/src/facades/glasses.ts +++ b/mobile/modules/island/src/facades/glasses.ts @@ -7,8 +7,11 @@ * Read-models project the raw glasses store into the stable shape OEM UIs consume, * so the device store's field layout doesn't leak into every screen. */ -import BluetoothSdk from "@mentra/bluetooth-sdk" -import type {ButtonPressEvent, TouchEvent} from "@mentra/bluetooth-sdk" +// Internal btsdk surface — connectSimulated/reconnect/controller live on the full +// surface, not the public entry. Relative path (the alias doesn't resolve in island's +// standalone build); jest moduleNameMapper + tsconfig both resolve it. +import BluetoothSdk from "../../../bluetooth-sdk/build/_internal" +import type {ButtonPressEvent, TouchEvent} from "../../../bluetooth-sdk/build/_internal" import {useGlassesStore} from "../stores/glasses" import {isGlassesReady} from "../services/GlassesReadiness" import {getModelCapabilities, type DeviceTypes} from "../types" @@ -53,6 +56,12 @@ export const glasses = { connectDefault: (): Promise => BluetoothSdk.connectDefault(), disconnect: (): Promise => BluetoothSdk.disconnect(), forget: (): Promise => BluetoothSdk.forget(), + /** Connect to a specific (discovered) device. */ + connect: (...args: Parameters) => BluetoothSdk.connect(...args), + /** Connect the built-in simulated glasses (dev/testing). */ + connectSimulated: (): Promise => BluetoothSdk.connectSimulated(), + /** Set a device as the `connectDefault()` target. */ + setDefault: (...args: Parameters) => BluetoothSdk.setDefaultDevice(...args), // --- read-model (projected from the island-owned glasses store) --- status: (): GlassesStatusSnapshot => projectStatus(), @@ -83,6 +92,13 @@ export const glasses = { return () => sub.remove() }, + // --- ring controller (optional input device) --- + controller: { + connectDefault: (): Promise => BluetoothSdk.connectDefaultController(), + disconnect: (): Promise => BluetoothSdk.disconnectController(), + forget: (): Promise => BluetoothSdk.forgetController(), + }, + // --- sub-facades --- wifi: glassesWifi, settings: glassesSettings, diff --git a/mobile/modules/island/src/facades/phoneNotifications.ts b/mobile/modules/island/src/facades/phoneNotifications.ts new file mode 100644 index 0000000000..ca05e90b35 --- /dev/null +++ b/mobile/modules/island/src/facades/phoneNotifications.ts @@ -0,0 +1,50 @@ +/** + * phoneNotifications facade — `toolkit.phoneNotifications`: forward phone + * notifications to the glasses, with a per-app blocklist. Android-only (the + * NotificationListenerService); the getters return sensible defaults on iOS. + * + * Enabled-state + blocklist are island settings (auto-synced to the native + * listener); the installed-apps list + the listener permission are CrustModule. + */ +import {Platform} from "react-native" +import CrustModule from "crust" +import {useSettingsStore, SETTINGS} from "../stores/settings" + +export const phoneNotifications = { + /** Is phone-notification forwarding enabled? (false on iOS) */ + enabled: (): boolean => { + if (Platform.OS !== "android") return false + return useSettingsStore.getState().getSetting(SETTINGS.notifications_enabled.key) ?? true + }, + /** Enable/disable phone-notification forwarding. (no-op on iOS) */ + setEnabled: (enabled: boolean) => { + if (Platform.OS !== "android") return + return useSettingsStore.getState().setSetting(SETTINGS.notifications_enabled.key, enabled) + }, + + /** The phone's installed apps (package, name, icon). (empty on iOS) */ + installedApps: (): Promise> => { + if (Platform.OS !== "android") return Promise.resolve([]) + return CrustModule.getInstalledApps() + }, + + /** The per-app blocklist (package names whose notifications are NOT forwarded). */ + blocklist: (): string[] => { + const b = useSettingsStore.getState().getSetting(SETTINGS.notifications_blocklist.key) + return Array.isArray(b) ? b : [] + }, + /** Replace the per-app blocklist. */ + setBlocklist: (packageNames: string[]) => + useSettingsStore.getState().setSetting(SETTINGS.notifications_blocklist.key, packageNames), + + /** Does the app have the Android notification-listener permission? (false on iOS) */ + hasListenerPermission: (): Promise => { + if (Platform.OS !== "android") return Promise.resolve(false) + return CrustModule.hasNotificationListenerPermission() + }, + /** Open the OS settings to grant the notification-listener permission. (no-op on iOS) */ + requestListenerPermission: () => { + if (Platform.OS !== "android") return + return CrustModule.openNotificationListenerSettings() + }, +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 101f54f82a..05fb6671ea 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -19,6 +19,7 @@ import {dev} from "./facades/dev" import {incidents} from "./facades/incidents" import {miniapps} from "./facades/miniapps" import {pairing} from "./facades/pairing" +import {phoneNotifications} from "./facades/phoneNotifications" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -57,6 +58,8 @@ export const toolkit = { miniapps, /** First-time glasses discovery + pairing. */ pairing, + /** Phone→glasses notification forwarding (Android). */ + phoneNotifications, display: { mirror: displayMirror, }, From c534a9c50af8fd9c8b5ecbab272cdeb768a9810e Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 20:23:30 -0700 Subject: [PATCH 29/80] island: add toolkit.permissions facade (lean, over react-native-permissions + manifest) check(feature)/request(feature) map OEM feature keys (microphone/camera/location/ background_location/calendar/bluetooth/phone_state/post_notifications) to OS permissions per platform (Android 31+ bluetooth, 33+ post_notifications handled); openSettings via Linking; requirementsForMiniapp reads the island app-registry manifest. NO host UI coupling (i18n/theme/AlertUtils stay in the host's PermissionsUtils on top). Emit build confirms react-native-permissions resolves in island's standalone build. 281 jest green. --- mobile/jest.setup.js | 6 + .../modules/island/src/facades/permissions.ts | 116 ++++++++++++++++++ mobile/modules/island/src/island.ts | 3 + 3 files changed, 125 insertions(+) create mode 100644 mobile/modules/island/src/facades/permissions.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 106c31d7cf..5f252634ff 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -347,6 +347,12 @@ jest.mock("@mentra/island", () => { setView: jest.fn(), }, }, + permissions: { + check: jest.fn(() => Promise.resolve(false)), + request: jest.fn(() => Promise.resolve(false)), + openSettings: jest.fn(() => Promise.resolve()), + requirementsForMiniapp: jest.fn(() => Promise.resolve([])), + }, phoneNotifications: { enabled: jest.fn(() => false), setEnabled: jest.fn(() => Promise.resolve({is_ok: () => true})), diff --git a/mobile/modules/island/src/facades/permissions.ts b/mobile/modules/island/src/facades/permissions.ts new file mode 100644 index 0000000000..5d0826a8e3 --- /dev/null +++ b/mobile/modules/island/src/facades/permissions.ts @@ -0,0 +1,116 @@ +/** + * permissions facade — `toolkit.permissions`: the raw OS-permission ops the runtime + * needs, mapped from feature keys to react-native-permissions. This is the LEAN + * surface (check/request/openSettings/requirementsForMiniapp) — the host keeps its + * own UI helpers (rationale dialogs, denied-warnings) on top; island stays free of + * i18n/theme/AlertUtils. + */ +import {Platform, Linking, PermissionsAndroid, type Permission as AndroidPermission} from "react-native" +import {check, request, PERMISSIONS, RESULTS, type Permission} from "react-native-permissions" + +import appRegistry from "../services/AppRegistry" +import type {AppletPermission} from "../types" + +/** OEM-facing feature keys (mapped to OS permissions per platform). */ +export const PermissionFeatures = { + MICROPHONE: "microphone", + CAMERA: "camera", + CALENDAR: "calendar", + LOCATION: "location", + BACKGROUND_LOCATION: "background_location", + BLUETOOTH: "bluetooth", + PHONE_STATE: "phone_state", + POST_NOTIFICATIONS: "post_notifications", +} as const + +const ANDROID = PermissionsAndroid.PERMISSIONS +const apiLevel = typeof Platform.Version === "number" ? Platform.Version : parseInt(String(Platform.Version), 10) + +function iosPerms(feature: string): Permission[] { + switch (feature) { + case PermissionFeatures.MICROPHONE: + return [PERMISSIONS.IOS.MICROPHONE] + case PermissionFeatures.CAMERA: + return [PERMISSIONS.IOS.CAMERA] + case PermissionFeatures.CALENDAR: + return [PERMISSIONS.IOS.CALENDARS] + case PermissionFeatures.LOCATION: + return [PERMISSIONS.IOS.LOCATION_WHEN_IN_USE] + case PermissionFeatures.BACKGROUND_LOCATION: + return [PERMISSIONS.IOS.LOCATION_ALWAYS] + case PermissionFeatures.BLUETOOTH: + return [PERMISSIONS.IOS.BLUETOOTH] + default: + return [] + } +} + +function androidPerms(feature: string): AndroidPermission[] { + switch (feature) { + case PermissionFeatures.MICROPHONE: + return [ANDROID.RECORD_AUDIO] + case PermissionFeatures.CAMERA: + return [ANDROID.CAMERA] + case PermissionFeatures.CALENDAR: + return [ANDROID.READ_CALENDAR, ANDROID.WRITE_CALENDAR] + case PermissionFeatures.LOCATION: + case PermissionFeatures.BACKGROUND_LOCATION: + return [ANDROID.ACCESS_FINE_LOCATION] + case PermissionFeatures.PHONE_STATE: + return [ANDROID.READ_PHONE_STATE] + case PermissionFeatures.BLUETOOTH: + return apiLevel >= 31 ? [ANDROID.BLUETOOTH_SCAN, ANDROID.BLUETOOTH_CONNECT, ANDROID.BLUETOOTH_ADVERTISE] : [] + case PermissionFeatures.POST_NOTIFICATIONS: + return apiLevel >= 33 ? [ANDROID.POST_NOTIFICATIONS] : [] + default: + return [] + } +} + +export const permissions = { + /** Is the given feature's OS permission currently granted? */ + async check(feature: string): Promise { + if (Platform.OS === "android") { + const perms = androidPerms(feature) + if (perms.length === 0) return true // nothing to ask on this OS/version + for (const p of perms) { + if (await PermissionsAndroid.check(p)) return true + } + return false + } + const perms = iosPerms(feature) + if (perms.length === 0) return true + for (const p of perms) { + const status = await check(p) + if (status !== RESULTS.GRANTED && status !== RESULTS.LIMITED) return false + } + return true + }, + + /** Request the given feature's OS permission. Resolves to whether it's granted. */ + async request(feature: string): Promise { + if (Platform.OS === "android") { + const perms = androidPerms(feature) + if (perms.length === 0) return true + const results = await PermissionsAndroid.requestMultiple(perms) + return perms.some((p) => results[p] === PermissionsAndroid.RESULTS.GRANTED) + } + const perms = iosPerms(feature) + if (perms.length === 0) return true + let granted = true + for (const p of perms) { + const result = await request(p) + if (result !== RESULTS.GRANTED && result !== RESULTS.LIMITED) granted = false + } + return granted + }, + + /** Open the OS settings app (so the user can grant a blocked permission). */ + openSettings: (): Promise => Linking.openSettings(), + + /** The permissions a given installed miniapp declares it needs. */ + async requirementsForMiniapp(packageName: string): Promise { + const apps = await appRegistry.getInstalledMiniapps() + return apps.find((a) => a.packageName === packageName)?.permissions ?? [] + }, +} diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 05fb6671ea..eeb789c91e 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -20,6 +20,7 @@ import {incidents} from "./facades/incidents" import {miniapps} from "./facades/miniapps" import {pairing} from "./facades/pairing" import {phoneNotifications} from "./facades/phoneNotifications" +import {permissions} from "./facades/permissions" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -60,6 +61,8 @@ export const toolkit = { pairing, /** Phone→glasses notification forwarding (Android). */ phoneNotifications, + /** OS permissions (check/request/openSettings + per-miniapp requirements). */ + permissions, display: { mirror: displayMirror, }, From c1d2407286af04a287968d5848d3c84242e2b143 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 20:27:06 -0700 Subject: [PATCH 30/80] island: add toolkit.notifications (inbound alerts island->host) + wire crashloop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NotificationsEmitter (a plain listener set — avoids the events typings gap) is the island->host alert channel; toolkit.notifications.onNotification(cb) subscribes. MentraJSRouter now emits a miniapp_crashloop notification alongside its existing onCrashloop host callback (non-breaking). version_incompatible + connection_failed_persistent land as those detectors are built (the emitter + IslandNotification kinds are ready). Emit build + 281 jest green. --- mobile/jest.setup.js | 3 ++ .../island/src/facades/notifications.ts | 15 ++++++ mobile/modules/island/src/index.ts | 1 + mobile/modules/island/src/island.ts | 3 ++ .../island/src/services/MentraJSRouter.ts | 2 + .../src/services/NotificationsEmitter.ts | 47 +++++++++++++++++++ 6 files changed, 71 insertions(+) create mode 100644 mobile/modules/island/src/facades/notifications.ts create mode 100644 mobile/modules/island/src/services/NotificationsEmitter.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index 5f252634ff..f5ee9111f6 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -347,6 +347,9 @@ jest.mock("@mentra/island", () => { setView: jest.fn(), }, }, + notifications: { + onNotification: jest.fn(() => () => {}), + }, permissions: { check: jest.fn(() => Promise.resolve(false)), request: jest.fn(() => Promise.resolve(false)), diff --git a/mobile/modules/island/src/facades/notifications.ts b/mobile/modules/island/src/facades/notifications.ts new file mode 100644 index 0000000000..bfc887a8ef --- /dev/null +++ b/mobile/modules/island/src/facades/notifications.ts @@ -0,0 +1,15 @@ +/** + * notifications facade — `toolkit.notifications`: inbound alerts island→host. The + * OEM subscribes and renders them however it likes (modal, toast, banner). Today: + * miniapp crashloop; version-incompatible miniapp and persistent connection loss + * land as their detectors are wired into the emitter. + */ +import {islandNotifications, type IslandNotification} from "../services/NotificationsEmitter" + +export type {IslandNotification, IslandNotificationKind} from "../services/NotificationsEmitter" + +export const notifications = { + /** Subscribe to island-raised notifications; returns an unsubscribe. */ + onNotification: (cb: (notification: IslandNotification) => void): (() => void) => + islandNotifications.subscribe(cb), +} diff --git a/mobile/modules/island/src/index.ts b/mobile/modules/island/src/index.ts index 0932bde97e..6ba87bf6b7 100644 --- a/mobile/modules/island/src/index.ts +++ b/mobile/modules/island/src/index.ts @@ -80,6 +80,7 @@ export { // The namespaced OEM-facing toolkit API (the "(A) host API"). Additive — grows one // facade at a time alongside the flat exports below. See ./island. export {toolkit} from "./island" +export type {IslandNotification, IslandNotificationKind} from "./facades/notifications" export type {WifiSearchResult} from "./facades/glassesWifi" export type {DisplayMirrorEvent} from "./facades/displayMirror" export type { diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index eeb789c91e..70b60d5e04 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -21,6 +21,7 @@ import {miniapps} from "./facades/miniapps" import {pairing} from "./facades/pairing" import {phoneNotifications} from "./facades/phoneNotifications" import {permissions} from "./facades/permissions" +import {notifications} from "./facades/notifications" import {useGlassesStore} from "./stores/glasses" import {useDisplayStore} from "./stores/display" import {useCoreStore} from "./stores/core" @@ -63,6 +64,8 @@ export const toolkit = { phoneNotifications, /** OS permissions (check/request/openSettings + per-miniapp requirements). */ permissions, + /** Inbound alerts island→host (crashloop, version-incompatible, connection loss). */ + notifications, display: { mirror: displayMirror, }, diff --git a/mobile/modules/island/src/services/MentraJSRouter.ts b/mobile/modules/island/src/services/MentraJSRouter.ts index 9eea6c6315..dff996293a 100644 --- a/mobile/modules/island/src/services/MentraJSRouter.ts +++ b/mobile/modules/island/src/services/MentraJSRouter.ts @@ -41,6 +41,7 @@ import type localMiniappRuntime from "./LocalMiniappRuntime" import type {InstalledMiniappManifest} from "./LocalMiniappRuntime" import type {MentraJSCrashController} from "./MentraJSCrashController" import {MentraJSLogRingBuffer, MentraJSLogThrottle, redactSecrets} from "./MentraJSLogPipeline" +import {islandNotifications} from "./NotificationsEmitter" import type {MentraUIRouter} from "./MentraUIRouter" import {miniappRunningRegistry} from "./MiniappRunningRegistry" @@ -288,6 +289,7 @@ export class MentraJSRouter { const outcome = controller.onCrash(packageName, reason) if (outcome.surfaceCrashloopBanner) { this.onCrashloop?.(packageName, reason) + islandNotifications.emit({kind: "miniapp_crashloop", packageName, reason, timestamp: Date.now()}) return } if (outcome.showRestartToast) { diff --git a/mobile/modules/island/src/services/NotificationsEmitter.ts b/mobile/modules/island/src/services/NotificationsEmitter.ts new file mode 100644 index 0000000000..49da4a7edf --- /dev/null +++ b/mobile/modules/island/src/services/NotificationsEmitter.ts @@ -0,0 +1,47 @@ +/** + * Island notifications emitter — the island→host alert channel behind + * `toolkit.notifications`. Island-detected conditions the host should surface + * (miniapp crashloop, version-incompatible miniapp, persistent connection loss) + * are pushed here; the facade fans them out to host subscribers. + * + * A plain listener set (not Node's EventEmitter) to avoid the `events` typings + * gap in island's standalone build. + */ +export type IslandNotificationKind = + | "miniapp_crashloop" + | "version_incompatible" + | "connection_failed_persistent" + +export interface IslandNotification { + kind: IslandNotificationKind + /** The miniapp this concerns, when applicable. */ + packageName?: string + /** Epoch millis when raised. */ + timestamp: number + /** Human-readable summary the host can show. */ + reason: string + /** Kind-specific extras (versions, attempt counts, …). */ + metadata?: Record +} + +const listeners = new Set<(n: IslandNotification) => void>() + +export const islandNotifications = { + /** Raise a notification to all host subscribers. */ + emit(notification: IslandNotification): void { + for (const l of listeners) { + try { + l(notification) + } catch (err) { + console.warn(`islandNotifications: listener threw: ${(err as Error)?.message ?? err}`) + } + } + }, + /** Subscribe to island notifications; returns an unsubscribe. */ + subscribe(cb: (n: IslandNotification) => void): () => void { + listeners.add(cb) + return () => { + listeners.delete(cb) + } + }, +} From c6c2cf2f2ba994dd88257c06a01fe189dee2707d Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 20:28:09 -0700 Subject: [PATCH 31/80] docs(oem): document permissions, phoneNotifications, notifications + glasses long-tail/controller --- docs/glasses-oems/toolkit.mdx | 47 ++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index 4dfa583524..d445a20310 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -72,9 +72,17 @@ discrete input events. ### Connection ```ts -await toolkit.glasses.connectDefault() // connect the last-paired glasses +await toolkit.glasses.connectDefault() // connect the last-paired glasses +await toolkit.glasses.connect(device) // connect a specific (discovered) device +await toolkit.glasses.connectSimulated() // built-in simulated glasses (dev) +await toolkit.glasses.setDefault(device) // make it the connectDefault() target await toolkit.glasses.disconnect() await toolkit.glasses.forget() + +// optional ring controller: +await toolkit.glasses.controller.connectDefault() +await toolkit.glasses.controller.disconnect() +await toolkit.glasses.controller.forget() ``` ### Status & info (read-models) @@ -237,6 +245,43 @@ toolkit.dev.savedUrls() toolkit.dev.reconnectCloud() ``` +## `toolkit.permissions` + +OS permissions — the raw check/request ops (your UI adds the rationale dialogs). + +```ts +await toolkit.permissions.check("microphone") // granted? +await toolkit.permissions.request("location") // -> granted? +await toolkit.permissions.openSettings() // open OS settings +await toolkit.permissions.requirementsForMiniapp(pkg) // -> AppletPermission[] +``` + +Feature keys: `microphone · camera · calendar · location · background_location · bluetooth · phone_state · post_notifications`. + +## `toolkit.phoneNotifications` + +Forward phone notifications to the glasses, with a per-app blocklist. **Android-only** +(getters return safe defaults on iOS). + +```ts +toolkit.phoneNotifications.enabled(); toolkit.phoneNotifications.setEnabled(true) +await toolkit.phoneNotifications.installedApps() // [{packageName, appName, icon}] +toolkit.phoneNotifications.blocklist(); toolkit.phoneNotifications.setBlocklist([...]) +await toolkit.phoneNotifications.hasListenerPermission() +await toolkit.phoneNotifications.requestListenerPermission() +``` + +## `toolkit.notifications` + +Inbound alerts island→host — conditions island detects, your UI renders. + +```ts +const off = toolkit.notifications.onNotification((n) => { + // n.kind: "miniapp_crashloop" | "version_incompatible" | "connection_failed_persistent" + // n.packageName, n.reason, n.timestamp, n.metadata +}) +``` + ## `toolkit.stores.*` — escape hatch (not the OEM contract) The raw device-state stores are also exposed under `toolkit.stores` From f86fce2f3abccad64fb9d4ac14592f3039f7b85b Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 22:18:29 -0700 Subject: [PATCH 32/80] island: fix iOS background-location request + correct an init comment (review findings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial self-review of the whole PR found no boot-breaking bugs (circular imports are call-time-only; persistence carries over on the same default MMKV file; the cloud client constructs exactly once before services read it; shims are complete). Two real fixes: - permissions: iOS BACKGROUND_LOCATION now requests [LOCATION_WHEN_IN_USE, LOCATION_ALWAYS] (iOS won't grant ALWAYS without WHEN_IN_USE first) — matches the host PermissionsUtils; a cold request previously couldn't escalate. - island.ts: corrected the backwards comment on which init() call is the no-op. --- mobile/modules/island/src/facades/permissions.ts | 4 +++- mobile/modules/island/src/island.ts | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mobile/modules/island/src/facades/permissions.ts b/mobile/modules/island/src/facades/permissions.ts index 5d0826a8e3..eb9c69f83e 100644 --- a/mobile/modules/island/src/facades/permissions.ts +++ b/mobile/modules/island/src/facades/permissions.ts @@ -37,7 +37,9 @@ function iosPerms(feature: string): Permission[] { case PermissionFeatures.LOCATION: return [PERMISSIONS.IOS.LOCATION_WHEN_IN_USE] case PermissionFeatures.BACKGROUND_LOCATION: - return [PERMISSIONS.IOS.LOCATION_ALWAYS] + // iOS won't grant ALWAYS without WHEN_IN_USE first — request both (matches + // the host PermissionsUtils mapping), else a cold request can't escalate. + return [PERMISSIONS.IOS.LOCATION_WHEN_IN_USE, PERMISSIONS.IOS.LOCATION_ALWAYS] case PermissionFeatures.BLUETOOTH: return [PERMISSIONS.IOS.BLUETOOTH] default: diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 70b60d5e04..8a45c640e0 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -37,8 +37,9 @@ export const toolkit = { async start() { await bootstrapStart() // Construct + connect the cloud client so the documented configure()+start() - // lifecycle yields a live toolkit.session. Idempotent — the Mentra app's - // explicit cloudClient.init() is then a no-op. + // lifecycle yields a live toolkit.session for OEMs. Idempotent — when the + // Mentra app has already called cloudClient.init() (synchronously, right after + // start()), THIS call is the no-op. cloudClientService.init() }, /** Stop the runtime: tear down the cloud client + mark stopped. */ From 106581b6573d1e4c13f97aeb08610c082eeaf6e5 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 16 Jun 2026 23:23:10 -0700 Subject: [PATCH 33/80] docs(oem): fix Runtime Toolkit reference drift - status Note: the full Phase-1 surface is shipped (only OTA + gallery deferred), not 'settings/permissions/session landing'. - add the missing toolkit.miniapps section (lifecycle; WebView is a host component). - toolkit.stores.*: add cloudClientStatus + settings to the list. - replace the misleading 'adapter seam' section: the only OEM seam is auth via configure(); configureRuntime is transitional Mentra-app internals OEMs never touch. - frontmatter description: list the full surface. - pairing.pair/setDefault take a discovered device (from searchResults), not a bare id. --- docs/glasses-oems/toolkit.mdx | 60 ++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index d445a20310..0089fa8de0 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -1,14 +1,13 @@ --- title: "Runtime Toolkit" -description: "The toolkit.* API an OEM host calls to drive MentraOS on its glasses — connection, status, wifi, speech, and display, over a device-agnostic runtime." +description: "The toolkit.* API an OEM host calls to drive MentraOS on its glasses — connection, pairing, device + user settings, wifi, speech, display, cloud session, miniapps, permissions, notifications, and incidents, over a device-agnostic runtime." --- -**Status: preview (Phase 1).** The runtime toolkit is being built domain by domain. -The surface documented here is shipped and stable; more domains (settings, -permissions, gallery, session) are landing behind the same patterns. The runtime is -not yet packaged for external install — reach out to -[help@mentra.glass](mailto:help@mentra.glass) to integrate today. +**Status: preview (Phase 1).** The runtime toolkit covers the full Phase-1 surface +documented below. Firmware **OTA** and the media **gallery** are intentionally +deferred to a later phase. The runtime is not yet packaged for external install — +reach out to [help@mentra.glass](mailto:help@mentra.glass) to integrate today. ## What it is @@ -151,8 +150,8 @@ toolkit.pairing.scan() // start scanning toolkit.pairing.scanning() // is a scan in progress? toolkit.pairing.searchResults() // discovered devices (snapshot) const off = toolkit.pairing.onFound((r) => { … }) // subscribe; returns unsubscribe -await toolkit.pairing.pair(deviceId) // connect to a discovered device -await toolkit.pairing.setDefault(deviceId) // make it the connectDefault() target +await toolkit.pairing.pair(device) // connect to a discovered device (from searchResults) +await toolkit.pairing.setDefault(device) // make it the connectDefault() target toolkit.pairing.onPairFailure((e) => { … }) toolkit.pairing.onGlassesNotReady((e) => { … }) ``` @@ -282,11 +281,31 @@ const off = toolkit.notifications.onNotification((n) => { }) ``` +## `toolkit.miniapps` + +Miniapp lifecycle. The miniapp **WebView is a host component** — island ships the +bridge primitives (`buildMiniappGlobalsScript`, `buildMentraUiShim`, the MentraJS +router), exported from `@mentra/island` directly; your app mounts the `` and +wires it to those. This facade is the lifecycle half. + +```ts +toolkit.miniapps.list() // installed miniapps (snapshot) +const off = toolkit.miniapps.onChanged((apps) => { … }) // subscribe; returns unsubscribe +await toolkit.miniapps.refresh() // re-fetch the installed list +await toolkit.miniapps.start(app, opts?) // start + foreground a miniapp +await toolkit.miniapps.stop(packageName) +await toolkit.miniapps.setForeground(packageName) +toolkit.miniapps.clearForeground() +await toolkit.miniapps.stopAll() +await toolkit.miniapps.install(url, opts?) +await toolkit.miniapps.uninstall(packageName, version?) +``` + ## `toolkit.stores.*` — escape hatch (not the OEM contract) The raw device-state stores are also exposed under `toolkit.stores` -(`glasses`, `display`, `core`, `connection`, `gallerySync`) so the first-party Mentra -app can keep using them directly during migration. +(`glasses`, `display`, `core`, `connection`, `gallerySync`, `cloudClientStatus`, +`settings`) so the first-party Mentra app can keep using them directly during migration. `toolkit.stores.*` is a **Mentra-app convenience, not the OEM contract.** It exposes @@ -294,12 +313,17 @@ the internal store shape, which can change. OEMs should use the typed facades ab prefer a facade wherever one exists. -## How a host wires in (the adapter seam) +## What the host provides + +The boundary is small and explicit: the host provides its **UI**, its **login**, and +optional **config** — island owns the entire runtime. The only required seam is +`auth.getSubjectToken` (passed to `configure()`); island owns token exchange, refresh, +storage, the cloud client, glasses/BLE, settings, speech, display, and the miniapp +runtime from there. -A few runtime services need things only the host can provide (persistent storage, the -backend client, audio playback). The toolkit never imports host code; instead it -declares small typed **ports**, and the host injects **adapters** at boot through a -single dependency-injection seam. The host implements the interface; the toolkit owns -the behavior. OEM hosts implement the same ports with their own backing — that is what -keeps the runtime device- and host-agnostic. (This seam is what the remaining -domains — settings, permissions, gallery — land behind.) + +You may see internal `configureRuntime(...)` wiring in the first-party Mentra app — +that's **transitional** scaffolding for migrating Mentra's own screens, not part of +the OEM contract. It deletes itself as each domain lands in island. An OEM only ever +calls `configure({auth, config?})` + `start()`. + From 5142fc49d15056fc42cf0209300509f5439c1e14 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 09:23:37 -0700 Subject: [PATCH 34/80] island: fix 2 review-bot findings (glasses.settings device sync, permissions group grant) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HIGH: glasses.settings.set() now actually reaches the glasses for ANY host. The device-settings -> BLE sync was a host MantleManager subscription, so an OEM's set() only updated the store. Moved it into island (GlassesSettingsSync), started by toolkit.start(); removed the MantleManager duplicate (no double-push). The jest mock's toolkit.start/stop now drive the real sync so MantleManager's settings-forwarding tests exercise it where it lives. - MED: toolkit.permissions check/request on Android required only ANY mapped permission in a group; now require EVERY one (bluetooth = scan+connect+advertise, calendar = read+write) — matches the iOS branch. Note: the on-connect full push (all device settings when glasses connect) still lives in the Mentra app's MantleManager — moving that into island for OEMs is a follow-up (needs a glasses-connect hook); it does not affect the Mentra app's first boot. Typecheck + island emit build + 281 jest green. --- mobile/jest.setup.js | 13 ++++++- .../modules/island/src/facades/permissions.ts | 9 +++-- mobile/modules/island/src/island.ts | 10 ++++- .../src/services/GlassesSettingsSync.ts | 37 +++++++++++++++++++ mobile/src/services/MantleManager.ts | 20 ++-------- 5 files changed, 65 insertions(+), 24 deletions(-) create mode 100644 mobile/modules/island/src/services/GlassesSettingsSync.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index f5ee9111f6..d792d1a9b1 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -246,6 +246,9 @@ jest.mock("@mentra/island", () => { // before the move, so requireActual preserves that exact behavior. const realSettings = jest.requireActual("./modules/island/src/stores/settings") const realRestComms = jest.requireActual("./modules/island/src/services/RestComms") + // toolkit.start() starts the island-owned device-settings -> glasses BLE sync; use + // the real one so its behavior is exercised where it now lives (not MantleManager). + const realGlassesSettingsSync = jest.requireActual("./modules/island/src/services/GlassesSettingsSync") const appStatusState = { apps: [], refresh: jest.fn(), @@ -281,8 +284,14 @@ jest.mock("@mentra/island", () => { // jest.fn()s so host/screen tests can assert delegation without native btsdk. toolkit: { configure: jest.fn(), - start: jest.fn(() => Promise.resolve()), - stop: jest.fn(() => Promise.resolve()), + start: jest.fn(() => { + realGlassesSettingsSync.startGlassesSettingsSync() + return Promise.resolve() + }), + stop: jest.fn(() => { + realGlassesSettingsSync.stopGlassesSettingsSync() + return Promise.resolve() + }), glasses: { connectDefault: jest.fn(() => Promise.resolve()), disconnect: jest.fn(() => Promise.resolve()), diff --git a/mobile/modules/island/src/facades/permissions.ts b/mobile/modules/island/src/facades/permissions.ts index eb9c69f83e..de4c5b29c5 100644 --- a/mobile/modules/island/src/facades/permissions.ts +++ b/mobile/modules/island/src/facades/permissions.ts @@ -75,10 +75,12 @@ export const permissions = { if (Platform.OS === "android") { const perms = androidPerms(feature) if (perms.length === 0) return true // nothing to ask on this OS/version + // Every permission in the group must be granted (e.g. bluetooth = + // scan+connect+advertise, calendar = read+write) — matches the iOS branch. for (const p of perms) { - if (await PermissionsAndroid.check(p)) return true + if (!(await PermissionsAndroid.check(p))) return false } - return false + return true } const perms = iosPerms(feature) if (perms.length === 0) return true @@ -95,7 +97,8 @@ export const permissions = { const perms = androidPerms(feature) if (perms.length === 0) return true const results = await PermissionsAndroid.requestMultiple(perms) - return perms.some((p) => results[p] === PermissionsAndroid.RESULTS.GRANTED) + // Granted only when EVERY permission in the group was granted. + return perms.every((p) => results[p] === PermissionsAndroid.RESULTS.GRANTED) } const perms = iosPerms(feature) if (perms.length === 0) return true diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 8a45c640e0..1298a8a4c9 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -10,6 +10,7 @@ */ import {configure, start as bootstrapStart, stop as bootstrapStop} from "./runtime/bootstrap" import {cloudClientService} from "./services/CloudClientService" +import {startGlassesSettingsSync, stopGlassesSettingsSync} from "./services/GlassesSettingsSync" import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {speech} from "./facades/speech" @@ -33,7 +34,8 @@ import {useSettingsStore} from "./stores/settings" export const toolkit = { /** Front door — hand island auth + config, then start/stop the runtime. */ configure, - /** Start the runtime: mark started + construct/connect the cloud client. */ + /** Start the runtime: mark started + construct/connect the cloud client + begin + * syncing device settings to the glasses. */ async start() { await bootstrapStart() // Construct + connect the cloud client so the documented configure()+start() @@ -41,9 +43,13 @@ export const toolkit = { // Mentra app has already called cloudClient.init() (synchronously, right after // start()), THIS call is the no-op. cloudClientService.init() + // Push device-setting changes to the glasses for ANY host, so + // toolkit.glasses.settings.set() reaches the device (not just the Mentra app). + startGlassesSettingsSync() }, - /** Stop the runtime: tear down the cloud client + mark stopped. */ + /** Stop the runtime: tear down the settings sync + cloud client + mark stopped. */ async stop() { + stopGlassesSettingsSync() cloudClientService.stop() await bootstrapStop() }, diff --git a/mobile/modules/island/src/services/GlassesSettingsSync.ts b/mobile/modules/island/src/services/GlassesSettingsSync.ts new file mode 100644 index 0000000000..a4dedc4aa2 --- /dev/null +++ b/mobile/modules/island/src/services/GlassesSettingsSync.ts @@ -0,0 +1,37 @@ +/** + * Glasses settings sync — island-owned. Pushes `BLUETOOTH_SETTING_KEYS` changes to + * the connected glasses over the bluetooth-sdk, so `toolkit.glasses.settings.set()` + * (and any other settings-store write) actually reaches the device for ANY host — + * not just the first-party Mentra app, where this used to live in MantleManager. + * + * Started by `toolkit.start()`. Idempotent. + */ +import {shallow} from "zustand/shallow" + +import BluetoothSdk from "../../../bluetooth-sdk/build/_internal" +import {useSettingsStore} from "../stores/settings" + +let unsubscribe: (() => void) | null = null + +export function startGlassesSettingsSync(): void { + if (unsubscribe) return + unsubscribe = useSettingsStore.subscribe( + (state) => state.getBluetoothSettings(), + (settings: Record, previous: Record) => { + // Push only the keys that changed (matching the prior MantleManager sync). + const changed: Record = {} + for (const key in settings) { + if (settings[key] !== previous[key]) changed[key] = settings[key] + } + if (Object.keys(changed).length > 0) { + BluetoothSdk.updateBluetoothSettings(changed) + } + }, + {equalityFn: shallow}, + ) +} + +export function stopGlassesSettingsSync(): void { + unsubscribe?.() + unsubscribe = null +} diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index c1c804ec12..3459c8ce02 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -532,23 +532,9 @@ class MantleManager { {equalityFn: shallow}, ) - // Subscribe to settings forwarded to the Bluetooth SDK. - useSettingsStore.subscribe( - (state) => state.getBluetoothSettings(), - (state: Record, previousState: Record) => { - const bluetoothSettingsObj: Record = {} - - for (const key in state) { - const k = key as keyof Record - if (state[k] !== previousState[k]) { - bluetoothSettingsObj[k] = state[k] as any - } - } - // console.log("MANTLE: Bluetooth settings changed", bluetoothSettingsObj) - BluetoothSdk.updateBluetoothSettings(bluetoothSettingsObj) - }, - {equalityFn: shallow}, - ) + // (Device-settings -> glasses BLE sync now lives in island's + // GlassesSettingsSync, started by toolkit.start(), so toolkit.glasses.settings.set() + // reaches the device for any host. Removed here to avoid a double-push.) useSettingsStore.subscribe( (state) => ({ From 73dfd0e749eac3c5f510333ebe0714cf2165f09e Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 09:54:50 -0700 Subject: [PATCH 35/80] toolkit.glasses.settings: scope available() to real settings + doc keys table + rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - glasses.settings.available() now excludes device/controller IDENTITY (set by pairing) and internal RUNTIME flags (local_stt_fallback_active, offline_captions_running) — not just auth — so it returns only the ~26 user-tunable device settings. - docs: add a full keys table (type/default/values) to toolkit.glasses.settings; note that identity/runtime are read via glasses.info(), not settings. - docs: rename 'Runtime Toolkit' -> 'Mentra OEM Integration Toolkit' everywhere (title + body). Never 'runtime toolkit'. Typecheck + island emit build + 281 jest green. --- docs/glasses-oems/toolkit.mdx | 48 +++++++++++++++++-- .../island/src/facades/glassesSettings.ts | 23 +++++++-- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index 0089fa8de0..dd0ca1344f 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -1,10 +1,10 @@ --- -title: "Runtime Toolkit" +title: "Mentra OEM Integration Toolkit" description: "The toolkit.* API an OEM host calls to drive MentraOS on its glasses — connection, pairing, device + user settings, wifi, speech, display, cloud session, miniapps, permissions, notifications, and incidents, over a device-agnostic runtime." --- -**Status: preview (Phase 1).** The runtime toolkit covers the full Phase-1 surface +**Status: preview (Phase 1).** The Mentra OEM Integration Toolkit covers the full Phase-1 surface documented below. Firmware **OTA** and the media **gallery** are intentionally deferred to a later phase. The runtime is not yet packaged for external install — reach out to [help@mentra.glass](mailto:help@mentra.glass) to integrate today. @@ -12,8 +12,8 @@ reach out to [help@mentra.glass](mailto:help@mentra.glass) to integrate today. ## What it is -The **runtime toolkit** is the API an OEM host app (the phone-side app that talks to -your glasses) calls to run MentraOS. It ships as the `@mentra/island` module and +The **Mentra OEM Integration Toolkit** is the API an OEM host app (the phone-side app +that talks to your glasses) calls to run MentraOS. It ships as the `@mentra/island` module and exposes a single namespaced object, **`toolkit`**: ```ts @@ -137,9 +137,47 @@ toolkit.glasses.settings.get(key) toolkit.glasses.settings.set(key, value) // also pushes to the device toolkit.glasses.settings.onChanged(key, (v) => { … }) toolkit.glasses.settings.descriptor(key) -toolkit.glasses.settings.available() // the device-setting keys +toolkit.glasses.settings.available() // the device-setting keys (live list) ``` +`available()` is the authoritative list at runtime (it varies by model/capabilities). +The current keys, their types, and defaults: + +| Key | Type | Default | Notes / values | +|---|---|---|---| +| `brightness` | number | `50` | display brightness, `0`–`100` | +| `auto_brightness` | boolean | `true` | auto-adjust brightness | +| `head_up_angle` | number | `45` | head-up activation angle (degrees) | +| `screen_disabled` | boolean | `false` | turn the display off | +| `contextual_dashboard` | boolean | `true` | show the dashboard on head-up | +| `dashboard_height` | number | `4` | dashboard vertical position | +| `dashboard_depth` | number | `2` | dashboard distance | +| `use_native_dashboard` | boolean | `true` | native dashboard vs JS | +| `menu_apps` | `string[] \| null` | `null` | button-menu app package names | +| `button_photo_size` | enum | `"max"` | `small` \| `medium` \| `large` \| `max` | +| `button_video_settings` | object | `{ width: 1920, height: 1080, fps: 30 }` | button video recording | +| `button_camera_led` | boolean | `true` | capture LED indicator | +| `button_max_recording_time` | number | `10` | max button video length (seconds) | +| `camera_fov` | object | `{ fov: 118, roi_position: 0 }` | camera field of view | +| `preferred_mic` | enum | `"auto"` | `auto` \| `glasses` \| `phone` | +| `lc3_frame_size` | enum (bytes) | `60` | `20` \| `40` \| `60` | +| `sensing_enabled` | boolean | `true` | onboard sensors | +| `voice_activity_detection_enabled` | boolean | `true` | voice-activity detection | +| `power_saving_mode` | boolean | `false` | low-power mode | +| `offline_mode` | boolean | `false` | offline operation | +| `gallery_mode` | boolean | `true` | on-device gallery | +| `calendar_events` | array | `[]` | synced calendar events | +| `twelve_hour_time` | boolean | `true` | 12-hour vs 24-hour clock | +| `metric_system` | boolean | `false` | metric vs imperial | +| `nex_chinese_captions` | boolean | `false` | Mentra Nex: Chinese captions | +| `nex_audio_playback` | boolean | `false` | Mentra Nex: audio playback | + + +Device **identity** (paired-device id/name/address, controller address) and internal +**runtime flags** (e.g. STT-fallback state) are intentionally *not* in this surface — +read the connected device via `toolkit.glasses.info()`. + + ## `toolkit.pairing` First-time glasses discovery + pairing. (Reconnecting the already-paired default is diff --git a/mobile/modules/island/src/facades/glassesSettings.ts b/mobile/modules/island/src/facades/glassesSettings.ts index c4fbd7ac6c..c541ddf08a 100644 --- a/mobile/modules/island/src/facades/glassesSettings.ts +++ b/mobile/modules/island/src/facades/glassesSettings.ts @@ -10,9 +10,26 @@ */ import {useSettingsStore, SETTINGS, BLUETOOTH_SETTING_KEYS} from "../stores/settings" -// Internal sync keys carried on BLUETOOTH_SETTING_KEYS for the device handshake — -// NOT user-tunable device settings, so kept out of the OEM-facing surface. -const INTERNAL_KEYS = new Set([SETTINGS.core_token.key, SETTINGS.auth_email.key]) +// Keys carried on BLUETOOTH_SETTING_KEYS for the device handshake that are NOT +// user-tunable settings — auth, device/controller IDENTITY (set by pairing), and +// internal RUNTIME flags. Kept out of the OEM-facing `available()` surface. +const INTERNAL_KEYS = new Set([ + // auth + SETTINGS.core_token.key, + SETTINGS.auth_email.key, + // glasses + controller identity (managed by pairing, not the user) + SETTINGS.pending_wearable.key, + SETTINGS.default_wearable.key, + SETTINGS.device_name.key, + SETTINGS.device_address.key, + SETTINGS.pending_controller.key, + SETTINGS.default_controller.key, + SETTINGS.controller_device_name.key, + SETTINGS.controller_address.key, + // internal runtime flags + SETTINGS.local_stt_fallback_active.key, + SETTINGS.offline_captions_running.key, +]) const DEVICE_KEYS = BLUETOOTH_SETTING_KEYS.filter((k) => !INTERNAL_KEYS.has(k)) export const glassesSettings = { From 6cc5b88d71446987834c98ed6fe7ab637af15ad1 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 10:15:41 -0700 Subject: [PATCH 36/80] island: phoneNotifications config sync to native is island-owned (review-bot fix) Same class as the glasses.settings fix: toolkit.phoneNotifications.setEnabled/setBlocklist updated the store but the CrustModule.setNotificationConfig push lived in a host MantleManager subscription, so an OEM's toggle never reached the listener. Moved it into island's PhoneNotificationsSync (initial push + change-subscription), started by toolkit.start(); removed the MantleManager subscription + initial call + the dead method. jest mock start/stop drive it so the notification-sync tests exercise it where it lives. Typecheck + emit + 281 jest green. --- mobile/jest.setup.js | 3 ++ mobile/modules/island/src/island.ts | 4 ++ .../src/services/PhoneNotificationsSync.ts | 44 +++++++++++++++++++ mobile/src/services/MantleManager.ts | 32 +++----------- 4 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 mobile/modules/island/src/services/PhoneNotificationsSync.ts diff --git a/mobile/jest.setup.js b/mobile/jest.setup.js index d792d1a9b1..05ffc90ecc 100644 --- a/mobile/jest.setup.js +++ b/mobile/jest.setup.js @@ -249,6 +249,7 @@ jest.mock("@mentra/island", () => { // toolkit.start() starts the island-owned device-settings -> glasses BLE sync; use // the real one so its behavior is exercised where it now lives (not MantleManager). const realGlassesSettingsSync = jest.requireActual("./modules/island/src/services/GlassesSettingsSync") + const realPhoneNotificationsSync = jest.requireActual("./modules/island/src/services/PhoneNotificationsSync") const appStatusState = { apps: [], refresh: jest.fn(), @@ -286,10 +287,12 @@ jest.mock("@mentra/island", () => { configure: jest.fn(), start: jest.fn(() => { realGlassesSettingsSync.startGlassesSettingsSync() + realPhoneNotificationsSync.startPhoneNotificationsSync() return Promise.resolve() }), stop: jest.fn(() => { realGlassesSettingsSync.stopGlassesSettingsSync() + realPhoneNotificationsSync.stopPhoneNotificationsSync() return Promise.resolve() }), glasses: { diff --git a/mobile/modules/island/src/island.ts b/mobile/modules/island/src/island.ts index 1298a8a4c9..cd83ab761c 100644 --- a/mobile/modules/island/src/island.ts +++ b/mobile/modules/island/src/island.ts @@ -11,6 +11,7 @@ import {configure, start as bootstrapStart, stop as bootstrapStop} from "./runtime/bootstrap" import {cloudClientService} from "./services/CloudClientService" import {startGlassesSettingsSync, stopGlassesSettingsSync} from "./services/GlassesSettingsSync" +import {startPhoneNotificationsSync, stopPhoneNotificationsSync} from "./services/PhoneNotificationsSync" import {glasses} from "./facades/glasses" import {displayMirror} from "./facades/displayMirror" import {speech} from "./facades/speech" @@ -46,10 +47,13 @@ export const toolkit = { // Push device-setting changes to the glasses for ANY host, so // toolkit.glasses.settings.set() reaches the device (not just the Mentra app). startGlassesSettingsSync() + // Same for phone-notification config -> the native listener (Android). + startPhoneNotificationsSync() }, /** Stop the runtime: tear down the settings sync + cloud client + mark stopped. */ async stop() { stopGlassesSettingsSync() + stopPhoneNotificationsSync() cloudClientService.stop() await bootstrapStop() }, diff --git a/mobile/modules/island/src/services/PhoneNotificationsSync.ts b/mobile/modules/island/src/services/PhoneNotificationsSync.ts new file mode 100644 index 0000000000..a43603345c --- /dev/null +++ b/mobile/modules/island/src/services/PhoneNotificationsSync.ts @@ -0,0 +1,44 @@ +/** + * Phone-notifications sync — island-owned. Pushes the notification-forwarding + * config (`notifications_enabled` + `notifications_blocklist`) to the native + * NotificationListener via `CrustModule.setNotificationConfig`, so + * `toolkit.phoneNotifications.setEnabled()/setBlocklist()` actually reach the + * listener for ANY host — not just the Mentra app, where this used to live in + * MantleManager. Android-only; a no-op elsewhere. Started by `toolkit.start()`. + */ +import {shallow} from "zustand/shallow" +import CrustModule from "crust" + +import {useSettingsStore, SETTINGS} from "../stores/settings" + +let unsubscribe: (() => void) | null = null + +function pushConfig(): void { + const s = useSettingsStore.getState() + const enabled = Boolean(s.getSetting(SETTINGS.notifications_enabled.key)) + const blocklist = s.getSetting(SETTINGS.notifications_blocklist.key) + // Unconditional (matches the prior MantleManager sync) — the listener is + // Android-only, but CrustModule.setNotificationConfig is a safe no-op elsewhere; + // the Android-only gating lives on the facade's setEnabled/setBlocklist. + CrustModule.setNotificationConfig(enabled, Array.isArray(blocklist) ? blocklist : []).catch((err) => + console.warn(`PhoneNotificationsSync: setNotificationConfig failed: ${(err as Error)?.message ?? err}`), + ) +} + +export function startPhoneNotificationsSync(): void { + if (unsubscribe) return + pushConfig() // initial sync so the listener has the current config + unsubscribe = useSettingsStore.subscribe( + (state) => ({ + enabled: state.getSetting(SETTINGS.notifications_enabled.key), + blocklist: state.getSetting(SETTINGS.notifications_blocklist.key), + }), + () => pushConfig(), + {equalityFn: shallow}, + ) +} + +export function stopPhoneNotificationsSync(): void { + unsubscribe?.() + unsubscribe = null +} diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index 3459c8ce02..fb2cc129e4 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -343,7 +343,8 @@ class MantleManager { // settings are now in native; safe to attempt auto-connect attemptReconnectToDefaultWearable() }, 1000) - await this.syncNotificationSettingsToCrust() + // (Initial notification-config push now happens in island's + // PhoneNotificationsSync, started by toolkit.start().) this.initServices() this.initMiniapps() @@ -462,16 +463,6 @@ class MantleManager { } } - private async syncNotificationSettingsToCrust() { - const settings = useSettingsStore.getState() - const notificationsEnabled = Boolean(settings.getSetting(SETTINGS.notifications_enabled.key)) - const notificationsBlocklist = settings.getSetting(SETTINGS.notifications_blocklist.key) - await CrustModule.setNotificationConfig( - notificationsEnabled, - Array.isArray(notificationsBlocklist) ? notificationsBlocklist : [], - ) - } - private async setupPeriodicTasks() { this.sendCalendarEvents() // Calendar sync every hour @@ -532,20 +523,11 @@ class MantleManager { {equalityFn: shallow}, ) - // (Device-settings -> glasses BLE sync now lives in island's - // GlassesSettingsSync, started by toolkit.start(), so toolkit.glasses.settings.set() - // reaches the device for any host. Removed here to avoid a double-push.) - - useSettingsStore.subscribe( - (state) => ({ - notificationsEnabled: state.getSetting(SETTINGS.notifications_enabled.key), - notificationsBlocklist: state.getSetting(SETTINGS.notifications_blocklist.key), - }), - async () => { - await this.syncNotificationSettingsToCrust() - }, - {equalityFn: shallow}, - ) + // (Device-settings -> glasses BLE sync AND phone-notification config -> the + // native listener now live in island's GlassesSettingsSync / PhoneNotificationsSync, + // started by toolkit.start(), so toolkit.glasses.settings.set() / + // toolkit.phoneNotifications.* reach the device for any host. Removed here to + // avoid a double-sync.) // Remove old event subscriptions this.subs.forEach((sub) => sub.remove()) From 529e095f75fe3e5a608cb5079b5319f5f2eb71d1 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 10:20:34 -0700 Subject: [PATCH 37/80] island: glasses.settings sync also pushes the FULL set on (re)connect (audit) Completeness for the glasses.settings fix: the change-subscription handles set(), but a freshly-connected device also needs the phone's CURRENT settings (the Mentra app does this via MantleManager's boot push). GlassesSettingsSync now also subscribes to the glasses connection and pushes the full BLUETOOTH_SETTING_KEYS set on the connected transition, so an OEM's glasses get the right settings on connect, not just on later changes. Emit + 281 jest green. --- .../src/services/GlassesSettingsSync.ts | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/mobile/modules/island/src/services/GlassesSettingsSync.ts b/mobile/modules/island/src/services/GlassesSettingsSync.ts index a4dedc4aa2..5b6da1555b 100644 --- a/mobile/modules/island/src/services/GlassesSettingsSync.ts +++ b/mobile/modules/island/src/services/GlassesSettingsSync.ts @@ -1,8 +1,14 @@ /** - * Glasses settings sync — island-owned. Pushes `BLUETOOTH_SETTING_KEYS` changes to - * the connected glasses over the bluetooth-sdk, so `toolkit.glasses.settings.set()` - * (and any other settings-store write) actually reaches the device for ANY host — - * not just the first-party Mentra app, where this used to live in MantleManager. + * Glasses settings sync — island-owned. Keeps the connected glasses in sync with + * the phone's device settings (`BLUETOOTH_SETTING_KEYS`) over the bluetooth-sdk, so + * `toolkit.glasses.settings.set()` (and any other settings-store write) reaches the + * device for ANY host — not just the first-party Mentra app, where this used to live + * in MantleManager. + * + * Two triggers: + * 1. on change — push the keys that changed. + * 2. on (re)connect — push the FULL set, so a freshly-connected device gets the + * phone's current settings (not just future changes). * * Started by `toolkit.start()`. Idempotent. */ @@ -10,15 +16,19 @@ import {shallow} from "zustand/shallow" import BluetoothSdk from "../../../bluetooth-sdk/build/_internal" import {useSettingsStore} from "../stores/settings" +import {useGlassesStore} from "../stores/glasses" +import {isGlassesConnected} from "./GlassesReadiness" -let unsubscribe: (() => void) | null = null +let unsubChange: (() => void) | null = null +let unsubConnect: (() => void) | null = null export function startGlassesSettingsSync(): void { - if (unsubscribe) return - unsubscribe = useSettingsStore.subscribe( + if (unsubChange || unsubConnect) return + + // 1. Push only the keys that changed (matching the prior MantleManager sync). + unsubChange = useSettingsStore.subscribe( (state) => state.getBluetoothSettings(), (settings: Record, previous: Record) => { - // Push only the keys that changed (matching the prior MantleManager sync). const changed: Record = {} for (const key in settings) { if (settings[key] !== previous[key]) changed[key] = settings[key] @@ -29,9 +39,21 @@ export function startGlassesSettingsSync(): void { }, {equalityFn: shallow}, ) + + // 2. Push the full set whenever the glasses transition to connected. + let wasConnected = isGlassesConnected(useGlassesStore.getState().connection) + unsubConnect = useGlassesStore.subscribe(() => { + const connected = isGlassesConnected(useGlassesStore.getState().connection) + if (connected && !wasConnected) { + BluetoothSdk.updateBluetoothSettings(useSettingsStore.getState().getBluetoothSettings()) + } + wasConnected = connected + }) } export function stopGlassesSettingsSync(): void { - unsubscribe?.() - unsubscribe = null + unsubChange?.() + unsubConnect?.() + unsubChange = null + unsubConnect = null } From 8b16fe22f04bd0eef9105b83460cc8ab5bec7129 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 10:48:31 -0700 Subject: [PATCH 38/80] =?UTF-8?q?island:=20remove=20MantleManager=20boot?= =?UTF-8?q?=20settings-push=20=E2=80=94=20island=20on-connect=20is=20singl?= =?UTF-8?q?e=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the one residual duplicate: MantleManager pushed the full device-settings set to native at boot (before auto-reconnect), duplicating island's GlassesSettingsSync on-connect push. Removed the boot push; island's on-connect trigger now owns it (the auto-reconnect fires the connect transition -> push). One implementation, no drift. Updated the MantleManager test to simulate the connect transition for the full-push assertion. typecheck + emit + 281 jest green. --- mobile/src/services/MantleManager.test.ts | 4 ++++ mobile/src/services/MantleManager.ts | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mobile/src/services/MantleManager.test.ts b/mobile/src/services/MantleManager.test.ts index 55c85c4f7d..32410f1b74 100644 --- a/mobile/src/services/MantleManager.test.ts +++ b/mobile/src/services/MantleManager.test.ts @@ -203,6 +203,10 @@ describe("MantleManager", () => { it("syncs native status, routes events, and forwards Bluetooth SDK setting changes", async () => { jest.advanceTimersByTime(1000) + // island's GlassesSettingsSync pushes the FULL device-settings set on the glasses + // connect transition (previously a MantleManager boot push). Simulate the connect. + emitBluetoothSdkEvent("glasses_status", {connection: {state: "connected", fullyBooted: true}}) + expect(bluetoothSdkMock.updateBluetoothSettings).toHaveBeenCalledWith( expect.objectContaining({ contextual_dashboard: true, diff --git a/mobile/src/services/MantleManager.ts b/mobile/src/services/MantleManager.ts index fb2cc129e4..f3677577bf 100644 --- a/mobile/src/services/MantleManager.ts +++ b/mobile/src/services/MantleManager.ts @@ -336,11 +336,11 @@ class MantleManager { offlineSpeechModelService.startBackgroundDownloads() - // Give the native Bluetooth SDK some time to boot before sending initial settings. + // Give the native Bluetooth SDK some time to boot before auto-connecting. BgTimer.setTimeout(() => { - BluetoothSdk.updateBluetoothSettings(useSettingsStore.getState().getBluetoothSettings()) - console.log("MANTLE: Bluetooth settings sent to native SDK") - // settings are now in native; safe to attempt auto-connect + // (Device settings are pushed to native by island's GlassesSettingsSync — on + // every change AND on the connected transition — so there's no boot push here; + // the auto-reconnect below triggers that on-connect push. Single source of truth.) attemptReconnectToDefaultWearable() }, 1000) // (Initial notification-config push now happens in island's From e1adbcaa93f724080a96f511a47be344126deb6c Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 11:00:19 -0700 Subject: [PATCH 39/80] docs(oem): correct glasses.settings key table (recording time is minutes, photo-size enum, gallery_mode) - button_max_recording_time is in MINUTES, not seconds (UI appends "m", sends minutes, SocketComms calls it maxRecordingTimeMinutes). Values 3/5/10/15/20. - button_photo_size enum is low/medium/high/max (with resolutions), not small/large. - gallery_mode is a capture-to-gallery flag the app auto-manages, not an "on-device gallery" toggle; documented the actual semantics + when an OEM sets it. - head_up_angle range is 0-60 degrees. --- docs/glasses-oems/toolkit.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/glasses-oems/toolkit.mdx b/docs/glasses-oems/toolkit.mdx index dd0ca1344f..91c447ec4f 100644 --- a/docs/glasses-oems/toolkit.mdx +++ b/docs/glasses-oems/toolkit.mdx @@ -147,17 +147,17 @@ The current keys, their types, and defaults: |---|---|---|---| | `brightness` | number | `50` | display brightness, `0`–`100` | | `auto_brightness` | boolean | `true` | auto-adjust brightness | -| `head_up_angle` | number | `45` | head-up activation angle (degrees) | +| `head_up_angle` | number | `45` | head-up activation angle in degrees, `0`–`60` | | `screen_disabled` | boolean | `false` | turn the display off | | `contextual_dashboard` | boolean | `true` | show the dashboard on head-up | | `dashboard_height` | number | `4` | dashboard vertical position | | `dashboard_depth` | number | `2` | dashboard distance | | `use_native_dashboard` | boolean | `true` | native dashboard vs JS | | `menu_apps` | `string[] \| null` | `null` | button-menu app package names | -| `button_photo_size` | enum | `"max"` | `small` \| `medium` \| `large` \| `max` | +| `button_photo_size` | enum | `"max"` | `low` (960×720) \| `medium` (1440×1088) \| `high` (3264×2448) \| `max` (camera maximum) | | `button_video_settings` | object | `{ width: 1920, height: 1080, fps: 30 }` | button video recording | | `button_camera_led` | boolean | `true` | capture LED indicator | -| `button_max_recording_time` | number | `10` | max button video length (seconds) | +| `button_max_recording_time` | number | `10` | max button-triggered video length, in **minutes** (e.g. `3`, `5`, `10`, `15`, `20`) | | `camera_fov` | object | `{ fov: 118, roi_position: 0 }` | camera field of view | | `preferred_mic` | enum | `"auto"` | `auto` \| `glasses` \| `phone` | | `lc3_frame_size` | enum (bytes) | `60` | `20` \| `40` \| `60` | @@ -165,7 +165,7 @@ The current keys, their types, and defaults: | `voice_activity_detection_enabled` | boolean | `true` | voice-activity detection | | `power_saving_mode` | boolean | `false` | low-power mode | | `offline_mode` | boolean | `false` | offline operation | -| `gallery_mode` | boolean | `true` | on-device gallery | +| `gallery_mode` | boolean | `true` | capture-to-gallery enabled — when `true`, a camera-button press takes a photo to the on-device gallery. The Mentra app auto-manages this (enabled when the camera app is running or no other app is claiming button events); set it directly only if your host has no equivalent button-routing logic | | `calendar_events` | array | `[]` | synced calendar events | | `twelve_hour_time` | boolean | `true` | 12-hour vs 24-hour clock | | `metric_system` | boolean | `false` | metric vs imperial | From 68793c8b66e9bf807e8377b2728f5fd210a6bc2f Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Wed, 17 Jun 2026 11:10:45 -0700 Subject: [PATCH 40/80] mobile: route connection/pairing lifecycle calls through toolkit facade (Tier 1) Replace direct BluetoothSdk.* calls in the connection/pairing cluster with the @mentra/island toolkit facade, so the first-party app exercises the same surface OEMs use (and shrinks the app's direct btsdk coupling): - glasses lifecycle: connectDefault / disconnect / forget / connectSimulated / setDefault -> toolkit.glasses.* (btclassic, ConnectDeviceButton, DeviceStatus, LogoutUtils, Reconnect, settings/glasses, prep, PairGlassesCard) - controller: connect/disconnect/forget -> toolkit.glasses.controller.* (ConnectDeviceButton, DeviceStatus, settings/controller, scan-controller) - pairing: startScan -> toolkit.pairing.scan, connect -> toolkit.pairing.pair, setDefaultDevice -> toolkit.pairing.setDefault (scan, scan-controller) - OTA: requestVersionInfo -> toolkit.glasses.requestVersionInfo (check-for-updates) Drops the now-unused BluetoothSdk import from 11 files; keeps it only where a non-facade method is still used (Reconnect updateBluetoothSettings seed, DeviceStatus addListener). Tests updated to assert on the facade contract. --- mobile/src/__tests__/app/pairing/scan.test.tsx | 6 +++--- mobile/src/app/miniapps/settings/controller.tsx | 6 +++--- mobile/src/app/miniapps/settings/glasses.tsx | 7 +++---- mobile/src/app/ota/check-for-updates.tsx | 9 ++++----- mobile/src/app/pairing/btclassic.tsx | 4 ++-- mobile/src/app/pairing/prep-controller.tsx | 4 ++-- mobile/src/app/pairing/prep.tsx | 5 ++--- mobile/src/app/pairing/scan-controller.tsx | 11 ++++++----- mobile/src/app/pairing/scan.tsx | 13 +++++++------ .../src/components/glasses/ConnectDeviceButton.tsx | 10 +++++----- mobile/src/components/home/DeviceStatus.tsx | 10 +++++----- mobile/src/components/home/PairGlassesCard.tsx | 4 ++-- mobile/src/effects/Reconnect.test.ts | 7 +++++-- mobile/src/effects/Reconnect.tsx | 4 ++-- mobile/src/utils/LogoutUtils.ts | 6 +++--- 15 files changed, 54 insertions(+), 52 deletions(-) diff --git a/mobile/src/__tests__/app/pairing/scan.test.tsx b/mobile/src/__tests__/app/pairing/scan.test.tsx index 39943ceddf..ef1166f22a 100644 --- a/mobile/src/__tests__/app/pairing/scan.test.tsx +++ b/mobile/src/__tests__/app/pairing/scan.test.tsx @@ -138,7 +138,7 @@ import {render, fireEvent, waitFor} from "@testing-library/react-native" import type {ReactNode} from "react" import {Platform} from "react-native" -import BluetoothSdk from "@mentra/bluetooth-sdk" +import {toolkit} from "@mentra/island" import {useLocalSearchParams} from "expo-router" import {usePushUnder} from "@/contexts/NavigationHistoryContext" import {useNavigationStore} from "@/stores/navigation" @@ -189,7 +189,7 @@ describe("pairing scan screen", () => { const {getByText} = render() await waitFor(() => { - expect(BluetoothSdk.startScan).toHaveBeenCalledWith("Mentra Live") + expect(toolkit.pairing.scan).toHaveBeenCalledWith("Mentra Live") }) fireEvent.press(getByText("001")) @@ -202,7 +202,7 @@ describe("pairing scan screen", () => { }) }) - expect(BluetoothSdk.setDefaultDevice).toHaveBeenCalledWith({ + expect(toolkit.pairing.setDefault).toHaveBeenCalledWith({ id: "a", model: "Mentra Live", name: "MENTRA_LIVE_BLE_001", diff --git a/mobile/src/app/miniapps/settings/controller.tsx b/mobile/src/app/miniapps/settings/controller.tsx index edbf28c772..f6ccc624d5 100644 --- a/mobile/src/app/miniapps/settings/controller.tsx +++ b/mobile/src/app/miniapps/settings/controller.tsx @@ -13,7 +13,7 @@ import {Group} from "@/components/ui" import {RouteButton} from "@/components/ui/RouteButton" import {DeviceTypes} from "@/../../cloud/packages/types/src" -import BluetoothSdk from "@mentra/bluetooth-sdk-internal" +import {toolkit} from "@mentra/island" import {EmptyState} from "@/components/glasses/info/EmptyState" import {showAlert} from "@/contexts/ModalContext" @@ -35,7 +35,7 @@ function DeviceSettings() { }) if (result === 1) { try { - BluetoothSdk.forgetController() + toolkit.glasses.controller.forget() } catch (e) { console.log(e) } @@ -55,7 +55,7 @@ function DeviceSettings() { }) if (result === 1) { - BluetoothSdk.disconnectController() + toolkit.glasses.controller.disconnect() } } diff --git a/mobile/src/app/miniapps/settings/glasses.tsx b/mobile/src/app/miniapps/settings/glasses.tsx index f0a9d370e5..66872589a9 100644 --- a/mobile/src/app/miniapps/settings/glasses.tsx +++ b/mobile/src/app/miniapps/settings/glasses.tsx @@ -14,14 +14,13 @@ import {Group} from "@/components/ui" import {RouteButton} from "@/components/ui/RouteButton" import {Capabilities, DeviceTypes, getModelCapabilities} from "@/../../cloud/packages/types/src" -import BluetoothSdk from "@mentra/bluetooth-sdk" import OtaProgressSection from "@/components/glasses/OtaProgressSection" import {BatteryStatus} from "@/components/glasses/info/BatteryStatus" import {EmptyState} from "@/components/glasses/info/EmptyState" import {ButtonSettings} from "@/components/glasses/settings/ButtonSettings" import BrightnessSetting from "@/components/settings/BrightnessSetting" -import {useApps} from "@mentra/island" +import {toolkit, useApps} from "@mentra/island" // import showAlert from "@/utils/AlertUtils" import {showAlert} from "@/contexts/ModalContext" @@ -56,7 +55,7 @@ function DeviceSettings() { options: {allowDismiss: false}, }) if (result === 1) { - BluetoothSdk.forget() + toolkit.glasses.forget() // useAppletStatusStore.getState().stopAllApplets() // useAppletStatusStore.getState().refreshApplets() // // give us a second to forget the glasses before going back @@ -75,7 +74,7 @@ function DeviceSettings() { }) if (result === 1) { - BluetoothSdk.disconnect() + toolkit.glasses.disconnect() } } diff --git a/mobile/src/app/ota/check-for-updates.tsx b/mobile/src/app/ota/check-for-updates.tsx index 9114f60c76..9cb531a404 100644 --- a/mobile/src/app/ota/check-for-updates.tsx +++ b/mobile/src/app/ota/check-for-updates.tsx @@ -1,7 +1,6 @@ import {useFocusEffect} from "expo-router" import {useEffect, useState, useCallback, useRef} from "react" import {View, ActivityIndicator} from "react-native" -import BluetoothSdk from "@mentra/bluetooth-sdk" import {MINIMUM_OTA_STATUS_BUILD} from "@/app/ota/otaProgressTimeouts" import {MentraLogoStandalone} from "@/components/brands/MentraLogoStandalone" @@ -13,7 +12,7 @@ import {checkForOtaUpdate, getAsgOtaVersionUrl} from "@/effects/OtaUpdateChecker import {translate} from "@/i18n/translate" import {isGlassesConnected, selectGlassesConnected, useGlassesStore, waitForGlassesState} from "@/stores/glasses" import {SETTINGS, useSetting} from "@/stores/settings" -import {BgTimer} from "@mentra/island" +import {BgTimer, toolkit} from "@mentra/island" type CheckState = "checking" | "update_available" | "no_update" | "error" @@ -100,7 +99,7 @@ export default function OtaCheckForUpdatesScreen() { // Request version info since we don't have it yet console.log("OTA: Requesting version_info from glasses") - void BluetoothSdk.requestVersionInfo().catch((error) => { + void toolkit.glasses.requestVersionInfo().catch((error) => { console.warn("OTA: Failed to request version_info from glasses:", error) }) @@ -121,7 +120,7 @@ export default function OtaCheckForUpdatesScreen() { } // Match OtaUpdateChecker home path: BES often arrives late in version_info_3 (chip init after reflash). - void BluetoothSdk.requestVersionInfo().catch((error) => { + void toolkit.glasses.requestVersionInfo().catch((error) => { console.warn("OTA: Failed to refresh version_info before BES wait:", error) }) @@ -174,7 +173,7 @@ export default function OtaCheckForUpdatesScreen() { // Refresh version_info (build / fw) in case the store still held values from a prior session // before the native clear-on-connect + glasses_ready re-query completed. console.log("OTA: Requesting fresh version_info from glasses before HTTP compare") - void BluetoothSdk.requestVersionInfo().catch((error) => { + void toolkit.glasses.requestVersionInfo().catch((error) => { console.warn("OTA: Failed to refresh version_info before OTA compare:", error) }) diff --git a/mobile/src/app/pairing/btclassic.tsx b/mobile/src/app/pairing/btclassic.tsx index 809ba46582..40df28ca12 100644 --- a/mobile/src/app/pairing/btclassic.tsx +++ b/mobile/src/app/pairing/btclassic.tsx @@ -4,7 +4,7 @@ import {OnboardingGuide, OnboardingStep} from "@/components/onboarding/Onboardin import {translate} from "@/i18n" import {focusEffectPreventBack, usePushPrevious} from "@/contexts/NavigationHistoryContext" import {useGlassesStore} from "@/stores/glasses" -import BluetoothSdk from "@mentra/bluetooth-sdk" +import {toolkit} from "@mentra/island" import {SETTINGS, useSetting} from "@/stores/settings" import {SettingsNavigationUtils} from "@/utils/SettingsNavigationUtils" import {useCoreStore} from "@/stores/core" @@ -24,7 +24,7 @@ export default function BtClassicPairingScreen() { focusEffectPreventBack() const handleSuccess = () => { - BluetoothSdk.connectDefault().catch((error) => { + toolkit.glasses.connectDefault().catch((error) => { console.error("Failed to connect default glasses after Bluetooth Classic pairing:", error) }) pushPrevious() diff --git a/mobile/src/app/pairing/prep-controller.tsx b/mobile/src/app/pairing/prep-controller.tsx index cdb3e4c89a..ac77253990 100644 --- a/mobile/src/app/pairing/prep-controller.tsx +++ b/mobile/src/app/pairing/prep-controller.tsx @@ -12,7 +12,7 @@ import {showAlert} from "@/utils/AlertUtils" import {PermissionFeatures, checkConnectivityRequirementsUI, requestFeaturePermissions} from "@/utils/PermissionsUtils" import {useState} from "react" import GlassesTroubleshootingModal from "@/components/glasses/GlassesTroubleshootingModal" -import BluetoothSdk from "@mentra/bluetooth-sdk-internal" +import {toolkit} from "@mentra/island" type BluetoothPermission = Permission | "android.permission.BLUETOOTH" | "android.permission.BLUETOOTH_ADMIN" @@ -195,7 +195,7 @@ export default function PairingPrepScreen() { // skip pairing for simulated glasses: if (deviceModel.startsWith(DeviceTypes.SIMULATED)) { - await BluetoothSdk.connectSimulated() + await toolkit.glasses.connectSimulated() clearHistoryAndGoHome() return } diff --git a/mobile/src/app/pairing/prep.tsx b/mobile/src/app/pairing/prep.tsx index 088404e0a7..14c1eba123 100644 --- a/mobile/src/app/pairing/prep.tsx +++ b/mobile/src/app/pairing/prep.tsx @@ -15,8 +15,7 @@ import {useState} from "react" import GlassesTroubleshootingModal from "@/components/glasses/GlassesTroubleshootingModal" import {OnboardingGuide, OnboardingStep} from "@/components/onboarding/OnboardingGuide" import {CDN_BASE_URL} from "@/constants/appConfig" -import {useAppStatusStore} from "@mentra/island" -import BluetoothSdk from "@mentra/bluetooth-sdk-internal" +import {toolkit, useAppStatusStore} from "@mentra/island" type BluetoothPermission = Permission | "android.permission.BLUETOOTH" | "android.permission.BLUETOOTH_ADMIN" @@ -202,7 +201,7 @@ export default function PairingPrepScreen() { // skip pairing for simulated glasses: if (deviceModel.startsWith(DeviceTypes.SIMULATED)) { - await BluetoothSdk.connectSimulated() + await toolkit.glasses.connectSimulated() clearHistoryAndGoHome() return } diff --git a/mobile/src/app/pairing/scan-controller.tsx b/mobile/src/app/pairing/scan-controller.tsx index 0f57a64fcc..4f9ac6a7c2 100644 --- a/mobile/src/app/pairing/scan-controller.tsx +++ b/mobile/src/app/pairing/scan-controller.tsx @@ -1,4 +1,5 @@ -import BluetoothSdk, {type Device, type DeviceModel} from "@mentra/bluetooth-sdk-internal" +import {type Device, type DeviceModel} from "@mentra/bluetooth-sdk-internal" +import {toolkit} from "@mentra/island" import {useLocalSearchParams} from "expo-router" import {useEffect, useState} from "react" import {ActivityIndicator, Image, Platform, ScrollView, TouchableOpacity, View} from "react-native" @@ -37,8 +38,8 @@ export default function SelectGlassesBluetoothScreen() { if (event && event.actionType !== "GO_BACK" && event.actionType !== "POP") { return } - BluetoothSdk.disconnectController() - BluetoothSdk.forgetController() + toolkit.glasses.controller.disconnect() + toolkit.glasses.controller.forget() goBack() }, true) @@ -53,7 +54,7 @@ export default function SelectGlassesBluetoothScreen() { useEffect(() => { const initializeAndSearchForDevices = async () => { try { - await BluetoothSdk.startScan(deviceModel) + await toolkit.pairing.scan(deviceModel) } catch (error) { console.error("Failed to start controller scan:", error) } @@ -92,7 +93,7 @@ export default function SelectGlassesBluetoothScreen() { const startPairing = async (device: Device) => { setTimeout(() => { - BluetoothSdk.connect(device).catch((error) => { + toolkit.pairing.pair(device).catch((error) => { console.error("Failed to connect to controller:", error) }) }, 2000) diff --git a/mobile/src/app/pairing/scan.tsx b/mobile/src/app/pairing/scan.tsx index 233b695e4a..7e90b030c9 100644 --- a/mobile/src/app/pairing/scan.tsx +++ b/mobile/src/app/pairing/scan.tsx @@ -1,4 +1,5 @@ -import BluetoothSdk, {type Device, type DeviceModel} from "@mentra/bluetooth-sdk" +import {type Device, type DeviceModel} from "@mentra/bluetooth-sdk" +import {toolkit} from "@mentra/island" import {useLocalSearchParams} from "expo-router" import {useEffect, useState} from "react" import {ActivityIndicator, Image, Platform, ScrollView, TouchableOpacity, View} from "react-native" @@ -44,8 +45,8 @@ export default function SelectGlassesBluetoothScreen() { if (event && event.actionType !== "GO_BACK" && event.actionType !== "POP") { return } - BluetoothSdk.disconnect() - BluetoothSdk.forget() + toolkit.glasses.disconnect() + toolkit.glasses.forget() goBack() }, true) @@ -60,7 +61,7 @@ export default function SelectGlassesBluetoothScreen() { useEffect(() => { const initializeAndSearchForDevices = async () => { try { - await BluetoothSdk.startScan(deviceModel) + await toolkit.pairing.scan(deviceModel) } catch (error) { console.error("Failed to start glasses scan:", error) } @@ -101,7 +102,7 @@ export default function SelectGlassesBluetoothScreen() { const deviceTypesWithBtClassic = [DeviceTypes.LIVE] if (Platform.OS === "android" || bluetoothClassicConnected || !deviceTypesWithBtClassic.includes(device.model as DeviceTypes)) { setTimeout(() => { - BluetoothSdk.connect(device).catch((error) => { + toolkit.pairing.pair(device).catch((error) => { console.error("Failed to connect to glasses:", error) }) }, 2000) @@ -110,7 +111,7 @@ export default function SelectGlassesBluetoothScreen() { return } - await BluetoothSdk.setDefaultDevice(device) + await toolkit.pairing.setDefault(device) setDeviceName(device.name) // pair bt classic first: replace("/pairing/btclassic") diff --git a/mobile/src/components/glasses/ConnectDeviceButton.tsx b/mobile/src/components/glasses/ConnectDeviceButton.tsx index a6ae17b64e..ab4d61f18d 100644 --- a/mobile/src/components/glasses/ConnectDeviceButton.tsx +++ b/mobile/src/components/glasses/ConnectDeviceButton.tsx @@ -1,5 +1,5 @@ import {DeviceTypes} from "@/../../cloud/packages/types/src" -import {decideConnectButtonAction, BluetoothSdk} from "@mentra/island" +import {decideConnectButtonAction, toolkit} from "@mentra/island" import {ActivityIndicator, View} from "react-native" import {Button} from "@/components/ignite" @@ -36,7 +36,7 @@ export const ConnectDeviceButton = () => { return } - await BluetoothSdk.connectDefault() + await toolkit.glasses.connectDefault() } catch (err) { console.error("connect to glasses error:", err) showAlert("Connection Error", "Failed to connect to glasses. Please try again.", [{text: "OK"}]) @@ -47,7 +47,7 @@ export const ConnectDeviceButton = () => { const handleConnectOrDisconnect = async () => { const action = decideConnectButtonAction({hasDefaultWearable: !!defaultWearable, busy: isSearching}) if (action === "cancel") { - await BluetoothSdk.disconnect() + await toolkit.glasses.disconnect() } else { await connectGlasses() } @@ -124,7 +124,7 @@ export const ConnectControllerButton = () => { return } - await BluetoothSdk.connectDefaultController() + await toolkit.glasses.controller.connectDefault() } catch (err) { console.error("connect to glasses error:", err) showAlert("Connection Error", "Failed to connect to glasses. Please try again.", [{text: "OK"}]) @@ -135,7 +135,7 @@ export const ConnectControllerButton = () => { const handleConnectOrDisconnect = async () => { const action = decideConnectButtonAction({hasDefaultWearable: !!defaultController, busy: isSearching}) if (action === "cancel") { - await BluetoothSdk.disconnectController() + await toolkit.glasses.controller.disconnect() } else { await connectController() } diff --git a/mobile/src/components/home/DeviceStatus.tsx b/mobile/src/components/home/DeviceStatus.tsx index 4ec953262e..261d5d5236 100644 --- a/mobile/src/components/home/DeviceStatus.tsx +++ b/mobile/src/components/home/DeviceStatus.tsx @@ -7,7 +7,7 @@ import {Button, Icon, Text} from "@/components/ignite" import {useAppTheme} from "@/contexts/ThemeContext" import {useNavigationStore} from "@/stores/navigation" import {translate} from "@/i18n" -import {decideConnectButtonAction, BluetoothSdk} from "@mentra/island" +import {decideConnectButtonAction, toolkit, BluetoothSdk} from "@mentra/island" import {isGlassesConnected, isGlassesReady, useGlassesStore} from "@/stores/glasses" import {useSearchingState} from "@/hooks/useSearchingState" import {SETTINGS, useSetting} from "@/stores/settings" @@ -130,7 +130,7 @@ export const GlassesStatus = ({style}: {style?: ViewStyle}) => { if (!requirementsCheck) { return } - await BluetoothSdk.connectDefault() + await toolkit.glasses.connectDefault() } catch (error) { console.error("connect to glasses error:", error) showAlert("Connection Error", "Failed to connect to glasses. Please try again.", [{text: "OK"}]) @@ -140,7 +140,7 @@ export const GlassesStatus = ({style}: {style?: ViewStyle}) => { const handleConnectOrDisconnect = async () => { const action = decideConnectButtonAction({hasDefaultWearable: !!defaultWearable, busy: searching || nativeLinkBusy}) if (action === "cancel") { - await BluetoothSdk.disconnect() + await toolkit.glasses.disconnect() setIsCheckingConnectivity(false) resetSearching() } else { @@ -260,9 +260,9 @@ export const ControllerStatus = ({style}: {style?: ViewStyle}) => { const handleConnectOrDisconnect = async () => { if (isSearching) { - await BluetoothSdk.disconnectController() + await toolkit.glasses.controller.disconnect() } else { - await BluetoothSdk.connectDefaultController() + await toolkit.glasses.controller.connectDefault() } } diff --git a/mobile/src/components/home/PairGlassesCard.tsx b/mobile/src/components/home/PairGlassesCard.tsx index 9737f4a2ee..e6afc69df5 100644 --- a/mobile/src/components/home/PairGlassesCard.tsx +++ b/mobile/src/components/home/PairGlassesCard.tsx @@ -4,7 +4,7 @@ import {Button, Text} from "@/components/ignite" import {useAppTheme} from "@/contexts/ThemeContext" import {useNavigationStore} from "@/stores/navigation" import GlassView from "@/components/ui/GlassView" -import BluetoothSdk from "@mentra/bluetooth-sdk-internal" +import {toolkit} from "@mentra/island" import {useState} from "react" export const PairGlassesCard = ({style}: {style?: ViewStyle}) => { @@ -46,7 +46,7 @@ export const PairGlassesCard = ({style}: {style?: ViewStyle}) => { tx="home:start" preset="primary" onPress={() => { - BluetoothSdk.connectSimulated() + toolkit.glasses.connectSimulated() }} />