Compared to {@link #encodeAvifForBle(Bitmap, int, String)} this skips the Bitmap/RGBA + * round-trip entirely (the encoder consumes the 1-byte/pixel buffer directly) and emits a + * bitstream with no chroma planes. {@code speed} maps to AOM cpu-used (0..9, higher = faster; + * -1 = encoder default) - on the MTK8766's software AV1 encoder this is the dominant + * encode-time knob. + */ + public static byte[] encodeAvifForBleMono( + byte[] luma, int width, int height, int quality, int speed, String sourceJpegPath) + throws Exception { + boolean hasImu = hasImuMetadata(sourceJpegPath); + Log.d( + TAG, + "encodeAvifForBleMono: source=" + + sourceJpegPath + + " hasImuMetadata=" + + hasImu + + " quality=" + + quality + + " speed=" + + speed); + HeifCoder heifCoder = new HeifCoder(); + byte[] avif = heifCoder.encodeAvifMono(luma, width, height, width, quality, speed); + if (hasImu) { + String json = readImuJsonFromJpeg(sourceJpegPath); + if (json == null) { + Log.w( + TAG, + "encodeAvifForBleMono: hasImuMetadata true but readImuJsonFromJpeg" + + " returned null"); + } else { + try { + JSONObject payload = new JSONObject(json); + byte[] exifSegment = buildExifApp1Segment(payload); + byte[] exifTiff = Arrays.copyOfRange(exifSegment, 4, exifSegment.length); + byte[] withExif = AvifBmffExifInjector.injectExif(avif, exifTiff); + Log.d( + TAG, + "encodeAvifForBleMono: mono AVIF+EXIF, " + + withExif.length + + " bytes, rawHasExifMarker=" + + containsExifMarker(withExif)); + return withExif; + } catch (Exception injectError) { + Log.w( + TAG, + "encodeAvifForBleMono: EXIF path failed, sending plain mono AVIF: " + + injectError.getMessage()); + } + } + } + Log.d(TAG, "encodeAvifForBleMono: mono AVIF (no EXIF), " + avif.length + " bytes"); + return avif; + } + /** True when the device exposes an AV1 encoder for {@link AvifWriter}. */ static boolean isAv1EncoderAvailable() { MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); diff --git a/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessor.java b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessor.java new file mode 100644 index 0000000000..013cc701ca --- /dev/null +++ b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessor.java @@ -0,0 +1,305 @@ +package com.mentra.asg_client.io.media.core; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Rect; +import android.util.Log; +import androidx.annotation.Nullable; +import java.io.IOException; + +/** + * Phase 1 of the BLE photo pipeline rework: decode the source JPEG straight into a single-channel + * luma ("Y plane") buffer, crop to a region of interest, apply a contrast stretch + mild unsharp, + * and only rehydrate to a full RGBA {@link Bitmap} at the very last step for the AVIF encoder + * (which only accepts {@link Bitmap} input). + * + *
This targets photos that exist to be read (documents, signs, screens) rather than viewed - + * color contributes nothing to legibility, so every intermediate buffer here is 1 byte/pixel + * instead of the 4 bytes/pixel the old ARGB decode -> resize -> sharpen chain used. That's where + * the previous pipeline spent most of its ~35MB peak per photo. + * + *
Chroma is never allocated on our side, but the AVIF encoder itself still emits a YUV420 + * (color) bitstream internally - true YUV400-only encode requires a native libavif/libaom shim + * that this library doesn't expose (Phase 2, out of scope here). The wire-size win from skipping + * chroma is small regardless (flat chroma planes already compress to near-nothing in AV1); the + * real payoff of this pass is the ~3x RAM reduction and matching increase in decode/encode speed. + */ +final class GrayscaleBleProcessor { + private static final String TAG = "GrayscaleBleProcessor"; + + // Same Laplacian-based unsharp weights as BleImageSharpener, ported to single-channel luma. + private static final int CENTER_W = 34; + private static final int NEIGHBOR_W = 6; + private static final int SCALE = 10; + + private GrayscaleBleProcessor() {} + + /** + * Result of the luma pipeline: a single-channel buffer plus its dimensions. Kept as raw bytes + * so a monochrome-capable encoder can consume it directly without an RGBA round-trip. + */ + static final class LumaImage { + final byte[] luma; + final int width; + final int height; + + LumaImage(byte[] luma, int width, int height) { + this.luma = luma; + this.width = width; + this.height = height; + } + + /** + * Rehydrates the luma buffer into a gray RGBA bitmap for encoders that only accept + * {@link Bitmap} input. Built one row at a time - never allocates a full-frame int[]. + * Caller owns recycling the returned bitmap. + */ + Bitmap toGrayscaleBitmap() { + Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + int[] row = new int[width]; + for (int y = 0; y < height; y++) { + int offset = y * width; + for (int x = 0; x < width; x++) { + int v = luma[offset + x] & 0xFF; + row[x] = 0xFF000000 | (v << 16) | (v << 8) | v; + } + bitmap.setPixels(row, 0, width, 0, y, width, 1); + } + return bitmap; + } + } + + /** + * Decode {@code jpegPath}, crop to {@code roi} (source-pixel coordinates; {@code null} means + * full frame - a future pass wires a real ROI in here from edge-density analysis), apply a + * contrast stretch + unsharp, and scale to fit within {@code targetWidth}x{@code targetHeight} + * (aspect-preserving, same semantics as the caps in {@code resolveBleParams}). + */ + static LumaImage processToLuma( + String jpegPath, @Nullable Rect roi, int targetWidth, int targetHeight) + throws IOException { + long start = System.currentTimeMillis(); + + BitmapFactory.Options bounds = new BitmapFactory.Options(); + bounds.inJustDecodeBounds = true; + BitmapFactory.decodeFile(jpegPath, bounds); + int srcWidth = bounds.outWidth; + int srcHeight = bounds.outHeight; + if (srcWidth <= 0 || srcHeight <= 0) { + throw new IOException("Failed to read JPEG bounds: " + jpegPath); + } + + Rect region = + (roi != null) + ? clampRect(roi, srcWidth, srcHeight) + : new Rect(0, 0, srcWidth, srcHeight); + + // Decode at the smallest power-of-two sample size that still comfortably covers the + // target resolution - we never materialize a full-res ARGB bitmap here. + int sampleSize = + computeSampleSize(region.width(), region.height(), targetWidth, targetHeight); + + BitmapFactory.Options decodeOpts = new BitmapFactory.Options(); + decodeOpts.inSampleSize = sampleSize; + // Source JPEG has no alpha; RGB_565 halves decode memory vs ARGB_8888 (2B/px vs 4B/px) + // and loses nothing we need since we reduce to luma immediately below. + decodeOpts.inPreferredConfig = Bitmap.Config.RGB_565; + Bitmap decoded = BitmapFactory.decodeFile(jpegPath, decodeOpts); + if (decoded == null) { + throw new IOException("Failed to decode JPEG: " + jpegPath); + } + + float scale = 1f / sampleSize; + Rect scaledRegion = + clampRect( + new Rect( + (int) (region.left * scale), + (int) (region.top * scale), + (int) (region.right * scale), + (int) (region.bottom * scale)), + decoded.getWidth(), + decoded.getHeight()); + + byte[] luma = extractCroppedLuma(decoded, scaledRegion); + decoded.recycle(); + + int lumaWidth = scaledRegion.width(); + int lumaHeight = scaledRegion.height(); + + applyContrastStretch(luma); + + int[] outDims = fitWithinAspect(lumaWidth, lumaHeight, targetWidth, targetHeight); + byte[] resized = resizeLumaBilinear(luma, lumaWidth, lumaHeight, outDims[0], outDims[1]); + byte[] sharpened = unsharpLuma(resized, outDims[0], outDims[1]); + + Log.d( + TAG, + "GrayscaleBleProcessor: " + + srcWidth + + "x" + + srcHeight + + " -> crop " + + lumaWidth + + "x" + + lumaHeight + + " -> out " + + outDims[0] + + "x" + + outDims[1] + + " in " + + (System.currentTimeMillis() - start) + + "ms"); + return new LumaImage(sharpened, outDims[0], outDims[1]); + } + + /** + * Convenience wrapper: full luma pipeline plus rehydration to a grayscale (R=G=B) bitmap ready + * for {@code PhotoExifMetadataWriter.encodeAvifForBle}; caller owns recycling it. + */ + static Bitmap process(String jpegPath, @Nullable Rect roi, int targetWidth, int targetHeight) + throws IOException { + return processToLuma(jpegPath, roi, targetWidth, targetHeight).toGrayscaleBitmap(); + } + + private static Rect clampRect(Rect r, int width, int height) { + int left = clampInt(r.left, 0, width - 1); + int top = clampInt(r.top, 0, height - 1); + int right = clampInt(r.right, left + 1, width); + int bottom = clampInt(r.bottom, top + 1, height); + return new Rect(left, top, right, bottom); + } + + private static int computeSampleSize( + int regionWidth, int regionHeight, int targetWidth, int targetHeight) { + int sample = 1; + while ((regionWidth / (sample * 2)) >= targetWidth + && (regionHeight / (sample * 2)) >= targetHeight) { + sample *= 2; + } + return sample; + } + + /** Reads one decoded-bitmap row at a time (width-sized int[]) - never a full-frame int[]. */ + private static byte[] extractCroppedLuma(Bitmap bitmap, Rect region) { + int width = region.width(); + int height = region.height(); + byte[] luma = new byte[width * height]; + int[] row = new int[width]; + for (int y = 0; y < height; y++) { + bitmap.getPixels(row, 0, width, region.left, region.top + y, width, 1); + int rowOffset = y * width; + for (int x = 0; x < width; x++) { + int px = row[x]; + int r = (px >> 16) & 0xFF; + int g = (px >> 8) & 0xFF; + int b = px & 0xFF; + // BT.601 luma weights - matches the camera's own YUV luma convention. + luma[rowOffset + x] = (byte) ((r * 66 + g * 129 + b * 25 + 128) >> 8); + } + } + return luma; + } + + /** In-place min/max stretch; skipped on near-flat/noisy frames to avoid amplifying noise. */ + private static void applyContrastStretch(byte[] luma) { + int min = 255; + int max = 0; + for (byte v : luma) { + int value = v & 0xFF; + if (value < min) min = value; + if (value > max) max = value; + } + int range = max - min; + if (range < 16) { + return; + } + float scale = 255f / range; + for (int i = 0; i < luma.length; i++) { + int v = Math.round(((luma[i] & 0xFF) - min) * scale); + luma[i] = (byte) clampInt(v, 0, 255); + } + } + + private static int[] fitWithinAspect(int width, int height, int targetWidth, int targetHeight) { + float aspect = (float) width / height; + int outWidth = targetWidth; + int outHeight = targetHeight; + if (aspect > (float) targetWidth / targetHeight) { + outHeight = Math.max(1, Math.round(targetWidth / aspect)); + } else { + outWidth = Math.max(1, Math.round(targetHeight * aspect)); + } + return new int[] {outWidth, outHeight}; + } + + private static byte[] resizeLumaBilinear( + byte[] src, int srcWidth, int srcHeight, int dstWidth, int dstHeight) { + if (srcWidth == dstWidth && srcHeight == dstHeight) { + return src; + } + byte[] dst = new byte[dstWidth * dstHeight]; + float xRatio = (float) srcWidth / dstWidth; + float yRatio = (float) srcHeight / dstHeight; + for (int y = 0; y < dstHeight; y++) { + float sy = (y + 0.5f) * yRatio - 0.5f; + int y0 = clampInt((int) Math.floor(sy), 0, srcHeight - 1); + int y1 = clampInt(y0 + 1, 0, srcHeight - 1); + float fy = sy - y0; + int rowOffset0 = y0 * srcWidth; + int rowOffset1 = y1 * srcWidth; + int dstRowOffset = y * dstWidth; + for (int x = 0; x < dstWidth; x++) { + float sx = (x + 0.5f) * xRatio - 0.5f; + int x0 = clampInt((int) Math.floor(sx), 0, srcWidth - 1); + int x1 = clampInt(x0 + 1, 0, srcWidth - 1); + float fx = sx - x0; + + int p00 = src[rowOffset0 + x0] & 0xFF; + int p10 = src[rowOffset0 + x1] & 0xFF; + int p01 = src[rowOffset1 + x0] & 0xFF; + int p11 = src[rowOffset1 + x1] & 0xFF; + + float top = p00 + (p10 - p00) * fx; + float bottom = p01 + (p11 - p01) * fx; + int v = Math.round(top + (bottom - top) * fy); + dst[dstRowOffset + x] = (byte) clampInt(v, 0, 255); + } + } + return dst; + } + + /** + * Single-channel port of {@link BleImageSharpener}'s Laplacian unsharp: identity + 0.6x + * Laplacian, tuned for text strokes after a downscale. Border pixels copy through unchanged. + */ + private static byte[] unsharpLuma(byte[] src, int width, int height) { + if (width < 3 || height < 3) { + return src; + } + byte[] dst = new byte[width * height]; + System.arraycopy(src, 0, dst, 0, width); + System.arraycopy(src, (height - 1) * width, dst, (height - 1) * width, width); + + for (int y = 1; y < height - 1; y++) { + int row = y * width; + dst[row] = src[row]; + dst[row + width - 1] = src[row + width - 1]; + for (int x = 1; x < width - 1; x++) { + int i = row + x; + int c = src[i] & 0xFF; + int l = src[i - 1] & 0xFF; + int r = src[i + 1] & 0xFF; + int u = src[i - width] & 0xFF; + int d = src[i + width] & 0xFF; + + int v = (c * CENTER_W - (l + r + u + d) * NEIGHBOR_W) / SCALE; + dst[i] = (byte) clampInt(v, 0, 255); + } + } + return dst; + } + + private static int clampInt(int v, int min, int max) { + return Math.max(min, Math.min(max, v)); + } +} diff --git a/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/MediaCaptureService.java b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/MediaCaptureService.java index 2c78851823..1141c763dc 100644 --- a/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/MediaCaptureService.java +++ b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/MediaCaptureService.java @@ -148,6 +148,25 @@ private static final class UploadTarget { public static final int bleImageTargetHeight = 480; public static final int bleImageAvifQuality = 40; + // Flip to false to fall back to the legacy full-color BLE pipeline (full-res ARGB decode + + // createScaledBitmap + RGB BleImageSharpener + YUV420 AVIF). BLE photos exist to be read + // (documents/signs/screens), not viewed, so grayscale is the default - true here also cuts + // the per-photo peak transient memory from ~36MB down to ~4MB (see GrayscaleBleProcessor) + // and encodes a true monochrome (YUV400) AVIF with no chroma planes. + private static final boolean ENABLE_GRAYSCALE_BLE_PHOTOS = true; + + // AOM cpu-used for the mono BLE encode (0..9, higher = faster; -1 = plugin default). The + // MTK8766 has no hardware AV1 encoder, so software encode time dominates the BLE photo + // pipeline (4-5.5s of ~7.4s end-to-end at plugin default). Benchmarked at 1920x1440: speed 8 + // encodes ~2.6x faster than the default with comparable output size. + private static final int BLE_AVIF_ENCODE_SPEED = 8; + + // Crop BLE photos to the detected text/document region before encode (grayscale path only). + // The crop keeps native resolution on the region OCR actually reads instead of downscaling + // the whole frame; detection is conservative and falls back to full frame on weak signal + // (see TextRegionDetector). Kill switch for on-device tuning. + private static final boolean ENABLE_BLE_TEXT_ROI_CROP = true; + private static class BleParams { final int targetWidth; final int targetHeight; @@ -3970,14 +3989,7 @@ private void compressAndSendViaBle( PhotoCaptureTestHooks.addFakeDelay("COMPRESSION"); try { - // 1. Load original image - android.graphics.Bitmap original = - android.graphics.BitmapFactory.decodeFile(originalPath); - if (original == null) { - throw new Exception("Failed to decode image file"); - } - - // 2. Resolve BLE resize and quality parameters based on requested + // 1. Resolve BLE resize and quality parameters based on requested // size String requestedSize = photoRequestedSizes.get(requestId); if (requestedSize == null || requestedSize.isEmpty()) { @@ -3986,64 +3998,138 @@ private void compressAndSendViaBle( BleParams bleParams = resolveBleParams(requestedSize); - // Calculate new dimensions maintaining aspect ratio, constrained by - // requested target - int targetWidth = bleParams.targetWidth; - int targetHeight = bleParams.targetHeight; - float aspectRatio = - (float) original.getWidth() / original.getHeight(); + byte[] compressedData; + if (ENABLE_GRAYSCALE_BLE_PHOTOS) { + // 2. Grayscale luma pipeline: detect the text region, then + // decode subsampled, crop + contrast + unsharp on + // 1-byte/pixel buffers, then hand the luma plane straight to + // the monochrome (YUV400) AVIF encoder - no Bitmap/RGBA + // round-trip anywhere. BLE photos exist to be read + // (documents/signs/screens), not viewed, so color is dead + // weight; this path never materializes the ~30MB full-res + // ARGB bitmap the old decode+resize+sharpen chain used. The + // ROI crop keeps native resolution on the region OCR reads; + // null ROI (weak/spread signal) means full frame. + android.graphics.Rect textRoi = + ENABLE_BLE_TEXT_ROI_CROP + ? TextRegionDetector.detect(originalPath) + : null; + GrayscaleBleProcessor.LumaImage lumaImage = + GrayscaleBleProcessor.processToLuma( + originalPath, + textRoi, + bleParams.targetWidth, + bleParams.targetHeight); - if (aspectRatio > targetWidth / (float) targetHeight) { - targetHeight = (int) (targetWidth / aspectRatio); + Log.d( + TAG, + "📐 BLE image ready for encode: " + + lumaImage.width + + "x" + + lumaImage.height + + " grayscale=true (mono AVIF, speed=" + + BLE_AVIF_ENCODE_SPEED + + ")"); + + // 3. Encode as monochrome AVIF only (no JPEG fallback over + // BLE) + compressedData = + PhotoExifMetadataWriter.encodeAvifForBleMono( + lumaImage.luma, + lumaImage.width, + lumaImage.height, + bleParams.avifQuality, + BLE_AVIF_ENCODE_SPEED, + originalPath); } else { - targetWidth = (int) (targetHeight * aspectRatio); - } + // Legacy full-color path. + android.graphics.Bitmap original = + android.graphics.BitmapFactory.decodeFile( + originalPath); + if (original == null) { + throw new Exception("Failed to decode image file"); + } - // 3. Resize bitmap, then restore edge contrast: the downscale - // low-pass-filters text strokes, and a mild unsharp costs almost - // no bytes while visibly improving glyph legibility. - android.graphics.Bitmap scaled = - android.graphics.Bitmap.createScaledBitmap( - original, targetWidth, targetHeight, true); - // createScaledBitmap returns the SAME object when the size - // already matches (medium ladder == capture resolution), so - // recycling unconditionally would kill the bitmap mid-pipeline. - if (scaled != original) { - original.recycle(); - } - android.graphics.Bitmap resized = - BleImageSharpener.sharpen(mContext, scaled); - if (resized != scaled) { - scaled.recycle(); - } + // Calculate new dimensions maintaining aspect ratio, + // constrained by requested target + int targetWidth = bleParams.targetWidth; + int targetHeight = bleParams.targetHeight; + float aspectRatio = + (float) original.getWidth() / original.getHeight(); - // 4. Encode as AVIF only (no JPEG fallback over BLE) - Log.d( - TAG, - "BLE AVIF encode: originalPath=" - + originalPath - + " hasImuMetadata=" - + PhotoExifMetadataWriter.hasImuMetadata( - originalPath)); - byte[] compressedData; - try { - compressedData = - PhotoExifMetadataWriter.encodeAvifForBle( - resized, bleParams.avifQuality, originalPath); - } finally { - resized.recycle(); + if (aspectRatio > targetWidth / (float) targetHeight) { + targetHeight = (int) (targetWidth / aspectRatio); + } else { + targetWidth = (int) (targetHeight * aspectRatio); + } + + // Resize bitmap, then restore edge contrast: the downscale + // low-pass-filters text strokes, and a mild unsharp costs + // almost no bytes while visibly improving glyph legibility. + android.graphics.Bitmap scaled = + android.graphics.Bitmap.createScaledBitmap( + original, targetWidth, targetHeight, true); + // createScaledBitmap returns the SAME object when the size + // already matches (medium ladder == capture resolution), so + // recycling unconditionally would kill the bitmap mid-pipeline. + if (scaled != original) { + original.recycle(); + } + android.graphics.Bitmap resized = + BleImageSharpener.sharpen(mContext, scaled); + if (resized != scaled) { + scaled.recycle(); + } + + Log.d( + TAG, + "📐 BLE image ready for encode: " + + resized.getWidth() + + "x" + + resized.getHeight() + + " grayscale=false"); + + // 3. Encode as AVIF only (no JPEG fallback over BLE) + Log.d( + TAG, + "BLE AVIF encode: originalPath=" + + originalPath + + " hasImuMetadata=" + + PhotoExifMetadataWriter.hasImuMetadata( + originalPath)); + try { + compressedData = + PhotoExifMetadataWriter.encodeAvifForBle( + resized, + bleParams.avifQuality, + originalPath); + } finally { + resized.recycle(); + } } Log.d(TAG, "Successfully encoded as AVIF for BLE"); long compressionTime = System.currentTimeMillis() - startTime; recordTiming(requestId, "ble_compress_done"); + File originalFileForSize = new File(originalPath); + long originalSizeBytes = originalFileForSize.length(); + double originalSizeKb = originalSizeBytes / 1024.0; + double compressedSizeKb = compressedData.length / 1024.0; Log.d( TAG, "✅ Compressed photo for BLE: " + originalPath - + " -> " + + " (" + + originalSizeBytes + + " bytes / " + + String.format(Locale.US, "%.1f", originalSizeKb) + + " KB) -> AVIF " + compressedData.length - + " bytes"); + + " bytes / " + + String.format( + Locale.US, "%.1f", compressedSizeKb) + + " KB, grayscale=" + + ENABLE_GRAYSCALE_BLE_PHOTOS); Log.d(TAG, "⏱️ Compression took: " + compressionTime + "ms"); // 5. Save compressed data to temporary file with bleImgId as name diff --git a/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/TextRegionDetector.java b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/TextRegionDetector.java new file mode 100644 index 0000000000..c87504404c --- /dev/null +++ b/asg_client/app/src/main/java/com/mentra/asg_client/io/media/core/TextRegionDetector.java @@ -0,0 +1,236 @@ +package com.mentra.asg_client.io.media.core; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Rect; +import android.util.Log; +import androidx.annotation.Nullable; + +/** + * Finds the text/document region of a photo so the BLE pipeline can crop to it instead of + * spending its byte budget on background. Pure-Java edge-density analysis on a tiny (~320px) + * grayscale decode - no ML, runs in ~10-20ms on the MTK8766. + * + *
Approach (per docs/agents/BLE_PHOTO_QUALITY_PLAN.md): Sobel gradient magnitude accumulated
+ * into a coarse grid, then the bounding box of cells whose edge density clearly exceeds the
+ * frame average, padded by 10%. Detection is deliberately conservative - cropping away real text
+ * is far worse for OCR than sending some extra background - so it returns {@code null} (no crop)
+ * whenever the signal is weak, spread across the whole frame, or the crop would barely shrink
+ * the image.
+ */
+final class TextRegionDetector {
+ private static final String TAG = "TextRegionDetector";
+
+ /** Analysis raster cap; keeps Sobel + grid work trivial. */
+ private static final int ANALYSIS_MAX_DIM = 320;
+
+ private static final int GRID_COLS = 8;
+ private static final int GRID_ROWS = 6;
+
+ /** A cell counts as "text" when its mean gradient exceeds this multiple of the frame mean. */
+ private static final float CELL_ACTIVATION_RATIO = 1.5f;
+
+ /** Below this mean gradient (0..255 scale) the frame has no usable edge signal at all. */
+ private static final float MIN_FRAME_MEAN_GRADIENT = 4f;
+
+ /** Crops that keep more than this fraction of the frame area are not worth the risk. */
+ private static final float MAX_USEFUL_AREA_FRACTION = 0.80f;
+
+ /** Padding added around the detected bounding box, as a fraction of its size. */
+ private static final float PADDING_FRACTION = 0.10f;
+
+ private TextRegionDetector() {}
+
+ /**
+ * Returns the detected text region in source-pixel coordinates of {@code jpegPath}, or
+ * {@code null} when no clearly concentrated edge region exists (caller should send the full
+ * frame).
+ */
+ @Nullable
+ static Rect detect(String jpegPath) {
+ try {
+ long start = System.currentTimeMillis();
+
+ BitmapFactory.Options bounds = new BitmapFactory.Options();
+ bounds.inJustDecodeBounds = true;
+ BitmapFactory.decodeFile(jpegPath, bounds);
+ int srcWidth = bounds.outWidth;
+ int srcHeight = bounds.outHeight;
+ if (srcWidth <= 0 || srcHeight <= 0) {
+ return null;
+ }
+
+ int sampleSize = 1;
+ while (srcWidth / (sampleSize * 2) >= ANALYSIS_MAX_DIM
+ && srcHeight / (sampleSize * 2) >= ANALYSIS_MAX_DIM) {
+ sampleSize *= 2;
+ }
+ BitmapFactory.Options decodeOpts = new BitmapFactory.Options();
+ decodeOpts.inSampleSize = sampleSize;
+ decodeOpts.inPreferredConfig = Bitmap.Config.RGB_565;
+ Bitmap decoded = BitmapFactory.decodeFile(jpegPath, decodeOpts);
+ if (decoded == null) {
+ return null;
+ }
+
+ int w = decoded.getWidth();
+ int h = decoded.getHeight();
+ byte[] luma = new byte[w * h];
+ int[] row = new int[w];
+ for (int y = 0; y < h; y++) {
+ decoded.getPixels(row, 0, w, 0, y, w, 1);
+ int off = y * w;
+ for (int x = 0; x < w; x++) {
+ int px = row[x];
+ int r = (px >> 16) & 0xFF;
+ int g = (px >> 8) & 0xFF;
+ int b = px & 0xFF;
+ luma[off + x] = (byte) ((r * 66 + g * 129 + b * 25 + 128) >> 8);
+ }
+ }
+ decoded.recycle();
+
+ Rect analysisRect = detectOnLuma(luma, w, h);
+ if (analysisRect == null) {
+ Log.d(
+ TAG,
+ "No text region detected in "
+ + (System.currentTimeMillis() - start)
+ + "ms - sending full frame");
+ return null;
+ }
+
+ // Map analysis coordinates back to source pixels.
+ float scaleX = (float) srcWidth / w;
+ float scaleY = (float) srcHeight / h;
+ Rect srcRect =
+ new Rect(
+ Math.max(0, (int) (analysisRect.left * scaleX)),
+ Math.max(0, (int) (analysisRect.top * scaleY)),
+ Math.min(srcWidth, (int) Math.ceil(analysisRect.right * scaleX)),
+ Math.min(srcHeight, (int) Math.ceil(analysisRect.bottom * scaleY)));
+ Log.d(
+ TAG,
+ "Text region "
+ + srcRect.width()
+ + "x"
+ + srcRect.height()
+ + " at ("
+ + srcRect.left
+ + ","
+ + srcRect.top
+ + ") of "
+ + srcWidth
+ + "x"
+ + srcHeight
+ + " ("
+ + Math.round(
+ 100f
+ * srcRect.width()
+ * srcRect.height()
+ / ((float) srcWidth * srcHeight))
+ + "% area) in "
+ + (System.currentTimeMillis() - start)
+ + "ms");
+ return srcRect;
+ } catch (Exception | OutOfMemoryError e) {
+ // Detection is an optimization; any failure means "send the full frame".
+ Log.w(TAG, "Text region detection failed - sending full frame", e);
+ return null;
+ }
+ }
+
+ /**
+ * Core detection on a luma raster; exposed for unit tests. Returns the padded bounding box in
+ * raster coordinates, or {@code null} for "no crop".
+ */
+ @Nullable
+ static Rect detectOnLuma(byte[] luma, int width, int height) {
+ if (width < GRID_COLS * 2 || height < GRID_ROWS * 2) {
+ return null;
+ }
+
+ // Sobel gradient magnitude accumulated per grid cell.
+ long[] cellEnergy = new long[GRID_COLS * GRID_ROWS];
+ long[] cellPixels = new long[GRID_COLS * GRID_ROWS];
+ long totalEnergy = 0;
+ for (int y = 1; y < height - 1; y++) {
+ int rowOff = y * width;
+ int cellRow = Math.min(GRID_ROWS - 1, y * GRID_ROWS / height);
+ for (int x = 1; x < width - 1; x++) {
+ int i = rowOff + x;
+ int tl = luma[i - width - 1] & 0xFF;
+ int t = luma[i - width] & 0xFF;
+ int tr = luma[i - width + 1] & 0xFF;
+ int l = luma[i - 1] & 0xFF;
+ int r = luma[i + 1] & 0xFF;
+ int bl = luma[i + width - 1] & 0xFF;
+ int b = luma[i + width] & 0xFF;
+ int br = luma[i + width + 1] & 0xFF;
+
+ int gx = (tr + 2 * r + br) - (tl + 2 * l + bl);
+ int gy = (bl + 2 * b + br) - (tl + 2 * t + tr);
+ int mag = Math.abs(gx) + Math.abs(gy);
+
+ int cellCol = Math.min(GRID_COLS - 1, x * GRID_COLS / width);
+ int cell = cellRow * GRID_COLS + cellCol;
+ cellEnergy[cell] += mag;
+ cellPixels[cell]++;
+ totalEnergy += mag;
+ }
+ }
+
+ long totalPixels = (long) (width - 2) * (height - 2);
+ float frameMean = totalEnergy / (float) totalPixels;
+ if (frameMean < MIN_FRAME_MEAN_GRADIENT) {
+ return null;
+ }
+
+ // Bounding box of activated cells.
+ float threshold = frameMean * CELL_ACTIVATION_RATIO;
+ int minCol = GRID_COLS;
+ int maxCol = -1;
+ int minRow = GRID_ROWS;
+ int maxRow = -1;
+ for (int cr = 0; cr < GRID_ROWS; cr++) {
+ for (int cc = 0; cc < GRID_COLS; cc++) {
+ int cell = cr * GRID_COLS + cc;
+ if (cellPixels[cell] == 0) {
+ continue;
+ }
+ float mean = cellEnergy[cell] / (float) cellPixels[cell];
+ if (mean >= threshold) {
+ if (cc < minCol) minCol = cc;
+ if (cc > maxCol) maxCol = cc;
+ if (cr < minRow) minRow = cr;
+ if (cr > maxRow) maxRow = cr;
+ }
+ }
+ }
+ if (maxCol < 0) {
+ return null;
+ }
+
+ // Cell box -> pixel box.
+ int left = minCol * width / GRID_COLS;
+ int right = (maxCol + 1) * width / GRID_COLS;
+ int top = minRow * height / GRID_ROWS;
+ int bottom = (maxRow + 1) * height / GRID_ROWS;
+
+ // 10% padding on each side.
+ int padX = Math.round((right - left) * PADDING_FRACTION);
+ int padY = Math.round((bottom - top) * PADDING_FRACTION);
+ left = Math.max(0, left - padX);
+ right = Math.min(width, right + padX);
+ top = Math.max(0, top - padY);
+ bottom = Math.min(height, bottom + padY);
+
+ // Not worth cropping when the region is basically the whole frame.
+ float areaFraction = (right - left) * (bottom - top) / ((float) width * height);
+ if (areaFraction > MAX_USEFUL_AREA_FRACTION) {
+ return null;
+ }
+
+ return new Rect(left, top, right, bottom);
+ }
+}
diff --git a/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessorTest.java b/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessorTest.java
new file mode 100644
index 0000000000..3bf6f23333
--- /dev/null
+++ b/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/GrayscaleBleProcessorTest.java
@@ -0,0 +1,107 @@
+package com.mentra.asg_client.io.media.core;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.Rect;
+import java.io.File;
+import java.io.FileOutputStream;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
+import org.robolectric.annotation.GraphicsMode;
+
+// Native graphics so BitmapFactory/Canvas produce real pixels (legacy shadows decode to zeros).
+@RunWith(RobolectricTestRunner.class)
+@Config(sdk = 33)
+@GraphicsMode(GraphicsMode.Mode.NATIVE)
+public class GrayscaleBleProcessorTest {
+
+ @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ /** Writes a JPEG with a white background and a black left half, like a stark document edge. */
+ private File writeTestJpeg(int width, int height) throws Exception {
+ Bitmap source = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+ Canvas canvas = new Canvas(source);
+ canvas.drawColor(Color.WHITE);
+ Paint black = new Paint();
+ black.setColor(Color.BLACK);
+ canvas.drawRect(0, 0, width / 2f, height, black);
+
+ File jpeg = tempFolder.newFile("test.jpg");
+ try (FileOutputStream fos = new FileOutputStream(jpeg)) {
+ source.compress(Bitmap.CompressFormat.JPEG, 92, fos);
+ }
+ source.recycle();
+ return jpeg;
+ }
+
+ @Test
+ public void processToLumaPreservesAspectWithinTargetCaps() throws Exception {
+ File jpeg = writeTestJpeg(640, 480);
+
+ GrayscaleBleProcessor.LumaImage luma =
+ GrayscaleBleProcessor.processToLuma(jpeg.getAbsolutePath(), null, 320, 320);
+
+ assertThat(luma.width).isEqualTo(320);
+ assertThat(luma.height).isEqualTo(240);
+ assertThat(luma.luma).hasSize(320 * 240);
+ }
+
+ @Test
+ public void processToLumaKeepsDarkAndBrightRegionsDistinct() throws Exception {
+ File jpeg = writeTestJpeg(640, 480);
+
+ GrayscaleBleProcessor.LumaImage luma =
+ GrayscaleBleProcessor.processToLuma(jpeg.getAbsolutePath(), null, 320, 320);
+
+ int midRow = luma.height / 2;
+ int leftSample = luma.luma[midRow * luma.width + luma.width / 4] & 0xFF;
+ int rightSample = luma.luma[midRow * luma.width + (3 * luma.width) / 4] & 0xFF;
+ assertThat(leftSample).isLessThan(64);
+ assertThat(rightSample).isGreaterThan(192);
+ }
+
+ @Test
+ public void roiCropsToRequestedRegion() throws Exception {
+ File jpeg = writeTestJpeg(640, 480);
+
+ // Crop entirely inside the white (right) half; result should be uniformly bright.
+ Rect roi = new Rect(400, 100, 600, 300);
+ GrayscaleBleProcessor.LumaImage luma =
+ GrayscaleBleProcessor.processToLuma(jpeg.getAbsolutePath(), roi, 320, 320);
+
+ assertThat(luma.width).isGreaterThan(0);
+ assertThat(luma.height).isGreaterThan(0);
+ long sum = 0;
+ for (byte b : luma.luma) {
+ sum += (b & 0xFF);
+ }
+ double mean = sum / (double) luma.luma.length;
+ assertThat(mean).isGreaterThan(180.0);
+ }
+
+ @Test
+ public void toGrayscaleBitmapProducesEqualChannels() throws Exception {
+ File jpeg = writeTestJpeg(320, 240);
+
+ Bitmap gray =
+ GrayscaleBleProcessor.process(jpeg.getAbsolutePath(), null, 160, 160);
+ try {
+ assertThat(gray.getWidth()).isEqualTo(160);
+ assertThat(gray.getHeight()).isEqualTo(120);
+ int px = gray.getPixel(gray.getWidth() - 10, gray.getHeight() / 2);
+ assertThat(Color.red(px)).isEqualTo(Color.green(px));
+ assertThat(Color.green(px)).isEqualTo(Color.blue(px));
+ assertThat(Color.alpha(px)).isEqualTo(255);
+ } finally {
+ gray.recycle();
+ }
+ }
+}
diff --git a/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/TextRegionDetectorTest.java b/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/TextRegionDetectorTest.java
new file mode 100644
index 0000000000..77147d7f0e
--- /dev/null
+++ b/asg_client/app/src/test/java/com/mentra/asg_client/io/media/core/TextRegionDetectorTest.java
@@ -0,0 +1,118 @@
+package com.mentra.asg_client.io.media.core;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.Rect;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.Random;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+import org.robolectric.annotation.Config;
+import org.robolectric.annotation.GraphicsMode;
+
+// Native graphics so BitmapFactory/Canvas produce real pixels (legacy shadows decode to zeros).
+@RunWith(RobolectricTestRunner.class)
+@Config(sdk = 33)
+@GraphicsMode(GraphicsMode.Mode.NATIVE)
+public class TextRegionDetectorTest {
+
+ @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private static final int W = 320;
+ private static final int H = 240;
+
+ /** Fills a luma raster with a flat background value. */
+ private static byte[] flatLuma(int value) {
+ byte[] luma = new byte[W * H];
+ java.util.Arrays.fill(luma, (byte) value);
+ return luma;
+ }
+
+ /** Draws dense text-like strokes (alternating dark segments) into a luma region. */
+ private static void drawTextBlock(byte[] luma, Rect region) {
+ for (int y = region.top; y < region.bottom; y++) {
+ if ((y % 6) >= 2) {
+ continue; // line spacing
+ }
+ for (int x = region.left; x < region.right; x++) {
+ if ((x / 3) % 2 == 0) {
+ luma[y * W + x] = (byte) 20;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void detectsConcentratedTextBlock() {
+ byte[] luma = flatLuma(230);
+ Rect textBlock = new Rect(40, 60, 160, 120);
+ drawTextBlock(luma, textBlock);
+
+ Rect roi = TextRegionDetector.detectOnLuma(luma, W, H);
+
+ assertThat(roi).isNotNull();
+ // ROI must cover the text block (padding may extend it) without being the whole frame.
+ assertThat(roi.contains(textBlock.centerX(), textBlock.centerY())).isTrue();
+ assertThat(roi.left).isLessThanOrEqualTo(textBlock.left);
+ assertThat(roi.right).isGreaterThanOrEqualTo(textBlock.right);
+ float areaFraction = roi.width() * roi.height() / (float) (W * H);
+ assertThat(areaFraction).isLessThan(0.8f);
+ }
+
+ @Test
+ public void returnsNullForFlatFrame() {
+ byte[] luma = flatLuma(128);
+ assertThat(TextRegionDetector.detectOnLuma(luma, W, H)).isNull();
+ }
+
+ @Test
+ public void returnsNullWhenEdgesCoverWholeFrame() {
+ // Uniform noise everywhere: edge energy is spread, no concentrated region.
+ byte[] luma = new byte[W * H];
+ Random random = new Random(42);
+ random.nextBytes(luma);
+ assertThat(TextRegionDetector.detectOnLuma(luma, W, H)).isNull();
+ }
+
+ @Test
+ public void detectMapsRoiBackToSourcePixels() throws Exception {
+ // 640x480 JPEG, text confined to the top-left quadrant.
+ Bitmap source = Bitmap.createBitmap(640, 480, Bitmap.Config.ARGB_8888);
+ Canvas canvas = new Canvas(source);
+ canvas.drawColor(Color.WHITE);
+ Paint dark = new Paint();
+ dark.setColor(Color.BLACK);
+ for (int line = 0; line < 8; line++) {
+ int y = 40 + line * 20;
+ for (int seg = 0; seg < 20; seg++) {
+ if (seg % 2 == 0) {
+ canvas.drawRect(40 + seg * 10, y, 40 + seg * 10 + 8, y + 6, dark);
+ }
+ }
+ }
+ File jpeg = tempFolder.newFile("doc.jpg");
+ try (FileOutputStream fos = new FileOutputStream(jpeg)) {
+ source.compress(Bitmap.CompressFormat.JPEG, 92, fos);
+ }
+ source.recycle();
+
+ Rect roi = TextRegionDetector.detect(jpeg.getAbsolutePath());
+
+ assertThat(roi).isNotNull();
+ // Text lives in x:[40,240), y:[40,200) - ROI should sit in the top-left, not span the
+ // whole frame.
+ assertThat(roi.contains(140, 120)).isTrue();
+ assertThat(roi.right).isLessThan(640);
+ assertThat(roi.bottom).isLessThan(480);
+ float areaFraction = roi.width() * roi.height() / (float) (640 * 480);
+ assertThat(areaFraction).isLessThan(0.8f);
+ }
+}
diff --git a/asg_client/third_party/avif-coder-mono/README.md b/asg_client/third_party/avif-coder-mono/README.md
new file mode 100644
index 0000000000..934ea1cb8e
--- /dev/null
+++ b/asg_client/third_party/avif-coder-mono/README.md
@@ -0,0 +1,53 @@
+# avif-coder-mono
+
+Mentra fork of [awxkee/avif-coder](https://github.com/awxkee/avif-coder) at tag
+`1.7.3`, consumed by `asg_client` as `app/libs/avif-coder-mono-1.7.3-mono1.aar`.
+
+## What the fork adds
+
+Upstream `HeifCoder.encodeAvif()` only accepts an Android `Bitmap` and always
+encodes chroma 4:2:0. The BLE photo path on the glasses produces grayscale
+images for on-phone text recognition, where a Bitmap round-trip wastes ~5MB of
+RAM per photo (1 byte/pixel luma forced into a 4 byte/pixel RGBA container)
+and the encoder wastes time on empty chroma planes.
+
+The fork is additive only (no upstream code modified):
+
+- `JniMonoEncoder.cpp` - new JNI entry point `encodeAvifMonoImpl` that
+ consumes a raw 8-bit luma buffer and emits a true monochrome (YUV400) AVIF
+ via libheif's `heif_colorspace_monochrome`. Also exposes the AOM `speed`
+ (cpu-used, 0..9) parameter, which upstream never sets - on the MTK8766 the
+ default is the dominant cost of the 4-5.5s software AV1 encode.
+- `HeifCoder.kt` - new public `encodeAvifMono(luma, width, height, stride,
+ quality, speed)` method.
+- ABIs trimmed to `armeabi-v7a` + `arm64-v8a` (all Mentra Live ships).
+
+Package/class names are unchanged (`com.radzivon.bartoshyk.avif.coder`), so
+the AAR is a drop-in replacement for the upstream JitPack artifact, including
+for the existing `encodeAvif`/`decode` call sites.
+
+Monochrome support was validated against the exact bundled library versions
+(libheif 1.17.6 + AOM 3.8.1): encode succeeds, output decodes back as
+colorspace=monochrome with no chroma planes, and any spec-compliant AVIF
+decoder (libavif/dav1d on Android, ImageIO on iOS 16+) reads it - YUV400 is
+first-class in the AV1 bitstream spec, not an extension.
+
+## Rebuilding the AAR
+
+```bash
+git clone --branch 1.7.3 https://github.com/awxkee/avif-coder.git
+cd avif-coder
+git apply /path/to/avif-coder-1.7.3-mono.patch
+echo "sdk.dir=$ANDROID_HOME" > local.properties # needs NDK 26.1.10909125 + CMake 3.22.1
+JAVA_HOME=/path/to/jdk17 ./gradlew :avif-coder:assembleRelease
+cp avif-coder/build/outputs/aar/avif-coder-release.aar \
+ ../asg_client/app/libs/avif-coder-mono-1.7.3-mono1.aar
+```
+
+The repo commits prebuilt static libs (libheif/libaom/etc.) per ABI under
+`avif-coder/src/main/cpp/lib/`, so only the JNI wrapper C++ is compiled - no
+third-party native rebuilds are required.
+
+Longer term this should move to a proper `Mentra-Community/avif-coder` fork
+published via JitPack; the patch file in this directory is the full diff to
+apply on top of upstream `1.7.3`.
diff --git a/asg_client/third_party/avif-coder-mono/avif-coder-1.7.3-mono.patch b/asg_client/third_party/avif-coder-mono/avif-coder-1.7.3-mono.patch
new file mode 100644
index 0000000000..90cfbd8409
--- /dev/null
+++ b/asg_client/third_party/avif-coder-mono/avif-coder-1.7.3-mono.patch
@@ -0,0 +1,312 @@
+diff --git a/avif-coder/build.gradle.kts b/avif-coder/build.gradle.kts
+index 09f731e..2343493 100644
+--- a/avif-coder/build.gradle.kts
++++ b/avif-coder/build.gradle.kts
+@@ -40,9 +40,9 @@ afterEvaluate {
+ publishing {
+ publications {
+ create