fix(mobile): apply exif orientation to android raw photos

mobile/android/app/CMakeLists.txt

@@ -7,6 +7,7 @@ project(native_buffer LANGUAGES C)
 
 add_library(native_buffer SHARED
     src/main/cpp/native_buffer.c
+    src/main/cpp/native_image.c
 )
 
 target_link_libraries(native_buffer jnigraphics)

mobile/android/app/src/main/cpp/native_image.c

@@ -0,0 +1,109 @@
+#include <jni.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <android/bitmap.h>
+
+// Cache-friendly block size for the tiled rotation (in pixels). 32x32 uint32 = 4KB, fits L1.
+#define TILE 32
+
+// EXIF orientation values (androidx.exifinterface.media.ExifInterface.ORIENTATION_*).
+enum {
+  ORIENTATION_FLIP_HORIZONTAL = 2,
+  ORIENTATION_ROTATE_180 = 3,
+  ORIENTATION_FLIP_VERTICAL = 4,
+  ORIENTATION_TRANSPOSE = 5,
+  ORIENTATION_ROTATE_90 = 6,
+  ORIENTATION_TRANSVERSE = 7,
+  ORIENTATION_ROTATE_270 = 8,
+};
+
+// The orientations that swap width and height. Must stay in sync with affine_for's dim usage.
+static int swaps_dims(int o) {
+  return o == ORIENTATION_ROTATE_90 || o == ORIENTATION_ROTATE_270 ||
+         o == ORIENTATION_TRANSPOSE || o == ORIENTATION_TRANSVERSE;
+}
+
+// A source pixel (sx, sy) maps to destination index base + sx*stepX + sy*stepY, where dw is the
+// destination width. This affine form covers all 8 EXIF orientations and matches the pixel layout
+// of Bitmap.createBitmap(src, matrixForExifOrientation(o)). int64_t so it stays correct on
+// armeabi-v7a (32-bit long) regardless of how large MAX_RAW_DECODE_PIXELS grows.
+static void affine_for(int o, int sw, int sh, int dw, int64_t *base, int64_t *stepX, int64_t *stepY) {
+  switch (o) {
+    case ORIENTATION_ROTATE_90:       *base = sh - 1;                             *stepX = dw;  *stepY = -1;  break;
+    case ORIENTATION_ROTATE_270:      *base = (int64_t) (sw - 1) * dw;            *stepX = -dw; *stepY = 1;   break;
+    case ORIENTATION_ROTATE_180:      *base = (int64_t) (sh - 1) * dw + (sw - 1); *stepX = -1;  *stepY = -dw; break;
+    case ORIENTATION_FLIP_HORIZONTAL: *base = sw - 1;                             *stepX = -1;  *stepY = dw;  break;
+    case ORIENTATION_FLIP_VERTICAL:   *base = (int64_t) (sh - 1) * dw;            *stepX = 1;   *stepY = -dw; break;
+    case ORIENTATION_TRANSPOSE:       *base = 0;                                  *stepX = dw;  *stepY = 1;   break;
+    case ORIENTATION_TRANSVERSE:      *base = (int64_t) (sw - 1) * dw + (sh - 1); *stepX = -dw; *stepY = -1;  break;
+    default:                          *base = 0;                                  *stepX = 1;   *stepY = dw;  break;
+  }
+}
+
+// Copy each source pixel (whole uint32, so channel order/premult is irrelevant) to its rotated
+// destination, walking TILE x TILE blocks so the scattered writes of a 90/270 transpose stay
+// cache-resident. dst is densely packed (rowBytes == dw*4, no padding), which the affine math relies on.
+static void rotate_tiled(const uint8_t *src, int srcStride, uint32_t *dst,
+                         int sw, int sh, int64_t base, int64_t stepX, int64_t stepY) {
+  for (int ty = 0; ty < sh; ty += TILE) {
+    int yEnd = ty + TILE < sh ? ty + TILE : sh;
+    for (int tx = 0; tx < sw; tx += TILE) {
+      int xEnd = tx + TILE < sw ? tx + TILE : sw;
+      for (int sy = ty; sy < yEnd; sy++) {
+        const uint32_t *srcRow = (const uint32_t *) (src + (size_t) sy * srcStride);
+        int64_t idx = base + (int64_t) sy * stepY + (int64_t) tx * stepX;
+        for (int sx = tx; sx < xEnd; sx++) {
+          dst[idx] = srcRow[sx];
+          idx += stepX;
+        }
+      }
+    }
+  }
+}
+
+// Rotates an RGBA_8888 bitmap to the given EXIF orientation into a freshly malloc'd buffer (free it
+// via NativeBuffer.free). Fills outInfo with {width, height, rowBytes} and returns the buffer
+// address, or 0 if the bitmap can't be handled (e.g. a non-8888 format) so the caller can fall back.
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_NativeImage_rotate(
+    JNIEnv *env, jclass clazz, jobject bitmap, jint orientation, jintArray outInfo) {
+  AndroidBitmapInfo info;
+  if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) {
+    return 0;
+  }
+  if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
+    return 0;
+  }
+
+  int sw = (int) info.width;
+  int sh = (int) info.height;
+  int dw = swaps_dims(orientation) ? sh : sw;
+  int dh = swaps_dims(orientation) ? sw : sh;
+
+  uint32_t *dst = (uint32_t *) malloc((size_t) dw * dh * 4);
+  if (dst == NULL) {
+    return 0;
+  }
+
+  void *srcPixels = NULL;
+  if (AndroidBitmap_lockPixels(env, bitmap, &srcPixels) != ANDROID_BITMAP_RESULT_SUCCESS) {
+    free(dst);
+    return 0;
+  }
+
+  int64_t base, stepX, stepY;
+  affine_for(orientation, sw, sh, dw, &base, &stepX, &stepY);
+  rotate_tiled((const uint8_t *) srcPixels, (int) info.stride, dst, sw, sh, base, stepX, stepY);
+
+  AndroidBitmap_unlockPixels(env, bitmap);
+
+  jint dims[3] = {dw, dh, dw * 4};
+  (*env)->SetIntArrayRegion(env, outInfo, 0, 3, dims);
+  // Keep ownership in C until the buffer is safely handed back: if outInfo was somehow too small,
+  // SetIntArrayRegion left a pending exception and Kotlin will never receive (or free) dst.
+  if ((*env)->ExceptionCheck(env)) {
+    free(dst);
+    return 0;
+  }
+  return (jlong) dst;
+}

mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt

@@ -0,0 +1,19 @@
+package app.alextran.immich
+
+import android.graphics.Bitmap
+
+object NativeImage {
+  init {
+    // rotate() is compiled into the native_buffer shared lib (which already links jnigraphics).
+    System.loadLibrary("native_buffer")
+  }
+
+  /**
+   * Rotates an RGBA_8888 [bitmap] to the given EXIF [orientation], writing the result into a freshly
+   * malloc'd native buffer. Returns the buffer address (free it with [NativeBuffer.free]) and fills
+   * [outInfo] with {width, height, rowBytes}. Returns 0 when the bitmap can't be handled (e.g. a
+   * non-8888 config) so the caller can fall back.
+   */
+  @JvmStatic
+  external fun rotate(bitmap: Bitmap, orientation: Int, outInfo: IntArray): Long
+}

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -12,7 +12,9 @@ import android.provider.MediaStore.Images
 import android.provider.MediaStore.Video
 import android.util.Size
 import androidx.annotation.RequiresApi
+import androidx.exifinterface.media.ExifInterface
 import app.alextran.immich.NativeBuffer
+import app.alextran.immich.NativeImage
 import kotlin.math.*
 import java.io.IOException
 import java.util.concurrent.Executors
@@ -74,6 +76,11 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
   companion object {
     val CANCELLED = Result.success<Map<String, Long>?>(null)
     val OPTIONS = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 }
+
+    // "Load original" decodes a raw at full res, and the orientation pass then walks every pixel, so
+    // cap the decode resolution to keep that bounded on huge DNGs. This only trims pixels on very
+    // large raws - they still come out upright, just downsampled.
+    const val MAX_RAW_DECODE_PIXELS = 24_000_000L
   }
 
   override fun getThumbhash(thumbhash: String, callback: (Result<Map<String, Long>>) -> Unit) {
@@ -181,35 +188,133 @@ class LocalImagesImpl(context: Context) : LocalImageApi {
     val id = assetId.toLong()
 
     signal.throwIfCanceled()
-    val bitmap = if (isVideo) {
-      decodeVideoThumbnail(id, size, signal)
-    } else {
-      decodeImage(id, size, signal)
-    }
-
     try {
-      signal.throwIfCanceled()
-      val res = bitmap.toNativeBuffer()
-      signal.throwIfCanceled()
+      val res = if (isVideo) {
+        decodeVideoThumbnail(id, size, signal).toNativeBuffer()
+      } else {
+        val (bitmap, orientation) = decodeImage(id, size, signal)
+        signal.throwIfCanceled()
+        if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) {
+          bitmap.toNativeBuffer()
+        } else {
+          rotateToNativeBuffer(bitmap, orientation, signal)
+        }
+      }
+      // Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would
+      // orphan it. Deliver it; Dart frees the buffer itself if the request was cancelled meanwhile.
       callback(Result.success(res))
     } catch (e: Exception) {
       callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e))
     }
   }
 
-  private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Bitmap {
+  // Returns the decoded bitmap plus the EXIF orientation that still needs applying. Only Q+ raw
+  // decodes come back unrotated (ImageDecoder / loadThumbnail skip EXIF for raw like DNG); every
+  // other path already orients itself, so it reports ORIENTATION_NORMAL.
+  private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Pair<Bitmap, Int> {
     signal.throwIfCanceled()
     val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id)
+    val handleRaw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && isRawMime(uri)
+    val orientation = if (handleRaw) rawOrientation(uri) else ExifInterface.ORIENTATION_NORMAL
+
     if (size.width <= 0 || size.height <= 0 || size.width > 768 || size.height > 768) {
-      return decodeSource(uri, size, signal)
+      // A "load original" request is unsized -> a full-res decode. For raw, cap it so the later
+      // orientation pass stays within a safe pixel budget.
+      val bitmap = if (handleRaw && (size.width <= 0 || size.height <= 0)) {
+        decodeRawCapped(uri, signal)
+      } else {
+        decodeSource(uri, size, signal)
+      }
+      return bitmap to orientation
     }
 
-    return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+    val bitmap = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
       resolver.loadThumbnail(uri, size, signal)
     } else {
       signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) }
       Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS)
     }
+    return bitmap to orientation
+  }
+
+  private fun isRawMime(uri: Uri): Boolean {
+    val mime = resolver.getType(uri) ?: return false
+    return mime.startsWith("image/x-") || mime == "image/dng"
+  }
+
+  private fun rawOrientation(uri: Uri): Int {
+    return resolver.openInputStream(uri)?.use {
+      ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
+    } ?: ExifInterface.ORIENTATION_NORMAL
+  }
+
+  // Full-res raw decode for "load original", sampled down to MAX_RAW_DECODE_PIXELS (power of two).
+  // Caps resolution only; the caller still rotates the result, so even huge raws end up upright.
+  @RequiresApi(Build.VERSION_CODES.Q)
+  private fun decodeRawCapped(uri: Uri, signal: CancellationSignal): Bitmap {
+    signal.throwIfCanceled()
+    return ImageDecoder.decodeBitmap(ImageDecoder.createSource(resolver, uri)) { decoder, info, _ ->
+      val pixels = info.size.width.toLong() * info.size.height.toLong()
+      var sample = 1
+      while (pixels / (sample.toLong() * sample) > MAX_RAW_DECODE_PIXELS) {
+        sample *= 2
+      }
+      if (sample > 1) {
+        decoder.setTargetSampleSize(sample)
+      }
+      decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE
+      decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB))
+    }
+  }
+
+  // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded
+  // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no
+  // intermediate rotated bitmap), falling back to Skia for any config the native path can't take.
+  private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map<String, Long> {
+    signal.throwIfCanceled()
+    // Force ARGB_8888 so both the native pass and the Skia fallback are 4 bytes/pixel: the native
+    // rotate needs a lockable 8888 buffer, and toNativeBuffer() below allocates width*height*4 (an
+    // F16/HDR decode would otherwise under-allocate). No-op for the common already-8888 case.
+    val src = if (bitmap.config != Bitmap.Config.ARGB_8888) {
+      val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false)
+      bitmap.recycle()
+      converted ?: throw IOException("could not convert bitmap to ARGB_8888")
+    } else {
+      bitmap
+    }
+    try {
+      val info = IntArray(3)
+      val pointer = NativeImage.rotate(src, orientation, info)
+      if (pointer != 0L) {
+        return mapOf(
+          "pointer" to pointer,
+          "width" to info[0].toLong(),
+          "height" to info[1].toLong(),
+          "rowBytes" to info[2].toLong()
+        )
+      }
+      // Native path declined (unsupported config) -> rotate via Skia, then copy out.
+      val matrix = matrixForExifOrientation(orientation) ?: return src.toNativeBuffer()
+      return Bitmap.createBitmap(src, 0, 0, src.width, src.height, matrix, true).toNativeBuffer()
+    } finally {
+      if (!src.isRecycled) src.recycle()
+    }
+  }
+
+  // EXIF orientation (1-8) -> transform matrix, or null when no rotation/flip is needed.
+  private fun matrixForExifOrientation(orientation: Int): Matrix? {
+    val matrix = Matrix()
+    when (orientation) {
+      ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f)
+      ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f)
+      ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f)
+      ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f)
+      ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f)
+      ExifInterface.ORIENTATION_TRANSPOSE -> matrix.apply { postRotate(270f); postScale(-1f, 1f) }
+      ExifInterface.ORIENTATION_TRANSVERSE -> matrix.apply { postRotate(90f); postScale(-1f, 1f) }
+      else -> return null
+    }
+    return matrix
   }
 
   private fun decodeVideoThumbnail(id: Long, target: Size, signal: CancellationSignal): Bitmap {