diff --git a/mobile/android/app/CMakeLists.txt b/mobile/android/app/CMakeLists.txt index 133bde4fc0070..9cad7b915abd6 100644 --- a/mobile/android/app/CMakeLists.txt +++ b/mobile/android/app/CMakeLists.txt @@ -7,6 +7,7 @@ project(native_buffer LANGUAGES C) add_library(native_buffer SHARED src/main/cpp/native_buffer.c + src/main/cpp/native_image.c ) target_link_libraries(native_buffer jnigraphics) diff --git a/mobile/android/app/src/main/cpp/native_image.c b/mobile/android/app/src/main/cpp/native_image.c new file mode 100644 index 0000000000000..07bb7e9a2c47f --- /dev/null +++ b/mobile/android/app/src/main/cpp/native_image.c @@ -0,0 +1,109 @@ +#include +#include +#include +#include + +// Cache-friendly block size for the tiled rotation (in pixels). 32x32 uint32 = 4KB, fits L1. +#define TILE 32 + +// EXIF orientation values (androidx.exifinterface.media.ExifInterface.ORIENTATION_*). +enum { + ORIENTATION_FLIP_HORIZONTAL = 2, + ORIENTATION_ROTATE_180 = 3, + ORIENTATION_FLIP_VERTICAL = 4, + ORIENTATION_TRANSPOSE = 5, + ORIENTATION_ROTATE_90 = 6, + ORIENTATION_TRANSVERSE = 7, + ORIENTATION_ROTATE_270 = 8, +}; + +// The orientations that swap width and height. Must stay in sync with affine_for's dim usage. +static int swaps_dims(int o) { + return o == ORIENTATION_ROTATE_90 || o == ORIENTATION_ROTATE_270 || + o == ORIENTATION_TRANSPOSE || o == ORIENTATION_TRANSVERSE; +} + +// A source pixel (sx, sy) maps to destination index base + sx*stepX + sy*stepY, where dw is the +// destination width. This affine form covers all 8 EXIF orientations and matches the pixel layout +// of Bitmap.createBitmap(src, matrixForExifOrientation(o)). int64_t so it stays correct on +// armeabi-v7a (32-bit long) regardless of how large MAX_RAW_DECODE_PIXELS grows. +static void affine_for(int o, int sw, int sh, int dw, int64_t *base, int64_t *stepX, int64_t *stepY) { + switch (o) { + case ORIENTATION_ROTATE_90: *base = sh - 1; *stepX = dw; *stepY = -1; break; + case ORIENTATION_ROTATE_270: *base = (int64_t) (sw - 1) * dw; *stepX = -dw; *stepY = 1; break; + case ORIENTATION_ROTATE_180: *base = (int64_t) (sh - 1) * dw + (sw - 1); *stepX = -1; *stepY = -dw; break; + case ORIENTATION_FLIP_HORIZONTAL: *base = sw - 1; *stepX = -1; *stepY = dw; break; + case ORIENTATION_FLIP_VERTICAL: *base = (int64_t) (sh - 1) * dw; *stepX = 1; *stepY = -dw; break; + case ORIENTATION_TRANSPOSE: *base = 0; *stepX = dw; *stepY = 1; break; + case ORIENTATION_TRANSVERSE: *base = (int64_t) (sw - 1) * dw + (sh - 1); *stepX = -dw; *stepY = -1; break; + default: *base = 0; *stepX = 1; *stepY = dw; break; + } +} + +// Copy each source pixel (whole uint32, so channel order/premult is irrelevant) to its rotated +// destination, walking TILE x TILE blocks so the scattered writes of a 90/270 transpose stay +// cache-resident. dst is densely packed (rowBytes == dw*4, no padding), which the affine math relies on. +static void rotate_tiled(const uint8_t *src, int srcStride, uint32_t *dst, + int sw, int sh, int64_t base, int64_t stepX, int64_t stepY) { + for (int ty = 0; ty < sh; ty += TILE) { + int yEnd = ty + TILE < sh ? ty + TILE : sh; + for (int tx = 0; tx < sw; tx += TILE) { + int xEnd = tx + TILE < sw ? tx + TILE : sw; + for (int sy = ty; sy < yEnd; sy++) { + const uint32_t *srcRow = (const uint32_t *) (src + (size_t) sy * srcStride); + int64_t idx = base + (int64_t) sy * stepY + (int64_t) tx * stepX; + for (int sx = tx; sx < xEnd; sx++) { + dst[idx] = srcRow[sx]; + idx += stepX; + } + } + } + } +} + +// Rotates an RGBA_8888 bitmap to the given EXIF orientation into a freshly malloc'd buffer (free it +// via NativeBuffer.free). Fills outInfo with {width, height, rowBytes} and returns the buffer +// address, or 0 if the bitmap can't be handled (e.g. a non-8888 format) so the caller can fall back. +JNIEXPORT jlong JNICALL +Java_app_alextran_immich_NativeImage_rotate( + JNIEnv *env, jclass clazz, jobject bitmap, jint orientation, jintArray outInfo) { + AndroidBitmapInfo info; + if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) { + return 0; + } + if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) { + return 0; + } + + int sw = (int) info.width; + int sh = (int) info.height; + int dw = swaps_dims(orientation) ? sh : sw; + int dh = swaps_dims(orientation) ? sw : sh; + + uint32_t *dst = (uint32_t *) malloc((size_t) dw * dh * 4); + if (dst == NULL) { + return 0; + } + + void *srcPixels = NULL; + if (AndroidBitmap_lockPixels(env, bitmap, &srcPixels) != ANDROID_BITMAP_RESULT_SUCCESS) { + free(dst); + return 0; + } + + int64_t base, stepX, stepY; + affine_for(orientation, sw, sh, dw, &base, &stepX, &stepY); + rotate_tiled((const uint8_t *) srcPixels, (int) info.stride, dst, sw, sh, base, stepX, stepY); + + AndroidBitmap_unlockPixels(env, bitmap); + + jint dims[3] = {dw, dh, dw * 4}; + (*env)->SetIntArrayRegion(env, outInfo, 0, 3, dims); + // Keep ownership in C until the buffer is safely handed back: if outInfo was somehow too small, + // SetIntArrayRegion left a pending exception and Kotlin will never receive (or free) dst. + if ((*env)->ExceptionCheck(env)) { + free(dst); + return 0; + } + return (jlong) dst; +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt new file mode 100644 index 0000000000000..b1398d84b8d43 --- /dev/null +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/NativeImage.kt @@ -0,0 +1,19 @@ +package app.alextran.immich + +import android.graphics.Bitmap + +object NativeImage { + init { + // rotate() is compiled into the native_buffer shared lib (which already links jnigraphics). + System.loadLibrary("native_buffer") + } + + /** + * Rotates an RGBA_8888 [bitmap] to the given EXIF [orientation], writing the result into a freshly + * malloc'd native buffer. Returns the buffer address (free it with [NativeBuffer.free]) and fills + * [outInfo] with {width, height, rowBytes}. Returns 0 when the bitmap can't be handled (e.g. a + * non-8888 config) so the caller can fall back. + */ + @JvmStatic + external fun rotate(bitmap: Bitmap, orientation: Int, outInfo: IntArray): Long +} diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt index 3babad2e3718b..a98ccb9271592 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt @@ -12,7 +12,9 @@ import android.provider.MediaStore.Images import android.provider.MediaStore.Video import android.util.Size import androidx.annotation.RequiresApi +import androidx.exifinterface.media.ExifInterface import app.alextran.immich.NativeBuffer +import app.alextran.immich.NativeImage import kotlin.math.* import java.io.IOException import java.util.concurrent.Executors @@ -74,6 +76,11 @@ class LocalImagesImpl(context: Context) : LocalImageApi { companion object { val CANCELLED = Result.success?>(null) val OPTIONS = BitmapFactory.Options().apply { inPreferredConfig = Bitmap.Config.ARGB_8888 } + + // "Load original" decodes a raw at full res, and the orientation pass then walks every pixel, so + // cap the decode resolution to keep that bounded on huge DNGs. This only trims pixels on very + // large raws - they still come out upright, just downsampled. + const val MAX_RAW_DECODE_PIXELS = 24_000_000L } override fun getThumbhash(thumbhash: String, callback: (Result>) -> Unit) { @@ -181,35 +188,133 @@ class LocalImagesImpl(context: Context) : LocalImageApi { val id = assetId.toLong() signal.throwIfCanceled() - val bitmap = if (isVideo) { - decodeVideoThumbnail(id, size, signal) - } else { - decodeImage(id, size, signal) - } - try { - signal.throwIfCanceled() - val res = bitmap.toNativeBuffer() - signal.throwIfCanceled() + val res = if (isVideo) { + decodeVideoThumbnail(id, size, signal).toNativeBuffer() + } else { + val (bitmap, orientation) = decodeImage(id, size, signal) + signal.throwIfCanceled() + if (orientation == ExifInterface.ORIENTATION_NORMAL || orientation == ExifInterface.ORIENTATION_UNDEFINED) { + bitmap.toNativeBuffer() + } else { + rotateToNativeBuffer(bitmap, orientation, signal) + } + } + // Don't re-check cancellation here: res owns a malloc'd buffer, and bailing to CANCELLED would + // orphan it. Deliver it; Dart frees the buffer itself if the request was cancelled meanwhile. callback(Result.success(res)) } catch (e: Exception) { callback(if (e is OperationCanceledException) CANCELLED else Result.failure(e)) } } - private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Bitmap { + // Returns the decoded bitmap plus the EXIF orientation that still needs applying. Only Q+ raw + // decodes come back unrotated (ImageDecoder / loadThumbnail skip EXIF for raw like DNG); every + // other path already orients itself, so it reports ORIENTATION_NORMAL. + private fun decodeImage(id: Long, size: Size, signal: CancellationSignal): Pair { signal.throwIfCanceled() val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, id) + val handleRaw = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q && isRawMime(uri) + val orientation = if (handleRaw) rawOrientation(uri) else ExifInterface.ORIENTATION_NORMAL + if (size.width <= 0 || size.height <= 0 || size.width > 768 || size.height > 768) { - return decodeSource(uri, size, signal) + // A "load original" request is unsized -> a full-res decode. For raw, cap it so the later + // orientation pass stays within a safe pixel budget. + val bitmap = if (handleRaw && (size.width <= 0 || size.height <= 0)) { + decodeRawCapped(uri, signal) + } else { + decodeSource(uri, size, signal) + } + return bitmap to orientation } - return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val bitmap = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { resolver.loadThumbnail(uri, size, signal) } else { signal.setOnCancelListener { Images.Thumbnails.cancelThumbnailRequest(resolver, id) } Images.Thumbnails.getThumbnail(resolver, id, Images.Thumbnails.MINI_KIND, OPTIONS) } + return bitmap to orientation + } + + private fun isRawMime(uri: Uri): Boolean { + val mime = resolver.getType(uri) ?: return false + return mime.startsWith("image/x-") || mime == "image/dng" + } + + private fun rawOrientation(uri: Uri): Int { + return resolver.openInputStream(uri)?.use { + ExifInterface(it).getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } ?: ExifInterface.ORIENTATION_NORMAL + } + + // Full-res raw decode for "load original", sampled down to MAX_RAW_DECODE_PIXELS (power of two). + // Caps resolution only; the caller still rotates the result, so even huge raws end up upright. + @RequiresApi(Build.VERSION_CODES.Q) + private fun decodeRawCapped(uri: Uri, signal: CancellationSignal): Bitmap { + signal.throwIfCanceled() + return ImageDecoder.decodeBitmap(ImageDecoder.createSource(resolver, uri)) { decoder, info, _ -> + val pixels = info.size.width.toLong() * info.size.height.toLong() + var sample = 1 + while (pixels / (sample.toLong() * sample) > MAX_RAW_DECODE_PIXELS) { + sample *= 2 + } + if (sample > 1) { + decoder.setTargetSampleSize(sample) + } + decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE + decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB)) + } + } + + // ImageDecoder / loadThumbnail skip EXIF orientation for raw (e.g. DNG) on Q+, so the decoded + // bitmap comes back unrotated. Rotate it into the output buffer in native code (one pass, no + // intermediate rotated bitmap), falling back to Skia for any config the native path can't take. + private fun rotateToNativeBuffer(bitmap: Bitmap, orientation: Int, signal: CancellationSignal): Map { + signal.throwIfCanceled() + // Force ARGB_8888 so both the native pass and the Skia fallback are 4 bytes/pixel: the native + // rotate needs a lockable 8888 buffer, and toNativeBuffer() below allocates width*height*4 (an + // F16/HDR decode would otherwise under-allocate). No-op for the common already-8888 case. + val src = if (bitmap.config != Bitmap.Config.ARGB_8888) { + val converted = bitmap.copy(Bitmap.Config.ARGB_8888, false) + bitmap.recycle() + converted ?: throw IOException("could not convert bitmap to ARGB_8888") + } else { + bitmap + } + try { + val info = IntArray(3) + val pointer = NativeImage.rotate(src, orientation, info) + if (pointer != 0L) { + return mapOf( + "pointer" to pointer, + "width" to info[0].toLong(), + "height" to info[1].toLong(), + "rowBytes" to info[2].toLong() + ) + } + // Native path declined (unsupported config) -> rotate via Skia, then copy out. + val matrix = matrixForExifOrientation(orientation) ?: return src.toNativeBuffer() + return Bitmap.createBitmap(src, 0, 0, src.width, src.height, matrix, true).toNativeBuffer() + } finally { + if (!src.isRecycled) src.recycle() + } + } + + // EXIF orientation (1-8) -> transform matrix, or null when no rotation/flip is needed. + private fun matrixForExifOrientation(orientation: Int): Matrix? { + val matrix = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> matrix.apply { postRotate(270f); postScale(-1f, 1f) } + ExifInterface.ORIENTATION_TRANSVERSE -> matrix.apply { postRotate(90f); postScale(-1f, 1f) } + else -> return null + } + return matrix } private fun decodeVideoThumbnail(id: Long, target: Size, signal: CancellationSignal): Bitmap {