Merge pull request #2598 from vitorpamplona/claude/add-image-labeling-Yf6vg

Add AI-powered alt-text suggestions for images

amethyst/build.gradle

@@ -346,6 +346,11 @@ dependencies {
     playImplementation libs.google.mlkit.genai.prompt
     playImplementation libs.google.mlkit.genai.rewriting
 
+    // On-device alt-text suggestions: genai image description (preferred, descriptive sentences)
+    // with image-labeling as a keyword-join fallback for devices without AICore.
+    playImplementation libs.google.mlkit.genai.image.description
+    playImplementation libs.google.mlkit.image.labeling
+
     // PushNotifications
     playImplementation platform(libs.firebase.bom)
     playImplementation libs.firebase.messaging

amethyst/src/fdroid/java/com/vitorpamplona/amethyst/service/ai/MLKitImageLabelService.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.service.ai
+
+import android.content.Context
+import android.net.Uri
+
+@Suppress("UNUSED_PARAMETER")
+class MLKitImageLabelService(
+    context: Context,
+) {
+    suspend fun labelImage(uri: Uri): List<Pair<String, Float>> = emptyList()
+
+    suspend fun suggestAltText(uri: Uri): String? = null
+
+    fun close() {}
+}

amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/creators/uploads/ImageVideoDescription.kt

@@ -31,15 +31,24 @@ import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.layout.size
 import androidx.compose.foundation.layout.windowInsetsPadding
 import androidx.compose.foundation.text.KeyboardOptions
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.filled.AutoAwesome
+import androidx.compose.material.icons.filled.Close
+import androidx.compose.material3.AssistChip
+import androidx.compose.material3.AssistChipDefaults
 import androidx.compose.material3.Button
 import androidx.compose.material3.ButtonDefaults
+import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.HorizontalDivider
+import androidx.compose.material3.Icon
 import androidx.compose.material3.IconButton
 import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.OutlinedTextField
 import androidx.compose.material3.Slider
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.DisposableEffect
+import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableIntStateOf
@@ -50,12 +59,14 @@ import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.draw.clip
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.text.font.FontWeight
 import androidx.compose.ui.text.input.KeyboardCapitalization
 import androidx.compose.ui.text.style.TextOverflow
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.unit.sp
 import com.vitorpamplona.amethyst.R
+import com.vitorpamplona.amethyst.service.ai.MLKitImageLabelService
 import com.vitorpamplona.amethyst.service.uploads.MultiOrchestrator
 import com.vitorpamplona.amethyst.ui.actions.mediaServers.DEFAULT_MEDIA_SERVERS
 import com.vitorpamplona.amethyst.ui.actions.mediaServers.ServerName
@@ -100,6 +111,34 @@ fun ImageVideoDescription(
     var message by remember { mutableStateOf("") }
     var sensitiveContent by remember { mutableStateOf(false) }
 
+    val context = LocalContext.current
+    val firstImageUri =
+        remember(uris) {
+            uris.first().takeIf { it.media.isImage() == true && it.media.isGif().not() }?.media?.uri
+        }
+    val labelService = remember { MLKitImageLabelService(context.applicationContext) }
+    var isLabeling by remember { mutableStateOf(false) }
+    var aiSuggested by remember { mutableStateOf(false) }
+
+    DisposableEffect(labelService) {
+        onDispose { labelService.close() }
+    }
+
+    LaunchedEffect(firstImageUri) {
+        if (firstImageUri == null || message.isNotEmpty()) return@LaunchedEffect
+        isLabeling = true
+        val suggestion =
+            try {
+                labelService.suggestAltText(firstImageUri)
+            } finally {
+                isLabeling = false
+            }
+        if (suggestion != null && message.isEmpty()) {
+            message = suggestion
+            aiSuggested = true
+        }
+    }
+
     // 0 = Low, 1 = Medium, 2 = High, 3=UNCOMPRESSED
     var mediaQualitySlider by remember {
         mutableIntStateOf(if (uris.hasNonMedia()) 3 else 1)
@@ -238,13 +277,24 @@ fun ImageVideoDescription(
                             .fillMaxWidth()
                             .windowInsetsPadding(WindowInsets(0.dp, 0.dp, 0.dp, 0.dp)),
                     value = message,
-                    onValueChange = { message = it },
+                    onValueChange = {
+                        message = it
+                        aiSuggested = false
+                    },
                     placeholder = {
                         Text(
                             text = stringRes(R.string.content_description_example),
                             color = MaterialTheme.colorScheme.placeholderText,
                         )
                     },
+                    trailingIcon = {
+                        if (isLabeling) {
+                            CircularProgressIndicator(
+                                modifier = Modifier.size(20.dp),
+                                strokeWidth = 2.dp,
+                            )
+                        }
+                    },
                     keyboardOptions =
                         KeyboardOptions.Default.copy(
                             capitalization = KeyboardCapitalization.Sentences,
@@ -252,6 +302,38 @@ fun ImageVideoDescription(
                 )
             }
 
+            if (aiSuggested) {
+                Row(
+                    verticalAlignment = Alignment.CenterVertically,
+                    modifier =
+                        Modifier
+                            .fillMaxWidth()
+                            .padding(top = 4.dp),
+                ) {
+                    AssistChip(
+                        onClick = {
+                            message = ""
+                            aiSuggested = false
+                        },
+                        label = { Text(text = stringRes(R.string.ai_suggested_alt_text_hint)) },
+                        leadingIcon = {
+                            Icon(
+                                imageVector = Icons.Default.AutoAwesome,
+                                contentDescription = null,
+                                modifier = Modifier.size(AssistChipDefaults.IconSize),
+                            )
+                        },
+                        trailingIcon = {
+                            Icon(
+                                imageVector = Icons.Default.Close,
+                                contentDescription = stringRes(R.string.ai_suggested_alt_text_dismiss),
+                                modifier = Modifier.size(AssistChipDefaults.IconSize),
+                            )
+                        },
+                    )
+                }
+            }
+
             // Hide privacy toggle when any selected video will be compressed (compression already strips metadata)
             val isVideoWithCompression = uris.hasVideo() && mediaQualitySlider != 3
 

amethyst/src/main/res/values/strings.xml

@@ -633,6 +633,8 @@
 
     <string name="content_description">Description of the contents</string>
     <string name="content_description_example">A blue boat in a white sandy beach at sunset</string>
+    <string name="ai_suggested_alt_text_hint">AI-suggested, edit me</string>
+    <string name="ai_suggested_alt_text_dismiss">Dismiss AI suggestion</string>
 
     <string name="zap_type">Zap Type</string>
     <string name="zap_type_explainer">Zap Type for all options</string>

amethyst/src/play/java/com/vitorpamplona/amethyst/service/ai/MLKitImageLabelService.kt

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2025 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.service.ai
+
+import android.content.Context
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.net.Uri
+import com.google.mlkit.genai.common.FeatureStatus
+import com.google.mlkit.genai.imagedescription.ImageDescriber
+import com.google.mlkit.genai.imagedescription.ImageDescriberOptions
+import com.google.mlkit.genai.imagedescription.ImageDescription
+import com.google.mlkit.genai.imagedescription.ImageDescriptionRequest
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.label.ImageLabeler
+import com.google.mlkit.vision.label.ImageLabeling
+import com.google.mlkit.vision.label.defaults.ImageLabelerOptions
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.withContext
+import kotlin.coroutines.resume
+
+/**
+ * Unified alt-text suggestion service.
+ *
+ * Prefers Gemini-Nano-backed `genai-image-description` for full descriptive sentences when
+ * the device supports AICore; falls back to the legacy keyword `image-labeling` model otherwise.
+ */
+class MLKitImageLabelService(
+    private val context: Context,
+) {
+    private var labeler: ImageLabeler? = null
+    private var describer: ImageDescriber? = null
+
+    // FeatureStatus is an Int enum. Cached per-instance — describer availability does not flip
+    // mid-session in practice, and one composer mount only needs to ask AICore once.
+    @Volatile private var cachedGenAiStatus: Int? = null
+
+    private fun ensureLabeler(): ImageLabeler =
+        labeler ?: ImageLabeling.getClient(ImageLabelerOptions.DEFAULT_OPTIONS).also { labeler = it }
+
+    private fun ensureDescriber(): ImageDescriber? =
+        describer
+            ?: try {
+                ImageDescription
+                    .getClient(ImageDescriberOptions.builder(context).build())
+                    .also { describer = it }
+            } catch (_: Exception) {
+                null
+            }
+
+    suspend fun labelImage(uri: Uri): List<Pair<String, Float>> =
+        withContext(Dispatchers.IO) {
+            try {
+                val image = InputImage.fromFilePath(context, uri)
+                val client = ensureLabeler()
+                suspendCancellableCoroutine { cont ->
+                    client
+                        .process(image)
+                        .addOnSuccessListener { labels ->
+                            cont.resume(labels.map { it.text to it.confidence })
+                        }.addOnFailureListener {
+                            cont.resume(emptyList())
+                        }
+                }
+            } catch (_: Exception) {
+                emptyList()
+            }
+        }
+
+    suspend fun suggestAltText(uri: Uri): String? = describeWithGenAi(uri) ?: labelKeywords(uri)
+
+    private suspend fun describeWithGenAi(uri: Uri): String? =
+        withContext(Dispatchers.IO) {
+            val client = ensureDescriber() ?: return@withContext null
+            try {
+                val status =
+                    cachedGenAiStatus ?: client.checkFeatureStatus().get().also { cachedGenAiStatus = it }
+                if (status != FeatureStatus.AVAILABLE) return@withContext null
+                val bitmap = loadDownscaledBitmap(uri) ?: return@withContext null
+                val request = ImageDescriptionRequest.builder(bitmap).build()
+                client
+                    .runInference(request)
+                    .get()
+                    .description
+                    ?.trim()
+                    ?.takeIf { it.isNotEmpty() }
+            } catch (_: Exception) {
+                null
+            }
+        }
+
+    private suspend fun labelKeywords(uri: Uri): String? {
+        val labels = labelImage(uri)
+        val confident = labels.filter { it.second >= MIN_CONFIDENCE }.map { it.first }
+        if (confident.isEmpty()) return null
+        return confident.take(MAX_LABELS).joinToString(", ")
+    }
+
+    // Two-pass decode keeps a 12 MP camera shot from blowing past 40 MB of ARGB_8888 — the
+    // on-device describer downscales internally anyway, so a ~1024 px input is plenty.
+    private fun loadDownscaledBitmap(uri: Uri): Bitmap? =
+        try {
+            val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
+            context.contentResolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, bounds) }
+            val opts =
+                BitmapFactory.Options().apply {
+                    inSampleSize = sampleSizeFor(bounds.outWidth, bounds.outHeight, TARGET_DIM_PX)
+                }
+            context.contentResolver.openInputStream(uri)?.use { BitmapFactory.decodeStream(it, null, opts) }
+        } catch (_: Exception) {
+            null
+        }
+
+    private fun sampleSizeFor(
+        width: Int,
+        height: Int,
+        target: Int,
+    ): Int {
+        if (width <= 0 || height <= 0) return 1
+        var sample = 1
+        var maxDim = maxOf(width, height)
+        while (maxDim / sample > target) sample *= 2
+        return sample
+    }
+
+    fun close() {
+        labeler?.close()
+        labeler = null
+        describer?.close()
+        describer = null
+        cachedGenAiStatus = null
+    }
+
+    companion object {
+        private const val MIN_CONFIDENCE = 0.6f
+        private const val MAX_LABELS = 5
+        private const val TARGET_DIM_PX = 1024
+    }
+}

gradle/libs.versions.toml

@@ -40,6 +40,8 @@ kotlinxSerialization = "1.11.0"
 genaiProofreading = "1.0.0-beta1"
 genaiPrompt = "1.0.0-beta2"
 genaiRewriting = "1.0.0-beta1"
+genaiImageDescription = "1.0.0-beta1"
+imageLabeling = "16.0.0"
 languageId = "17.0.6"
 lifecycleRuntimeKtx = "2.10.0"
 lightcompressor-enhanced = "2.2.1"
@@ -148,6 +150,8 @@ jetbrains-compose-ui-tooling-preview = { module = "org.jetbrains.compose.ui:ui-t
 google-mlkit-genai-proofreading = { group = "com.google.mlkit", name = "genai-proofreading", version.ref = "genaiProofreading" }
 google-mlkit-genai-prompt = { group = "com.google.mlkit", name = "genai-prompt", version.ref = "genaiPrompt" }
 google-mlkit-genai-rewriting = { group = "com.google.mlkit", name = "genai-rewriting", version.ref = "genaiRewriting" }
+google-mlkit-genai-image-description = { group = "com.google.mlkit", name = "genai-image-description", version.ref = "genaiImageDescription" }
+google-mlkit-image-labeling = { group = "com.google.android.gms", name = "play-services-mlkit-image-labeling", version.ref = "imageLabeling" }
 google-mlkit-language-id = { group = "com.google.mlkit", name = "language-id", version.ref = "languageId" }
 google-mlkit-translate = { group = "com.google.mlkit", name = "translate", version.ref = "translate" }
 jackson-module-kotlin = { group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin", version.ref = "jacksonModuleKotlin" }