From f5d6b01d47da26a86a6473982eb45bb24cf7c59c Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Thu, 26 Mar 2026 18:38:05 +0000 Subject: [PATCH 1/8] feat(ai-logic): add hybrid on-device inference sample --- firebase-ai/app/build.gradle.kts | 3 +- .../firebase/quickstart/ai/MainActivity.kt | 6 + .../quickstart/ai/feature/hybrid/Expense.kt | 10 + .../hybrid/HybridInferenceViewModel.kt | 137 +++++++++++++ .../quickstart/ai/ui/HybridInferenceScreen.kt | 191 ++++++++++++++++++ .../ai/ui/HybridInferenceUiState.kt | 10 + .../ai/ui/navigation/FirebaseAISamples.kt | 10 + .../quickstart/ai/ui/navigation/Sample.kt | 3 +- gradle/libs.versions.toml | 3 +- 9 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt create mode 100644 firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt create mode 100644 firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt create mode 100644 firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceUiState.kt diff --git a/firebase-ai/app/build.gradle.kts b/firebase-ai/app/build.gradle.kts index 2a5a4efc4..6b47e5086 100644 --- a/firebase-ai/app/build.gradle.kts +++ b/firebase-ai/app/build.gradle.kts @@ -12,7 +12,7 @@ android { defaultConfig { applicationId = "com.google.firebase.quickstart.ai" - minSdk = 23 + minSdk = 26 targetSdk = 36 versionCode = 1 versionName = "1.0" @@ -73,6 +73,7 @@ dependencies { // Firebase implementation(platform(libs.firebase.bom)) implementation(libs.firebase.ai) + implementation(libs.firebase.ai.ondevice) // Image loading implementation(libs.coil.compose) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt index 06ee42f8d..51ed6ce39 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt @@ -27,6 +27,7 @@ import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.rememberNavController import com.google.firebase.quickstart.ai.feature.live.BidiViewModel +import com.google.firebase.quickstart.ai.feature.hybrid.HybridInferenceViewModel import com.google.firebase.quickstart.ai.feature.media.imagen.ImagenViewModel import com.google.firebase.quickstart.ai.feature.text.ChatViewModel import com.google.firebase.quickstart.ai.feature.text.ServerPromptTemplateViewModel @@ -36,6 +37,7 @@ import com.google.firebase.quickstart.ai.ui.ImagenScreen import com.google.firebase.quickstart.ai.ui.ServerPromptScreen import com.google.firebase.quickstart.ai.ui.StreamRealtimeScreen import com.google.firebase.quickstart.ai.ui.StreamRealtimeVideoScreen +import com.google.firebase.quickstart.ai.ui.HybridInferenceScreen import com.google.firebase.quickstart.ai.ui.SvgScreen import com.google.firebase.quickstart.ai.ui.navigation.FIREBASE_AI_SAMPLES import com.google.firebase.quickstart.ai.ui.navigation.MainMenuScreen @@ -123,6 +125,10 @@ class MainActivity : ComponentActivity() { StreamRealtimeVideoScreen(it) } } + + ScreenType.HYBRID -> { + (vm as? HybridInferenceViewModel)?.let { HybridInferenceScreen(it) } + } } } } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt new file mode 100644 index 000000000..384820cb7 --- /dev/null +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt @@ -0,0 +1,10 @@ +package com.google.firebase.quickstart.ai.feature.hybrid + +import kotlinx.serialization.Serializable + +@Serializable +data class Expense( + val id: String, + val name: String, + val price: Double +) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt new file mode 100644 index 000000000..8233c0bc9 --- /dev/null +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -0,0 +1,137 @@ +package com.google.firebase.quickstart.ai.feature.hybrid + +import android.graphics.Bitmap +import android.util.Log +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import com.google.firebase.Firebase +import com.google.firebase.ai.InferenceMode +import com.google.firebase.ai.OnDeviceConfig +import com.google.firebase.ai.ai +import com.google.firebase.ai.ondevice.DownloadStatus +import com.google.firebase.ai.ondevice.FirebaseAIOnDevice +import com.google.firebase.ai.ondevice.OnDeviceModelStatus +import com.google.firebase.ai.type.GenerativeBackend +import com.google.firebase.ai.type.PublicPreviewAPI +import com.google.firebase.ai.type.content +import com.google.firebase.quickstart.ai.ui.HybridInferenceUiState +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import kotlinx.serialization.Serializable +import java.util.UUID + +@Serializable +object HybridInferenceRoute + +@OptIn(PublicPreviewAPI::class) +class HybridInferenceViewModel : ViewModel() { + private val _uiState = MutableStateFlow(HybridInferenceUiState( + expenses = listOf( + Expense(UUID.randomUUID().toString(), "Lunch", 15.50), + Expense(UUID.randomUUID().toString(), "Coffee", 4.75) + ) + )) + val uiState: StateFlow = _uiState.asStateFlow() + + private val model = Firebase.ai(backend = GenerativeBackend.googleAI()) + .generativeModel( + modelName = "gemini-3.1-flash-lite-preview", + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_ON_DEVICE) + ) + + init { + checkAndDownloadModel() + } + + private fun checkAndDownloadModel() { + viewModelScope.launch { + try { + val status = FirebaseAIOnDevice.checkStatus() + updateStatus(status) + + if (status == OnDeviceModelStatus.DOWNLOADABLE) { + FirebaseAIOnDevice.download().collect { downloadStatus -> + when (downloadStatus) { + is DownloadStatus.DownloadStarted -> { + _uiState.update { it.copy(modelStatus = "Downloading model...") } + } + is DownloadStatus.DownloadInProgress -> { + val progress = downloadStatus.totalBytesDownloaded + _uiState.update { it.copy(modelStatus = "Downloading: $progress bytes downloaded") } + } + is DownloadStatus.DownloadCompleted -> { + _uiState.update { it.copy(modelStatus = "Model ready") } + } + is DownloadStatus.DownloadFailed -> { + _uiState.update { it.copy(modelStatus = "Download failed", errorMessage = "Model download failed") } + } + } + } + } + } catch (e: Exception) { + Log.e("HybridVM", "Error checking model status", e) + _uiState.update { it.copy(modelStatus = "Error checking status", errorMessage = e.message) } + } + } + } + + private fun updateStatus(status: OnDeviceModelStatus) { + val statusText = when (status) { + OnDeviceModelStatus.AVAILABLE -> "Model available" + OnDeviceModelStatus.DOWNLOADABLE -> "Model downloadable" + OnDeviceModelStatus.DOWNLOADING -> "Model downloading..." + OnDeviceModelStatus.UNAVAILABLE -> "On-device model unavailable" + else -> "Unknown" + } + _uiState.update { it.copy(modelStatus = statusText) } + } + + fun scanReceipt(bitmap: Bitmap) { + viewModelScope.launch { + _uiState.update { it.copy(isScanning = true, errorMessage = null) } + try { + val prompt = content { + image(bitmap) + text("Extract the store name and the total price from this receipt. Output only in CSV format like 'Store,Price'. Example: 'Starbucks,5.50'") + } + + val response = model.generateContent(prompt) + val text = response.text + Log.d("HybridVM", "Response is: $text") + if (text != null) { + parseAndAddExpense(text) + } else { + _uiState.update { it.copy(errorMessage = "Could not extract data") } + } + } catch (e: Exception) { + Log.e("HybridVM", "Error scanning receipt", e) + _uiState.update { it.copy(errorMessage = "Error: ${e.message}") } + } finally { + _uiState.update { it.copy(isScanning = false) } + } + } + } + + private fun parseAndAddExpense(text: String) { + // Simple parsing: "Store, Price" + val parts = text + // Sometimes the output contains single quotes + .replace("'", "") + .split(",", limit = 2) + if (parts.size >= 2) { + val name = parts[0].trim() + val priceStr = parts[1].trim() + .replace("$", "") + .replace(",", "") + val price = priceStr.toDoubleOrNull() ?: 0.0 + + val newExpense = Expense(UUID.randomUUID().toString(), name, price) + _uiState.update { it.copy(expenses = it.expenses + newExpense) } + } else { + _uiState.update { it.copy(errorMessage = "Unexpected AI output format: $text") } + } + } +} diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt new file mode 100644 index 000000000..67a76d3b6 --- /dev/null +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt @@ -0,0 +1,191 @@ +package com.google.firebase.quickstart.ai.ui + +import android.graphics.BitmapFactory +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.PickVisualMediaRequest +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Add +import androidx.compose.material.icons.filled.CameraAlt +import androidx.compose.material.icons.filled.ReceiptLong +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.FloatingActionButton +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.lifecycle.viewmodel.compose.viewModel +import com.google.firebase.quickstart.ai.feature.hybrid.HybridInferenceViewModel + +@Composable +fun HybridInferenceScreen( + viewModel: HybridInferenceViewModel = viewModel() +) { + val uiState by viewModel.uiState.collectAsState() + val context = LocalContext.current + + val launcher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.PickVisualMedia(), + onResult = { uri -> + uri?.let { + try { + context.contentResolver.openInputStream(it)?.use { stream -> + val bitmap = BitmapFactory.decodeStream(stream) + bitmap?.let { viewModel.scanReceipt(it) } + } + } catch (e: Exception) { + // Handle error + } + } + } + ) + + Scaffold( + floatingActionButton = { + FloatingActionButton( + onClick = { + launcher.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) + }, + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary + ) { + if (uiState.isScanning) { + CircularProgressIndicator( + modifier = Modifier.size(24.dp), + color = MaterialTheme.colorScheme.onPrimary, + strokeWidth = 2.dp + ) + } else { + Icon(Icons.Default.CameraAlt, contentDescription = "Scan Receipt") + } + } + } + ) { padding -> + Column( + modifier = Modifier + .fillMaxSize() + .padding(padding) + .padding(16.dp) + ) { + // Model Status Card + Card( + modifier = Modifier.fillMaxWidth(), + colors = CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.secondaryContainer + ) + ) { + Row( + modifier = Modifier.padding(12.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + Icons.Default.ReceiptLong, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSecondaryContainer + ) + Spacer(modifier = Modifier.size(12.dp)) + Column { + Text( + "Hybrid AI Status", + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + Text( + uiState.modelStatus, + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + } + } + } + + Spacer(modifier = Modifier.height(16.dp)) + + Text( + "Expenses", + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold + ) + + Spacer(modifier = Modifier.height(8.dp)) + + if (uiState.expenses.isEmpty()) { + Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { + Text("No expenses yet. Scan a receipt to add one.", color = Color.Gray) + } + } else { + LazyColumn( + modifier = Modifier.weight(1f), + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + items(uiState.expenses) { expense -> + ExpenseItem(expense.name, expense.price) + } + } + } + + if (uiState.errorMessage != null) { + Spacer(modifier = Modifier.height(8.dp)) + Text( + text = uiState.errorMessage!!, + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.bodySmall + ) + } + } + } +} + +@Composable +fun ExpenseItem(name: String, price: Double) { + Card( + modifier = Modifier.fillMaxWidth(), + elevation = CardDefaults.cardElevation(defaultElevation = 2.dp) + ) { + Row( + modifier = Modifier + .padding(16.dp) + .fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically + ) { + Text( + name, + style = MaterialTheme.typography.bodyLarge, + fontWeight = FontWeight.Medium + ) + Text( + "$${String.format("%.2f", price)}", + style = MaterialTheme.typography.bodyLarge, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.primary + ) + } + } +} diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceUiState.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceUiState.kt new file mode 100644 index 000000000..eb61e6e2a --- /dev/null +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceUiState.kt @@ -0,0 +1,10 @@ +package com.google.firebase.quickstart.ai.ui + +import com.google.firebase.quickstart.ai.feature.hybrid.Expense + +data class HybridInferenceUiState( + val expenses: List = emptyList(), + val isScanning: Boolean = false, + val modelStatus: String = "Checking model status...", + val errorMessage: String? = null +) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt index 8bb2cb153..913c811f8 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt @@ -2,6 +2,8 @@ package com.google.firebase.quickstart.ai.ui.navigation import com.google.firebase.quickstart.ai.feature.live.StreamAudioViewModel import com.google.firebase.quickstart.ai.feature.live.StreamVideoViewModel +import com.google.firebase.quickstart.ai.feature.hybrid.HybridInferenceRoute +import com.google.firebase.quickstart.ai.feature.hybrid.HybridInferenceViewModel import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeAudioRoute import com.google.firebase.quickstart.ai.feature.live.StreamRealtimeVideoRoute import com.google.firebase.quickstart.ai.feature.media.imagen.ImagenGenerationRoute @@ -239,5 +241,13 @@ val FIREBASE_AI_SAMPLES = listOf( screenType = ScreenType.SVG, viewModelClass = SvgViewModel::class, categories = listOf(Category.IMAGE, Category.TEXT) + ), + Sample( + title = "Hybrid Receipt Scanner", + description = "Use hybrid inference to scan receipts and extract expense data on-device whenever possible.", + route = HybridInferenceRoute, + screenType = ScreenType.HYBRID, + viewModelClass = HybridInferenceViewModel::class, + categories = listOf(Category.TEXT, Category.IMAGE) ) ) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt index 76bb0c934..19145bb15 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt @@ -21,7 +21,8 @@ enum class ScreenType { SVG, SERVER_PROMPT, BIDI, - BIDI_VIDEO + BIDI_VIDEO, + HYBRID } data class Sample( diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0afc49131..5fc9de341 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,7 +7,7 @@ composeBom = "2025.12.00" composeNavigation = "2.9.6" coreKtx = "1.17.0" espressoCore = "3.7.0" -firebaseBom = "34.7.0" +firebaseBom = "34.11.0" googleServices = "4.4.4" firebaseCrashlytics = "3.0.6" firebasePerf = "2.0.2" @@ -52,6 +52,7 @@ coil-network-okhttp = { module = "io.coil-kt.coil3:coil-network-okhttp", version coil-svg = { module = "io.coil-kt.coil3:coil-svg", version.ref = "coil3Compose" } compose-navigation = { group = "androidx.navigation", name = "navigation-compose", version.ref = "composeNavigation"} firebase-ai = { module = "com.google.firebase:firebase-ai" } +firebase-ai-ondevice = { module = "com.google.firebase:firebase-ai-ondevice", version = "16.0.0-beta01" } firebase-bom = { module = "com.google.firebase:firebase-bom", version.ref = "firebaseBom" } junit = { group = "junit", name = "junit", version.ref = "junit" } kotlinx-serialization-core = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinxSerializationCore" } From 737c18086c28bbf3a7c58a41e897b6508b97250c Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Thu, 26 Mar 2026 18:59:46 +0000 Subject: [PATCH 2/8] add a hybrid category --- .../firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt | 2 +- .../com/google/firebase/quickstart/ai/ui/navigation/Sample.kt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt index 913c811f8..a72196be2 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/FirebaseAISamples.kt @@ -248,6 +248,6 @@ val FIREBASE_AI_SAMPLES = listOf( route = HybridInferenceRoute, screenType = ScreenType.HYBRID, viewModelClass = HybridInferenceViewModel::class, - categories = listOf(Category.TEXT, Category.IMAGE) + categories = listOf(Category.TEXT, Category.IMAGE, Category.HYBRID) ) ) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt index 19145bb15..a51b56315 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/navigation/Sample.kt @@ -12,7 +12,8 @@ enum class Category( AUDIO("Audio"), DOCUMENT("Document"), FUNCTION_CALLING("Function calling"), - LIVE_API("Live API Streaming") + LIVE_API("Live API Streaming"), + HYBRID("Hybrid inference") } enum class ScreenType { From 03dff58c0449971298052ad508b48f130b7bacbd Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Thu, 26 Mar 2026 19:03:34 +0000 Subject: [PATCH 3/8] lint ? --- .../hybrid/HybridInferenceViewModel.kt | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt index 8233c0bc9..853787ed1 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -28,16 +28,17 @@ object HybridInferenceRoute @OptIn(PublicPreviewAPI::class) class HybridInferenceViewModel : ViewModel() { - private val _uiState = MutableStateFlow(HybridInferenceUiState( - expenses = listOf( - Expense(UUID.randomUUID().toString(), "Lunch", 15.50), - Expense(UUID.randomUUID().toString(), "Coffee", 4.75) + private val _uiState = MutableStateFlow( + HybridInferenceUiState( + expenses = listOf( + Expense(UUID.randomUUID().toString(), "Lunch", 15.50), + Expense(UUID.randomUUID().toString(), "Coffee", 4.75) + ) ) - )) + ) val uiState: StateFlow = _uiState.asStateFlow() - private val model = Firebase.ai(backend = GenerativeBackend.googleAI()) - .generativeModel( + private val model = Firebase.ai(backend = GenerativeBackend.googleAI()).generativeModel( modelName = "gemini-3.1-flash-lite-preview", onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_ON_DEVICE) ) @@ -58,15 +59,22 @@ class HybridInferenceViewModel : ViewModel() { is DownloadStatus.DownloadStarted -> { _uiState.update { it.copy(modelStatus = "Downloading model...") } } + is DownloadStatus.DownloadInProgress -> { val progress = downloadStatus.totalBytesDownloaded _uiState.update { it.copy(modelStatus = "Downloading: $progress bytes downloaded") } } + is DownloadStatus.DownloadCompleted -> { _uiState.update { it.copy(modelStatus = "Model ready") } } + is DownloadStatus.DownloadFailed -> { - _uiState.update { it.copy(modelStatus = "Download failed", errorMessage = "Model download failed") } + _uiState.update { + it.copy( + modelStatus = "Download failed", errorMessage = "Model download failed" + ) + } } } } @@ -119,15 +127,12 @@ class HybridInferenceViewModel : ViewModel() { // Simple parsing: "Store, Price" val parts = text // Sometimes the output contains single quotes - .replace("'", "") - .split(",", limit = 2) + .replace("'", "").split(",", limit = 2) if (parts.size >= 2) { val name = parts[0].trim() - val priceStr = parts[1].trim() - .replace("$", "") - .replace(",", "") + val priceStr = parts[1].trim().replace("$", "").replace(",", "") val price = priceStr.toDoubleOrNull() ?: 0.0 - + val newExpense = Expense(UUID.randomUUID().toString(), name, price) _uiState.update { it.copy(expenses = it.expenses + newExpense) } } else { From c0fbec11f58ffbf14bcb8be427e6e520aec3f4f3 Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Fri, 27 Mar 2026 16:21:22 +0000 Subject: [PATCH 4/8] address review feedback --- .../feature/hybrid/HybridInferenceViewModel.kt | 16 +++++++++++----- .../quickstart/ai/ui/HybridInferenceScreen.kt | 5 +++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt index 853787ed1..3dfc8912f 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -39,9 +39,9 @@ class HybridInferenceViewModel : ViewModel() { val uiState: StateFlow = _uiState.asStateFlow() private val model = Firebase.ai(backend = GenerativeBackend.googleAI()).generativeModel( - modelName = "gemini-3.1-flash-lite-preview", - onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_ON_DEVICE) - ) + modelName = "gemini-3.1-flash-lite-preview", + onDeviceConfig = OnDeviceConfig(mode = InferenceMode.PREFER_ON_DEVICE) + ) init { checkAndDownloadModel() @@ -103,7 +103,13 @@ class HybridInferenceViewModel : ViewModel() { try { val prompt = content { image(bitmap) - text("Extract the store name and the total price from this receipt. Output only in CSV format like 'Store,Price'. Example: 'Starbucks,5.50'") + text( + """ + Extract the store name and the total price from this receipt. + Output only in CSV format like 'Store:Price'. + Example: 'Starbucks:5.50'" + """.trimIndent() + ) } val response = model.generateContent(prompt) @@ -127,7 +133,7 @@ class HybridInferenceViewModel : ViewModel() { // Simple parsing: "Store, Price" val parts = text // Sometimes the output contains single quotes - .replace("'", "").split(",", limit = 2) + .replace("'", "").split(":", limit = 2) if (parts.size >= 2) { val name = parts[0].trim() val priceStr = parts[1].trim().replace("$", "").replace(",", "") diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt index 67a76d3b6..732e2f758 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt @@ -52,14 +52,15 @@ fun HybridInferenceScreen( val launcher = rememberLauncherForActivityResult( contract = ActivityResultContracts.PickVisualMedia(), onResult = { uri -> - uri?.let { + uri?.let { imageUri -> try { - context.contentResolver.openInputStream(it)?.use { stream -> + context.contentResolver.openInputStream(imageUri)?.use { stream -> val bitmap = BitmapFactory.decodeStream(stream) bitmap?.let { viewModel.scanReceipt(it) } } } catch (e: Exception) { // Handle error + e.printStackTrace() } } } From 15f6463d87432a93cff568ac7fb27d8e0b33b368 Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Fri, 27 Mar 2026 16:53:35 +0000 Subject: [PATCH 5/8] remove expense uuid --- .../firebase/quickstart/ai/feature/hybrid/Expense.kt | 1 - .../ai/feature/hybrid/HybridInferenceViewModel.kt | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt index 384820cb7..d2ebbb097 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt @@ -4,7 +4,6 @@ import kotlinx.serialization.Serializable @Serializable data class Expense( - val id: String, val name: String, val price: Double ) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt index 3dfc8912f..0a764dd8b 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -31,8 +31,8 @@ class HybridInferenceViewModel : ViewModel() { private val _uiState = MutableStateFlow( HybridInferenceUiState( expenses = listOf( - Expense(UUID.randomUUID().toString(), "Lunch", 15.50), - Expense(UUID.randomUUID().toString(), "Coffee", 4.75) + Expense("Lunch", 15.50), + Expense("Coffee", 4.75) ) ) ) @@ -107,7 +107,9 @@ class HybridInferenceViewModel : ViewModel() { """ Extract the store name and the total price from this receipt. Output only in CSV format like 'Store:Price'. - Example: 'Starbucks:5.50'" + Examples: + - 'FakeStore:5.50' + - 'SomeStore:2.35' """.trimIndent() ) } @@ -139,7 +141,7 @@ class HybridInferenceViewModel : ViewModel() { val priceStr = parts[1].trim().replace("$", "").replace(",", "") val price = priceStr.toDoubleOrNull() ?: 0.0 - val newExpense = Expense(UUID.randomUUID().toString(), name, price) + val newExpense = Expense(name, price) _uiState.update { it.copy(expenses = it.expenses + newExpense) } } else { _uiState.update { it.copy(errorMessage = "Unexpected AI output format: $text") } From 09a7700c4b4d8758be0b562ba3369eaa739ff140 Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Fri, 27 Mar 2026 17:11:12 +0000 Subject: [PATCH 6/8] camera intent and json parsing --- .../hybrid/HybridInferenceViewModel.kt | 28 +++++++++---------- .../quickstart/ai/ui/HybridInferenceScreen.kt | 22 ++++----------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt index 0a764dd8b..9e93f7f59 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -21,6 +21,7 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import java.util.UUID @Serializable @@ -106,10 +107,11 @@ class HybridInferenceViewModel : ViewModel() { text( """ Extract the store name and the total price from this receipt. - Output only in CSV format like 'Store:Price'. + Output only in JSON format containg 2 fields '{name,price}'. + Do not include any currency signs or backticks or any text around it. Examples: - - 'FakeStore:5.50' - - 'SomeStore:2.35' + - {"name": "FakeStore", "price": "2.0"} + - {"name": "SomeMarket", "price": "3.5"} """.trimIndent() ) } @@ -132,19 +134,15 @@ class HybridInferenceViewModel : ViewModel() { } private fun parseAndAddExpense(text: String) { - // Simple parsing: "Store, Price" - val parts = text - // Sometimes the output contains single quotes - .replace("'", "").split(":", limit = 2) - if (parts.size >= 2) { - val name = parts[0].trim() - val priceStr = parts[1].trim().replace("$", "").replace(",", "") - val price = priceStr.toDoubleOrNull() ?: 0.0 - - val newExpense = Expense(name, price) + val json = text + // The on-device model sometimes outputs backticks, so we remove those + .replace("```json", "") + .replace("```", "") + try { + val newExpense = Json.decodeFromString(json) _uiState.update { it.copy(expenses = it.expenses + newExpense) } - } else { - _uiState.update { it.copy(errorMessage = "Unexpected AI output format: $text") } + } catch (e: Exception) { + _uiState.update { it.copy(errorMessage = e.localizedMessage) } } } } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt index 732e2f758..abdace6d6 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt @@ -1,8 +1,6 @@ package com.google.firebase.quickstart.ai.ui -import android.graphics.BitmapFactory import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.PickVisualMediaRequest import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -49,20 +47,10 @@ fun HybridInferenceScreen( val uiState by viewModel.uiState.collectAsState() val context = LocalContext.current - val launcher = rememberLauncherForActivityResult( - contract = ActivityResultContracts.PickVisualMedia(), - onResult = { uri -> - uri?.let { imageUri -> - try { - context.contentResolver.openInputStream(imageUri)?.use { stream -> - val bitmap = BitmapFactory.decodeStream(stream) - bitmap?.let { viewModel.scanReceipt(it) } - } - } catch (e: Exception) { - // Handle error - e.printStackTrace() - } - } + val cameraLauncher = rememberLauncherForActivityResult( + contract = ActivityResultContracts.TakePicturePreview(), + onResult = { bitmap -> + bitmap?.let { viewModel.scanReceipt(it) } } ) @@ -70,7 +58,7 @@ fun HybridInferenceScreen( floatingActionButton = { FloatingActionButton( onClick = { - launcher.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) + cameraLauncher.launch(null) }, containerColor = MaterialTheme.colorScheme.primary, contentColor = MaterialTheme.colorScheme.onPrimary From aff8f5fdd6cf42f655e4969d5f4972be9504b0d0 Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Fri, 27 Mar 2026 17:35:31 +0000 Subject: [PATCH 7/8] show inference mode on screen --- .../quickstart/ai/feature/hybrid/Expense.kt | 3 ++- .../hybrid/HybridInferenceViewModel.kt | 21 ++++++++++------- .../quickstart/ai/ui/HybridInferenceScreen.kt | 23 +++++++++++++------ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt index d2ebbb097..60bf5cf32 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/Expense.kt @@ -5,5 +5,6 @@ import kotlinx.serialization.Serializable @Serializable data class Expense( val name: String, - val price: Double + val price: Double, + val inferenceMode: String = "" ) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt index 9e93f7f59..d0644026c 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/hybrid/HybridInferenceViewModel.kt @@ -6,6 +6,7 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.google.firebase.Firebase import com.google.firebase.ai.InferenceMode +import com.google.firebase.ai.InferenceSource import com.google.firebase.ai.OnDeviceConfig import com.google.firebase.ai.ai import com.google.firebase.ai.ondevice.DownloadStatus @@ -32,8 +33,8 @@ class HybridInferenceViewModel : ViewModel() { private val _uiState = MutableStateFlow( HybridInferenceUiState( expenses = listOf( - Expense("Lunch", 15.50), - Expense("Coffee", 4.75) + Expense("Lunch", 15.50, "Example data"), + Expense("Coffee", 4.75, "Example data") ) ) ) @@ -81,7 +82,6 @@ class HybridInferenceViewModel : ViewModel() { } } } catch (e: Exception) { - Log.e("HybridVM", "Error checking model status", e) _uiState.update { it.copy(modelStatus = "Error checking status", errorMessage = e.message) } } } @@ -109,6 +109,7 @@ class HybridInferenceViewModel : ViewModel() { Extract the store name and the total price from this receipt. Output only in JSON format containg 2 fields '{name,price}'. Do not include any currency signs or backticks or any text around it. + Use dots for decimals. Examples: - {"name": "FakeStore", "price": "2.0"} - {"name": "SomeMarket", "price": "3.5"} @@ -118,14 +119,18 @@ class HybridInferenceViewModel : ViewModel() { val response = model.generateContent(prompt) val text = response.text - Log.d("HybridVM", "Response is: $text") + val inferenceMode = if (response.inferenceSource == InferenceSource.ON_DEVICE) { + "On-device" + } else { + "Cloud" + } + Log.d("HybridVM", "$inferenceMode response: $text") if (text != null) { - parseAndAddExpense(text) + parseAndAddExpense(text, inferenceMode) } else { _uiState.update { it.copy(errorMessage = "Could not extract data") } } } catch (e: Exception) { - Log.e("HybridVM", "Error scanning receipt", e) _uiState.update { it.copy(errorMessage = "Error: ${e.message}") } } finally { _uiState.update { it.copy(isScanning = false) } @@ -133,13 +138,13 @@ class HybridInferenceViewModel : ViewModel() { } } - private fun parseAndAddExpense(text: String) { + private fun parseAndAddExpense(text: String, inferenceMode: String) { val json = text // The on-device model sometimes outputs backticks, so we remove those .replace("```json", "") .replace("```", "") try { - val newExpense = Json.decodeFromString(json) + val newExpense = Json.decodeFromString(json).copy(inferenceMode = inferenceMode) _uiState.update { it.copy(expenses = it.expenses + newExpense) } } catch (e: Exception) { _uiState.update { it.copy(errorMessage = e.localizedMessage) } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt index abdace6d6..6578fdda2 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt @@ -134,7 +134,7 @@ fun HybridInferenceScreen( verticalArrangement = Arrangement.spacedBy(8.dp) ) { items(uiState.expenses) { expense -> - ExpenseItem(expense.name, expense.price) + ExpenseItem(expense.name, expense.price, expense.inferenceMode) } } } @@ -152,7 +152,7 @@ fun HybridInferenceScreen( } @Composable -fun ExpenseItem(name: String, price: Double) { +fun ExpenseItem(name: String, price: Double, inferenceMode: String) { Card( modifier = Modifier.fillMaxWidth(), elevation = CardDefaults.cardElevation(defaultElevation = 2.dp) @@ -164,11 +164,20 @@ fun ExpenseItem(name: String, price: Double) { horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically ) { - Text( - name, - style = MaterialTheme.typography.bodyLarge, - fontWeight = FontWeight.Medium - ) + Column { + Text( + name, + style = MaterialTheme.typography.bodyLarge, + fontWeight = FontWeight.Medium + ) + if (inferenceMode.isNotEmpty()) { + Text( + inferenceMode, + style = MaterialTheme.typography.labelSmall, + color = Color.Gray + ) + } + } Text( "$${String.format("%.2f", price)}", style = MaterialTheme.typography.bodyLarge, From 497a4c2fa238d5dba1164837ebc7741a3ce2d9b7 Mon Sep 17 00:00:00 2001 From: Rosario Fernandes Date: Fri, 27 Mar 2026 18:04:05 +0000 Subject: [PATCH 8/8] add camera permission check --- .../quickstart/ai/ui/HybridInferenceScreen.kt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt index 6578fdda2..dd04de643 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/HybridInferenceScreen.kt @@ -1,7 +1,10 @@ package com.google.firebase.quickstart.ai.ui +import android.Manifest +import android.content.pm.PackageManager import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -54,11 +57,25 @@ fun HybridInferenceScreen( } ) + val permissionLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestPermission() + ) { isGranted -> + if (isGranted) { + cameraLauncher.launch(null) + } + } + Scaffold( floatingActionButton = { FloatingActionButton( onClick = { - cameraLauncher.launch(null) + val permissionCheckResult = + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) + if (permissionCheckResult == PackageManager.PERMISSION_GRANTED) { + cameraLauncher.launch(null) + } else { + permissionLauncher.launch(Manifest.permission.CAMERA) + } }, containerColor = MaterialTheme.colorScheme.primary, contentColor = MaterialTheme.colorScheme.onPrimary