Skip to content

Commit 14b36d1

Browse files
Android-PowerUserAndroid-PowerUser
authored
Add files via upload
1 parent de303d4 commit 14b36d1

File tree

1 file changed

+69
-98
lines changed

1 file changed

+69
-98
lines changed

‎app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningViewModel.kt‎

Lines changed: 69 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ class PhotoReasoningViewModel(
246246
is Command.ClickButton -> "Klick auf Button: \"${it.buttonText}\""
247247
is Command.TapCoordinates -> "Tippen auf Koordinaten: (${it.x}, ${it.y})"
248248
is Command.TakeScreenshot -> "Screenshot aufnehmen"
249+
else -> "Unbekannter Befehl"
249250
}
250251
}
251252

@@ -295,6 +296,7 @@ class PhotoReasoningViewModel(
295296
is Command.ClickButton -> "Klick auf Button: \"${command.buttonText}\""
296297
is Command.TapCoordinates -> "Tippen auf Koordinaten: (${command.x}, ${command.y})"
297298
is Command.TakeScreenshot -> "Screenshot aufnehmen"
299+
else -> "Unbekannter Befehl"
298300
}
299301

300302
_commandExecutionStatus.value = "Führe aus: $commandDescription (${index + 1}/${commands.size})"
@@ -368,136 +370,105 @@ class PhotoReasoningViewModel(
368370
// Show toast
369371
Toast.makeText(context, "Verarbeite Screenshot...", Toast.LENGTH_SHORT).show()
370372

371-
// Create message text with screen information if available
373+
// Create message text with screen info if available
372374
val messageText = if (screenInfo != null) {
373-
"Screenshot aufgenommen\n\n$screenInfo"
375+
"Screenshot aufgenommen\n\nBildschirmelemente:\n$screenInfo"
374376
} else {
375377
"Screenshot aufgenommen"
376378
}
377379

378-
// Add screenshot message to chat history
379-
val screenshotMessage = PhotoReasoningMessage(
380-
text = messageText,
381-
participant = PhotoParticipant.USER,
382-
imageUris = listOf(screenshotUri.toString())
383-
)
384-
_chatState.addMessage(screenshotMessage)
385-
_chatMessagesFlow.value = chatMessages
386-
387-
// Save chat history after adding screenshot
388-
saveChatHistory(context)
389-
390-
// Process the screenshot
391-
val imageRequest = imageRequestBuilder!!
380+
// Load the bitmap from the URI
381+
val request = imageRequestBuilder!!
392382
.data(screenshotUri)
383+
.size(1024, 1024)
393384
.precision(Precision.EXACT)
394385
.build()
395386

396-
try {
397-
val result = imageLoader!!.execute(imageRequest)
398-
if (result is SuccessResult) {
399-
Log.d(TAG, "Successfully processed screenshot")
400-
val bitmap = (result.drawable as BitmapDrawable).bitmap
401-
402-
// Add the screenshot to the current images
403-
val updatedImages = currentSelectedImages.toMutableList()
404-
updatedImages.add(bitmap)
405-
406-
// Update the current selected images - only keep the latest screenshot
407-
currentSelectedImages = listOf(bitmap)
408-
409-
// Update status
410-
_commandExecutionStatus.value = "Screenshot hinzugefügt, sende an KI..."
411-
412-
// Show toast
413-
Toast.makeText(context, "Screenshot hinzugefügt, sende an KI...", Toast.LENGTH_SHORT).show()
414-
415-
// Create prompt with screen information if available
416-
val prompt = if (screenInfo != null) {
417-
"Analysiere diesen Screenshot. Hier sind die verfügbaren Bildschirmelemente: $screenInfo"
418-
} else {
419-
"Analysiere diesen Screenshot"
420-
}
421-
422-
// Re-send the query with only the latest screenshot
423-
reason(prompt, listOf(bitmap))
424-
425-
// Show a toast to indicate the screenshot was added
426-
Toast.makeText(context, "Screenshot zur Konversation hinzugefügt", Toast.LENGTH_SHORT).show()
427-
} else {
428-
Log.e(TAG, "Failed to process screenshot: result is not SuccessResult")
429-
_commandExecutionStatus.value = "Fehler bei der Screenshot-Verarbeitung"
430-
Toast.makeText(context, "Fehler bei der Screenshot-Verarbeitung", Toast.LENGTH_SHORT).show()
431-
432-
// Add error message to chat
433-
_chatState.addMessage(
434-
PhotoReasoningMessage(
435-
text = "Fehler bei der Screenshot-Verarbeitung",
436-
participant = PhotoParticipant.ERROR
437-
)
438-
)
439-
_chatMessagesFlow.value = chatMessages
440-
441-
// Save chat history after adding error message
442-
saveChatHistory(context)
443-
}
444-
} catch (e: Exception) {
445-
Log.e(TAG, "Error processing screenshot: ${e.message}", e)
446-
_commandExecutionStatus.value = "Fehler bei der Screenshot-Verarbeitung: ${e.message}"
447-
Toast.makeText(context, "Fehler bei der Screenshot-Verarbeitung: ${e.message}", Toast.LENGTH_SHORT).show()
387+
val result = withContext(Dispatchers.IO) {
388+
imageLoader!!.execute(request)
389+
}
390+
391+
if (result is SuccessResult) {
392+
// Get the bitmap from the result
393+
val bitmap = (result.drawable as BitmapDrawable).bitmap
448394

449-
// Add error message to chat
450-
_chatState.addMessage(
451-
PhotoReasoningMessage(
452-
text = "Fehler bei der Screenshot-Verarbeitung: ${e.message}",
453-
participant = PhotoParticipant.ERROR
454-
)
395+
// Add the screenshot to the conversation
396+
val screenshotMessage = PhotoReasoningMessage(
397+
text = messageText,
398+
participant = PhotoParticipant.USER,
399+
isPending = false,
400+
image = bitmap
455401
)
402+
403+
// Add the message to chat history
404+
_chatState.addMessage(screenshotMessage)
456405
_chatMessagesFlow.value = chatMessages
457406

458-
// Save chat history after adding error message
407+
// Save chat history
459408
saveChatHistory(context)
409+
410+
// Update the current selected images to only include this screenshot
411+
currentSelectedImages = listOf(bitmap)
412+
413+
// Update status
414+
_commandExecutionStatus.value = "Screenshot zur Konversation hinzugefügt"
415+
416+
// Show toast
417+
Toast.makeText(context, "Screenshot zur Konversation hinzugefügt", Toast.LENGTH_SHORT).show()
418+
419+
// Rebuild chat history to include the new screenshot
420+
rebuildChatHistory()
421+
} else {
422+
Log.e(TAG, "Failed to load screenshot bitmap")
423+
_commandExecutionStatus.value = "Fehler beim Laden des Screenshots"
424+
425+
// Show toast
426+
Toast.makeText(context, "Fehler beim Laden des Screenshots", Toast.LENGTH_SHORT).show()
460427
}
461428
} catch (e: Exception) {
462429
Log.e(TAG, "Error adding screenshot to conversation: ${e.message}", e)
463430
_commandExecutionStatus.value = "Fehler beim Hinzufügen des Screenshots: ${e.message}"
464-
Toast.makeText(context, "Fehler beim Hinzufügen des Screenshots: ${e.message}", Toast.LENGTH_SHORT).show()
465431

466-
// Add error message to chat
467-
_chatState.addMessage(
468-
PhotoReasoningMessage(
469-
text = "Fehler beim Hinzufügen des Screenshots: ${e.message}",
470-
participant = PhotoParticipant.ERROR
471-
)
472-
)
473-
_chatMessagesFlow.value = chatMessages
474-
475-
// Save chat history after adding error message
476-
saveChatHistory(context)
432+
// Show toast
433+
Toast.makeText(context, "Fehler beim Hinzufügen des Screenshots: ${e.message}", Toast.LENGTH_SHORT).show()
477434
}
478435
}
479436
}
480437

481438
/**
482-
* Load saved chat history from SharedPreferences and initialize chat with history
439+
* Load chat history from SharedPreferences
483440
*/
484441
fun loadChatHistory(context: android.content.Context) {
485-
val savedMessages = ChatHistoryPreferences.loadChatMessages(context)
486-
if (savedMessages.isNotEmpty()) {
487-
_chatState.clearMessages()
488-
savedMessages.forEach { _chatState.addMessage(it) }
489-
_chatMessagesFlow.value = chatMessages
490-
491-
// Rebuild the chat history for the AI
492-
rebuildChatHistory()
442+
PhotoReasoningApplication.applicationScope.launch(Dispatchers.Main) {
443+
try {
444+
val messages = ChatHistoryPreferences.loadChatMessages(context)
445+
446+
if (messages.isNotEmpty()) {
447+
// Clear current messages
448+
_chatState.clearMessages()
449+
450+
// Add loaded messages
451+
messages.forEach { _chatState.addMessage(it) }
452+
453+
// Update the flow
454+
_chatMessagesFlow.value = chatMessages
455+
456+
// Rebuild chat history
457+
rebuildChatHistory()
458+
}
459+
} catch (e: Exception) {
460+
Log.e(TAG, "Error loading chat history: ${e.message}", e)
461+
462+
// Show toast
463+
Toast.makeText(context, "Fehler beim Laden des Chat-Verlaufs: ${e.message}", Toast.LENGTH_SHORT).show()
464+
}
493465
}
494466
}
495467

496468
/**
497-
* Rebuild the chat history for the AI based on the current messages
469+
* Rebuild the chat history for the AI
498470
*/
499471
private fun rebuildChatHistory() {
500-
// Convert the current chat messages to Content objects for the chat history
501472
val history = mutableListOf<Content>()
502473

503474
// Group messages by participant to create proper conversation turns

0 commit comments

Comments
 (0)