diff --git a/pom.xml b/pom.xml index c6a9e0b..4b4b98b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.akka akka-javasdk-parent - 3.5.1 + 3.5.2 io.akka.ai diff --git a/src/main/java/demo/multiagent/application/ActivityView.java b/src/main/java/demo/multiagent/application/ActivityView.java index e06c431..d97143b 100644 --- a/src/main/java/demo/multiagent/application/ActivityView.java +++ b/src/main/java/demo/multiagent/application/ActivityView.java @@ -20,7 +20,7 @@ public record ActivityEntry( String finalAnswer ) {} - @Query("SELECT * as entries FROM activities WHERE userId = :userId") + @Query("SELECT * AS entries FROM activities WHERE userId = :userId") public QueryEffect getActivities(String userId) { return queryResult(); } diff --git a/src/main/java/demo/multiagent/application/EvaluatorAgent.java b/src/main/java/demo/multiagent/application/EvaluatorAgent.java index d601fa8..3ecc31a 100644 --- a/src/main/java/demo/multiagent/application/EvaluatorAgent.java +++ b/src/main/java/demo/multiagent/application/EvaluatorAgent.java @@ -5,6 +5,7 @@ import akka.javasdk.annotations.ComponentId; import akka.javasdk.client.ComponentClient; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; @ComponentId("evaluator-agent") @@ -24,7 +25,17 @@ public record EvaluationRequest( String finalAnswer ) {} - public record EvaluationResult(int score, String feedback) {} + public record EvaluationResult(String evaluation, String feedback) { + public boolean ok() { + return switch (evaluation.toLowerCase(Locale.ROOT)) { + case "correct" -> true; + case "incorrect" -> false; + default -> throw new IllegalArgumentException( + "Unknown evaluation result [" + evaluation + "]" + ); + }; + } + } private static final String SYSTEM_MESSAGE = // <1> """ @@ -37,27 +48,21 @@ public record EvaluationResult(int score, String feedback) {} 3. The overall quality, relevance, and helpfulness of the response 4. Any potential deviations or inconsistencies with user preferences - SCORING CRITERIA: - - Use a score from 1-5 where: - * 5 = Excellent response that fully addresses the question and respects all preferences - * 4 = Good response with minor issues but respects preferences - * 3 = Acceptable response that meets basic requirements and respects preferences - * 2 = Poor response with significant issues or minor preference violations - * 1 = Unacceptable response that fails to address the question or violates preferences + A response is "Incorrect" if it meets ANY of the following failure conditions: + - poor response with significant issues or minor preference violations + - unacceptable response that fails to address the question or violates preferences + + A response is "Correct" if it: + - fully addresses the question and respects all preferences + - good response with minor issues but respects preferences IMPORTANT: - - A score of 3 or higher means the answer passes evaluation (acceptable) - - A score below 3 means the answer fails evaluation (unacceptable) - - Any violations of user preferences should result in a failing score (below 3) since + - Any violations of user preferences should result in an incorrect evaluation since respecting user preferences is the most important criteria - Your response should be a JSON object with the following structure: - { - "score": , - "feedback": "", - } - - Do not include any explanations or text outside of the JSON structure. + Your response must be a single JSON object with the following fields: + - "evaluation": A string, either "Correct" or "Incorrect". + - "feedback": Specific feedback on what works well or deviations from preferences. """.stripIndent(); private final ComponentClient componentClient; @@ -81,7 +86,7 @@ public Effect evaluate(EvaluationRequest request) { return effects() .systemMessage(SYSTEM_MESSAGE) .userMessage(evaluationPrompt) - .responseAs(EvaluationResult.class) // <3> + .responseConformsTo(EvaluationResult.class) // <3> .thenReply(); } diff --git a/src/main/java/demo/multiagent/application/PlannerAgent.java b/src/main/java/demo/multiagent/application/PlannerAgent.java index 505fba0..6bbd934 100644 --- a/src/main/java/demo/multiagent/application/PlannerAgent.java +++ b/src/main/java/demo/multiagent/application/PlannerAgent.java @@ -76,7 +76,7 @@ public Effect createPlan(Request request) { return effects() .systemMessage(buildSystemMessage(request.agentSelection)) .userMessage(request.message()) - .responseAs(Plan.class) + .responseConformsTo(Plan.class) .thenReply(); } } diff --git a/src/main/java/demo/multiagent/application/PreferencesConsumer.java b/src/main/java/demo/multiagent/application/PreferencesConsumer.java index e786537..f9f52a5 100644 --- a/src/main/java/demo/multiagent/application/PreferencesConsumer.java +++ b/src/main/java/demo/multiagent/application/PreferencesConsumer.java @@ -46,13 +46,13 @@ public Effect onPreferenceAdded(PreferencesEvent.PreferenceAdded event) { .invoke(evaluationRequest); // <4> logger.info( - "Evaluation completed for session {}: score={}, feedback='{}'", + "Evaluation completed for session {}: evaluation={}, feedback='{}'", activity.sessionId(), - evaluationResult.score(), + evaluationResult.evaluation(), evaluationResult.feedback() ); - if (evaluationResult.score() < 3) { + if (!evaluationResult.ok()) { // run the workflow again to generate a better answer componentClient diff --git a/src/main/java/demo/multiagent/application/SelectorAgent.java b/src/main/java/demo/multiagent/application/SelectorAgent.java index 9f13a69..03b7aa1 100644 --- a/src/main/java/demo/multiagent/application/SelectorAgent.java +++ b/src/main/java/demo/multiagent/application/SelectorAgent.java @@ -59,7 +59,7 @@ public Effect selectAgents(String message) { return effects() .systemMessage(systemMessage) .userMessage(message) - .responseAs(AgentSelection.class) + .responseConformsTo(AgentSelection.class) .thenReply(); } } diff --git a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java index 3839739..777d9e3 100644 --- a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java +++ b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java @@ -168,7 +168,7 @@ private void setupInitialModelResponses() { evaluatorModel.fixedResponse( """ { - "score": 5, + "evaluation": "Correct", "feedback": "The suggestion is appropriate for the user." } """ @@ -177,13 +177,12 @@ private void setupInitialModelResponses() { private void setupUpdatedModelResponsesForPreference() { // Evaluator detects preference conflict and triggers new suggestion - evaluatorModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730 evaluatorModel .whenMessage(req -> req.contains("hate outdoor activities")) .reply( """ { - "score": 1, + "evaluation": "Incorrect", "feedback": "The previous suggestion conflicts with user preferences for indoor activities. Outdoor bike tours are not suitable." } """ @@ -198,7 +197,6 @@ private void setupUpdatedModelResponsesForPreference() { ); // Updated summary reflecting preference - summaryModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730 summaryModel .whenMessage(req -> req.contains("preference for indoor activities")) .reply(