diff --git a/pom.xml b/pom.xml
index c6a9e0b..4b4b98b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
io.akka
akka-javasdk-parent
- 3.5.1
+ 3.5.2
io.akka.ai
diff --git a/src/main/java/demo/multiagent/application/ActivityView.java b/src/main/java/demo/multiagent/application/ActivityView.java
index e06c431..d97143b 100644
--- a/src/main/java/demo/multiagent/application/ActivityView.java
+++ b/src/main/java/demo/multiagent/application/ActivityView.java
@@ -20,7 +20,7 @@ public record ActivityEntry(
String finalAnswer
) {}
- @Query("SELECT * as entries FROM activities WHERE userId = :userId")
+ @Query("SELECT * AS entries FROM activities WHERE userId = :userId")
public QueryEffect getActivities(String userId) {
return queryResult();
}
diff --git a/src/main/java/demo/multiagent/application/EvaluatorAgent.java b/src/main/java/demo/multiagent/application/EvaluatorAgent.java
index d601fa8..3ecc31a 100644
--- a/src/main/java/demo/multiagent/application/EvaluatorAgent.java
+++ b/src/main/java/demo/multiagent/application/EvaluatorAgent.java
@@ -5,6 +5,7 @@
import akka.javasdk.annotations.ComponentId;
import akka.javasdk.client.ComponentClient;
import java.util.List;
+import java.util.Locale;
import java.util.stream.Collectors;
@ComponentId("evaluator-agent")
@@ -24,7 +25,17 @@ public record EvaluationRequest(
String finalAnswer
) {}
- public record EvaluationResult(int score, String feedback) {}
+ public record EvaluationResult(String evaluation, String feedback) {
+ public boolean ok() {
+ return switch (evaluation.toLowerCase(Locale.ROOT)) {
+ case "correct" -> true;
+ case "incorrect" -> false;
+ default -> throw new IllegalArgumentException(
+ "Unknown evaluation result [" + evaluation + "]"
+ );
+ };
+ }
+ }
private static final String SYSTEM_MESSAGE = // <1>
"""
@@ -37,27 +48,21 @@ public record EvaluationResult(int score, String feedback) {}
3. The overall quality, relevance, and helpfulness of the response
4. Any potential deviations or inconsistencies with user preferences
- SCORING CRITERIA:
- - Use a score from 1-5 where:
- * 5 = Excellent response that fully addresses the question and respects all preferences
- * 4 = Good response with minor issues but respects preferences
- * 3 = Acceptable response that meets basic requirements and respects preferences
- * 2 = Poor response with significant issues or minor preference violations
- * 1 = Unacceptable response that fails to address the question or violates preferences
+ A response is "Incorrect" if it meets ANY of the following failure conditions:
+ - poor response with significant issues or minor preference violations
+ - unacceptable response that fails to address the question or violates preferences
+
+ A response is "Correct" if it:
+ - fully addresses the question and respects all preferences
+ - good response with minor issues but respects preferences
IMPORTANT:
- - A score of 3 or higher means the answer passes evaluation (acceptable)
- - A score below 3 means the answer fails evaluation (unacceptable)
- - Any violations of user preferences should result in a failing score (below 3) since
+ - Any violations of user preferences should result in an incorrect evaluation since
respecting user preferences is the most important criteria
- Your response should be a JSON object with the following structure:
- {
- "score": ,
- "feedback": "",
- }
-
- Do not include any explanations or text outside of the JSON structure.
+ Your response must be a single JSON object with the following fields:
+ - "evaluation": A string, either "Correct" or "Incorrect".
+ - "feedback": Specific feedback on what works well or deviations from preferences.
""".stripIndent();
private final ComponentClient componentClient;
@@ -81,7 +86,7 @@ public Effect evaluate(EvaluationRequest request) {
return effects()
.systemMessage(SYSTEM_MESSAGE)
.userMessage(evaluationPrompt)
- .responseAs(EvaluationResult.class) // <3>
+ .responseConformsTo(EvaluationResult.class) // <3>
.thenReply();
}
diff --git a/src/main/java/demo/multiagent/application/PlannerAgent.java b/src/main/java/demo/multiagent/application/PlannerAgent.java
index 505fba0..6bbd934 100644
--- a/src/main/java/demo/multiagent/application/PlannerAgent.java
+++ b/src/main/java/demo/multiagent/application/PlannerAgent.java
@@ -76,7 +76,7 @@ public Effect createPlan(Request request) {
return effects()
.systemMessage(buildSystemMessage(request.agentSelection))
.userMessage(request.message())
- .responseAs(Plan.class)
+ .responseConformsTo(Plan.class)
.thenReply();
}
}
diff --git a/src/main/java/demo/multiagent/application/PreferencesConsumer.java b/src/main/java/demo/multiagent/application/PreferencesConsumer.java
index e786537..f9f52a5 100644
--- a/src/main/java/demo/multiagent/application/PreferencesConsumer.java
+++ b/src/main/java/demo/multiagent/application/PreferencesConsumer.java
@@ -46,13 +46,13 @@ public Effect onPreferenceAdded(PreferencesEvent.PreferenceAdded event) {
.invoke(evaluationRequest); // <4>
logger.info(
- "Evaluation completed for session {}: score={}, feedback='{}'",
+ "Evaluation completed for session {}: evaluation={}, feedback='{}'",
activity.sessionId(),
- evaluationResult.score(),
+ evaluationResult.evaluation(),
evaluationResult.feedback()
);
- if (evaluationResult.score() < 3) {
+ if (!evaluationResult.ok()) {
// run the workflow again to generate a better answer
componentClient
diff --git a/src/main/java/demo/multiagent/application/SelectorAgent.java b/src/main/java/demo/multiagent/application/SelectorAgent.java
index 9f13a69..03b7aa1 100644
--- a/src/main/java/demo/multiagent/application/SelectorAgent.java
+++ b/src/main/java/demo/multiagent/application/SelectorAgent.java
@@ -59,7 +59,7 @@ public Effect selectAgents(String message) {
return effects()
.systemMessage(systemMessage)
.userMessage(message)
- .responseAs(AgentSelection.class)
+ .responseConformsTo(AgentSelection.class)
.thenReply();
}
}
diff --git a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java
index 3839739..777d9e3 100644
--- a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java
+++ b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java
@@ -168,7 +168,7 @@ private void setupInitialModelResponses() {
evaluatorModel.fixedResponse(
"""
{
- "score": 5,
+ "evaluation": "Correct",
"feedback": "The suggestion is appropriate for the user."
}
"""
@@ -177,13 +177,12 @@ private void setupInitialModelResponses() {
private void setupUpdatedModelResponsesForPreference() {
// Evaluator detects preference conflict and triggers new suggestion
- evaluatorModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730
evaluatorModel
.whenMessage(req -> req.contains("hate outdoor activities"))
.reply(
"""
{
- "score": 1,
+ "evaluation": "Incorrect",
"feedback": "The previous suggestion conflicts with user preferences for indoor activities. Outdoor bike tours are not suitable."
}
"""
@@ -198,7 +197,6 @@ private void setupUpdatedModelResponsesForPreference() {
);
// Updated summary reflecting preference
- summaryModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730
summaryModel
.whenMessage(req -> req.contains("preference for indoor activities"))
.reply(