Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<groupId>io.akka</groupId>
<artifactId>akka-javasdk-parent</artifactId>
<version>3.5.1</version>
<version>3.5.2</version>
</parent>

<groupId>io.akka.ai</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public record ActivityEntry(
String finalAnswer
) {}

@Query("SELECT * as entries FROM activities WHERE userId = :userId")
@Query("SELECT * AS entries FROM activities WHERE userId = :userId")
public QueryEffect<ActivityEntries> getActivities(String userId) {
return queryResult();
}
Expand Down
43 changes: 24 additions & 19 deletions src/main/java/demo/multiagent/application/EvaluatorAgent.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import akka.javasdk.annotations.ComponentId;
import akka.javasdk.client.ComponentClient;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;

@ComponentId("evaluator-agent")
Expand All @@ -24,7 +25,17 @@ public record EvaluationRequest(
String finalAnswer
) {}

public record EvaluationResult(int score, String feedback) {}
public record EvaluationResult(String evaluation, String feedback) {
public boolean ok() {
return switch (evaluation.toLowerCase(Locale.ROOT)) {
case "correct" -> true;
case "incorrect" -> false;
default -> throw new IllegalArgumentException(
"Unknown evaluation result [" + evaluation + "]"
);
};
}
}

private static final String SYSTEM_MESSAGE = // <1>
"""
Expand All @@ -37,27 +48,21 @@ public record EvaluationResult(int score, String feedback) {}
3. The overall quality, relevance, and helpfulness of the response
4. Any potential deviations or inconsistencies with user preferences

SCORING CRITERIA:
- Use a score from 1-5 where:
* 5 = Excellent response that fully addresses the question and respects all preferences
* 4 = Good response with minor issues but respects preferences
* 3 = Acceptable response that meets basic requirements and respects preferences
* 2 = Poor response with significant issues or minor preference violations
* 1 = Unacceptable response that fails to address the question or violates preferences
A response is "Incorrect" if it meets ANY of the following failure conditions:
- poor response with significant issues or minor preference violations
- unacceptable response that fails to address the question or violates preferences

A response is "Correct" if it:
- fully addresses the question and respects all preferences
- good response with minor issues but respects preferences

IMPORTANT:
- A score of 3 or higher means the answer passes evaluation (acceptable)
- A score below 3 means the answer fails evaluation (unacceptable)
- Any violations of user preferences should result in a failing score (below 3) since
- Any violations of user preferences should result in an incorrect evaluation since
respecting user preferences is the most important criteria

Your response should be a JSON object with the following structure:
{
"score": <integer from 1-5>,
"feedback": "<specific feedback on what works well or deviations from preferences>",
}

Do not include any explanations or text outside of the JSON structure.
Your response must be a single JSON object with the following fields:
- "evaluation": A string, either "Correct" or "Incorrect".
- "feedback": Specific feedback on what works well or deviations from preferences.
""".stripIndent();

private final ComponentClient componentClient;
Expand All @@ -81,7 +86,7 @@ public Effect<EvaluationResult> evaluate(EvaluationRequest request) {
return effects()
.systemMessage(SYSTEM_MESSAGE)
.userMessage(evaluationPrompt)
.responseAs(EvaluationResult.class) // <3>
.responseConformsTo(EvaluationResult.class) // <3>
.thenReply();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public Effect<Plan> createPlan(Request request) {
return effects()
.systemMessage(buildSystemMessage(request.agentSelection))
.userMessage(request.message())
.responseAs(Plan.class)
.responseConformsTo(Plan.class)
.thenReply();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,13 @@ public Effect onPreferenceAdded(PreferencesEvent.PreferenceAdded event) {
.invoke(evaluationRequest); // <4>

logger.info(
"Evaluation completed for session {}: score={}, feedback='{}'",
"Evaluation completed for session {}: evaluation={}, feedback='{}'",
activity.sessionId(),
evaluationResult.score(),
evaluationResult.evaluation(),
evaluationResult.feedback()
);

if (evaluationResult.score() < 3) {
if (!evaluationResult.ok()) {
// run the workflow again to generate a better answer

componentClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public Effect<AgentSelection> selectAgents(String message) {
return effects()
.systemMessage(systemMessage)
.userMessage(message)
.responseAs(AgentSelection.class)
.responseConformsTo(AgentSelection.class)
.thenReply();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ private void setupInitialModelResponses() {
evaluatorModel.fixedResponse(
"""
{
"score": 5,
"evaluation": "Correct",
"feedback": "The suggestion is appropriate for the user."
}
"""
Expand All @@ -177,13 +177,12 @@ private void setupInitialModelResponses() {

private void setupUpdatedModelResponsesForPreference() {
// Evaluator detects preference conflict and triggers new suggestion
evaluatorModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730
evaluatorModel
.whenMessage(req -> req.contains("hate outdoor activities"))
.reply(
"""
{
"score": 1,
"evaluation": "Incorrect",
"feedback": "The previous suggestion conflicts with user preferences for indoor activities. Outdoor bike tours are not suitable."
}
"""
Expand All @@ -198,7 +197,6 @@ private void setupUpdatedModelResponsesForPreference() {
);

// Updated summary reflecting preference
summaryModel.reset(); // FIXME this should not be needed, https://github.com/akka/akka-sdk/pull/730
summaryModel
.whenMessage(req -> req.contains("preference for indoor activities"))
.reply(
Expand Down
Loading