diff --git a/README.md b/README.md index 79773ed..fdcb2d5 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Build container image: ```shell mvn clean install -DskipTests ``` -Install the `akka` CLI as documented in [Install Akka CLI](https://doc.akka.io/reference/cli/index.html). +Install the `akka` CLI as documented in [Install Akka CLI](https://doc.akka.io/operations/cli/installation.html). Set up secret containing OpenAI API key: ```shell diff --git a/pom.xml b/pom.xml index e2bda63..4cf8a2a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ io.akka akka-javasdk-parent - 3.5.3 + 3.5.4 io.akka.ai diff --git a/src/main/java/demo/multiagent/application/ActivityAgent.java b/src/main/java/demo/multiagent/application/ActivityAgent.java index 8ac11c1..596a1c3 100644 --- a/src/main/java/demo/multiagent/application/ActivityAgent.java +++ b/src/main/java/demo/multiagent/application/ActivityAgent.java @@ -2,12 +2,12 @@ import akka.javasdk.agent.Agent; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.client.ComponentClient; import demo.multiagent.domain.AgentRequest; import java.util.stream.Collectors; -@ComponentId("activity-agent") +@Component(id = "activity-agent") @AgentDescription( name = "Activity Agent", description = """ diff --git a/src/main/java/demo/multiagent/application/ActivityView.java b/src/main/java/demo/multiagent/application/ActivityView.java index d97143b..323fd3f 100644 --- a/src/main/java/demo/multiagent/application/ActivityView.java +++ b/src/main/java/demo/multiagent/application/ActivityView.java @@ -1,6 +1,6 @@ package demo.multiagent.application; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.annotations.Consume; import akka.javasdk.annotations.DeleteHandler; import akka.javasdk.annotations.Query; @@ -8,7 +8,7 @@ import akka.javasdk.view.View; import java.util.List; -@ComponentId("activity-view") +@Component(id = "activity-view") public class ActivityView extends View { public record ActivityEntries(List entries) {} diff --git a/src/main/java/demo/multiagent/application/AgentTeamEvaluatorConsumer.java b/src/main/java/demo/multiagent/application/AgentTeamEvaluatorConsumer.java new file mode 100644 index 0000000..f6f7e70 --- /dev/null +++ b/src/main/java/demo/multiagent/application/AgentTeamEvaluatorConsumer.java @@ -0,0 +1,75 @@ +package demo.multiagent.application; + +import akka.javasdk.agent.evaluator.SummarizationEvaluator; +import akka.javasdk.agent.evaluator.ToxicityEvaluator; +import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Consume; +import akka.javasdk.client.ComponentClient; +import akka.javasdk.consumer.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ComponentId("agent-team-eval-consumer") +@Consume.FromWorkflow(AgentTeamWorkflow.class) +public class AgentTeamEvaluatorConsumer extends Consumer { // <1> + + private static final Logger logger = LoggerFactory.getLogger( + AgentTeamEvaluatorConsumer.class + ); + + private final ComponentClient componentClient; + + public AgentTeamEvaluatorConsumer(ComponentClient componentClient) { + this.componentClient = componentClient; + } + + public Effect onStateChanged(AgentTeamWorkflow.State state) { // <2> + if (state.status() == AgentTeamWorkflow.Status.COMPLETED) { + evalToxicity(state); + evalSummarization(state); + } + return effects().done(); + } + + private void evalToxicity(AgentTeamWorkflow.State state) { + var result = componentClient + .forAgent() + .inSession(sessionId()) + .method(ToxicityEvaluator::evaluate) // <3> + .invoke(state.finalAnswer()); + if (result.passed()) { + logger.debug("Eval toxicity passed, session [{}]", sessionId()); // <4> + } else { + logger.warn( + "Eval toxicity failed, session [{}], explanation: {}", + sessionId(), + result.explanation() + ); + } + } + + private void evalSummarization(AgentTeamWorkflow.State state) { + var agentsAnswers = String.join("\n\n", state.agentResponses().values()); + + var result = componentClient + .forAgent() + .inSession(sessionId()) + .method(SummarizationEvaluator::evaluate) + .invoke( + new SummarizationEvaluator.EvaluationRequest(agentsAnswers, state.finalAnswer()) + ); + if (result.passed()) { + logger.debug("Eval summarization passed, session [{}]", sessionId()); + } else { + logger.warn( + "Eval summarization failed, session [{}], explanation: {}", + sessionId(), + result.explanation() + ); + } + } + + private String sessionId() { + return messageContext().eventSubject().get(); + } +} diff --git a/src/main/java/demo/multiagent/application/AgentTeamWorkflow.java b/src/main/java/demo/multiagent/application/AgentTeamWorkflow.java index 99ecb24..4d7a83f 100644 --- a/src/main/java/demo/multiagent/application/AgentTeamWorkflow.java +++ b/src/main/java/demo/multiagent/application/AgentTeamWorkflow.java @@ -6,7 +6,7 @@ import static java.time.Duration.ofSeconds; import akka.Done; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.annotations.StepName; import akka.javasdk.client.ComponentClient; import akka.javasdk.client.DynamicMethodRef; @@ -20,7 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@ComponentId("agent-team") +@Component(id = "agent-team") public class AgentTeamWorkflow extends Workflow { // <1> public record Request(String userId, String message) {} diff --git a/src/main/java/demo/multiagent/application/EvaluatorAgent.java b/src/main/java/demo/multiagent/application/EvaluatorAgent.java index 3ecc31a..7e7e091 100644 --- a/src/main/java/demo/multiagent/application/EvaluatorAgent.java +++ b/src/main/java/demo/multiagent/application/EvaluatorAgent.java @@ -1,14 +1,15 @@ package demo.multiagent.application; import akka.javasdk.agent.Agent; +import akka.javasdk.agent.EvaluationResult; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.client.ComponentClient; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; -@ComponentId("evaluator-agent") +@Component(id = "evaluator-agent") @AgentDescription( name = "Evaluator Agent", description = """ @@ -25,13 +26,17 @@ public record EvaluationRequest( String finalAnswer ) {} - public record EvaluationResult(String evaluation, String feedback) { - public boolean ok() { - return switch (evaluation.toLowerCase(Locale.ROOT)) { + public record Result(String explanation, String label) implements EvaluationResult { + public boolean passed() { + if (label == null) throw new IllegalArgumentException( + "Model response must include label field" + ); + + return switch (label.toLowerCase(Locale.ROOT)) { case "correct" -> true; case "incorrect" -> false; default -> throw new IllegalArgumentException( - "Unknown evaluation result [" + evaluation + "]" + "Unknown evaluation result [" + label + "]" ); }; } @@ -61,8 +66,8 @@ public boolean ok() { respecting user preferences is the most important criteria Your response must be a single JSON object with the following fields: - - "evaluation": A string, either "Correct" or "Incorrect". - - "feedback": Specific feedback on what works well or deviations from preferences. + - "explanation": Specific feedback on what works well or deviations from preferences. + - "label": A string, either "Correct" or "Incorrect". """.stripIndent(); private final ComponentClient componentClient; @@ -71,7 +76,7 @@ public EvaluatorAgent(ComponentClient componentClient) { this.componentClient = componentClient; } - public Effect evaluate(EvaluationRequest request) { + public Effect evaluate(EvaluationRequest request) { var allPreferences = componentClient .forEventSourcedEntity(request.userId()) .method(PreferencesEntity::getPreferences) @@ -86,7 +91,7 @@ public Effect evaluate(EvaluationRequest request) { return effects() .systemMessage(SYSTEM_MESSAGE) .userMessage(evaluationPrompt) - .responseConformsTo(EvaluationResult.class) // <3> + .responseConformsTo(Result.class) // <3> .thenReply(); } diff --git a/src/main/java/demo/multiagent/application/PlannerAgent.java b/src/main/java/demo/multiagent/application/PlannerAgent.java index 6bbd934..cc93e3d 100644 --- a/src/main/java/demo/multiagent/application/PlannerAgent.java +++ b/src/main/java/demo/multiagent/application/PlannerAgent.java @@ -4,13 +4,13 @@ import akka.javasdk.agent.Agent; import akka.javasdk.agent.AgentRegistry; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import demo.multiagent.domain.AgentSelection; import demo.multiagent.domain.Plan; import demo.multiagent.domain.PlanStep; import java.util.List; -@ComponentId("planner-agent") +@Component(id = "planner-agent") @AgentDescription( name = "Planner", description = """ diff --git a/src/main/java/demo/multiagent/application/PreferencesConsumer.java b/src/main/java/demo/multiagent/application/PreferencesConsumer.java index f9f52a5..a22824a 100644 --- a/src/main/java/demo/multiagent/application/PreferencesConsumer.java +++ b/src/main/java/demo/multiagent/application/PreferencesConsumer.java @@ -1,6 +1,6 @@ package demo.multiagent.application; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.annotations.Consume; import akka.javasdk.client.ComponentClient; import akka.javasdk.consumer.Consumer; @@ -8,7 +8,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@ComponentId("preferences-consumer") +@Component(id = "preferences-consumer") @Consume.FromEventSourcedEntity(PreferencesEntity.class) // <1> public class PreferencesConsumer extends Consumer { // <2> @@ -46,13 +46,13 @@ public Effect onPreferenceAdded(PreferencesEvent.PreferenceAdded event) { .invoke(evaluationRequest); // <4> logger.info( - "Evaluation completed for session {}: evaluation={}, feedback='{}'", + "Evaluation completed for session {}: label={}, explanation='{}'", activity.sessionId(), - evaluationResult.evaluation(), - evaluationResult.feedback() + evaluationResult.label(), + evaluationResult.explanation() ); - if (!evaluationResult.ok()) { + if (!evaluationResult.passed()) { // run the workflow again to generate a better answer componentClient diff --git a/src/main/java/demo/multiagent/application/PreferencesEntity.java b/src/main/java/demo/multiagent/application/PreferencesEntity.java index c63c70f..579f8b5 100644 --- a/src/main/java/demo/multiagent/application/PreferencesEntity.java +++ b/src/main/java/demo/multiagent/application/PreferencesEntity.java @@ -1,13 +1,13 @@ package demo.multiagent.application; import akka.Done; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.eventsourcedentity.EventSourcedEntity; import demo.multiagent.domain.Preferences; import demo.multiagent.domain.PreferencesEvent; import java.util.List; -@ComponentId("preferences") +@Component(id = "preferences") public class PreferencesEntity extends EventSourcedEntity { public record AddPreference(String preference) {} diff --git a/src/main/java/demo/multiagent/application/SelectorAgent.java b/src/main/java/demo/multiagent/application/SelectorAgent.java index 03b7aa1..ffeeb39 100644 --- a/src/main/java/demo/multiagent/application/SelectorAgent.java +++ b/src/main/java/demo/multiagent/application/SelectorAgent.java @@ -4,10 +4,10 @@ import akka.javasdk.agent.Agent; import akka.javasdk.agent.AgentRegistry; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import demo.multiagent.domain.AgentSelection; -@ComponentId("selector-agent") +@Component(id = "selector-agent") @AgentDescription( name = "Selector Agent", description = """ diff --git a/src/main/java/demo/multiagent/application/SummarizerAgent.java b/src/main/java/demo/multiagent/application/SummarizerAgent.java index aa8de69..a6cbe9e 100644 --- a/src/main/java/demo/multiagent/application/SummarizerAgent.java +++ b/src/main/java/demo/multiagent/application/SummarizerAgent.java @@ -2,11 +2,11 @@ import akka.javasdk.agent.Agent; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import java.util.Collection; import java.util.stream.Collectors; -@ComponentId("summarizer-agent") +@Component(id = "summarizer-agent") @AgentDescription( name = "Summarizer", description = "An agent that creates a summary from responses provided by other agents" @@ -34,11 +34,11 @@ public Effect summarize(Request request) { var allResponses = request.agentsResponses .stream() .filter(response -> !response.startsWith("ERROR")) - .collect(Collectors.joining(" ")); + .collect(Collectors.joining("\n\n")); return effects() .systemMessage(buildSystemMessage(request.originalQuery)) - .userMessage("Summarize the following message: '" + allResponses + "'") + .userMessage("Summarize the following: \n" + allResponses) .thenReply(); } } diff --git a/src/main/java/demo/multiagent/application/WeatherAgent.java b/src/main/java/demo/multiagent/application/WeatherAgent.java index f4ada06..25b97a3 100644 --- a/src/main/java/demo/multiagent/application/WeatherAgent.java +++ b/src/main/java/demo/multiagent/application/WeatherAgent.java @@ -2,13 +2,13 @@ import akka.javasdk.agent.Agent; import akka.javasdk.annotations.AgentDescription; -import akka.javasdk.annotations.ComponentId; +import akka.javasdk.annotations.Component; import akka.javasdk.annotations.FunctionTool; import demo.multiagent.domain.AgentRequest; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -@ComponentId("weather-agent") +@Component(id = "weather-agent") @AgentDescription( name = "Weather Agent", description = """ diff --git a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java index 777d9e3..98ed1ca 100644 --- a/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java +++ b/src/test/java/demo/multiagent/api/ActivityEndpointIntegrationTest.java @@ -4,6 +4,8 @@ import akka.http.javadsl.model.StatusCodes; import akka.javasdk.JsonSupport; +import akka.javasdk.agent.evaluator.SummarizationEvaluator; +import akka.javasdk.agent.evaluator.ToxicityEvaluator; import akka.javasdk.testkit.TestKit; import akka.javasdk.testkit.TestKitSupport; import akka.javasdk.testkit.TestModelProvider; @@ -24,6 +26,8 @@ public class ActivityEndpointIntegrationTest extends TestKitSupport { private final TestModelProvider weatherModel = new TestModelProvider(); private final TestModelProvider summaryModel = new TestModelProvider(); private final TestModelProvider evaluatorModel = new TestModelProvider(); + private final TestModelProvider toxicityEvalModel = new TestModelProvider(); + private final TestModelProvider summarizationEvalModel = new TestModelProvider(); @Override protected TestKit.Settings testKitSettings() { @@ -35,7 +39,9 @@ protected TestKit.Settings testKitSettings() { .withModelProvider(ActivityAgent.class, activitiesModel) .withModelProvider(WeatherAgent.class, weatherModel) .withModelProvider(SummarizerAgent.class, summaryModel) - .withModelProvider(EvaluatorAgent.class, evaluatorModel); + .withModelProvider(EvaluatorAgent.class, evaluatorModel) + .withModelProvider(ToxicityEvaluator.class, toxicityEvalModel) + .withModelProvider(SummarizationEvaluator.class, summarizationEvalModel); } @Test @@ -168,11 +174,27 @@ private void setupInitialModelResponses() { evaluatorModel.fixedResponse( """ { - "evaluation": "Correct", - "feedback": "The suggestion is appropriate for the user." + "label": "Correct", + "explanation": "The suggestion is appropriate for the user." } """ ); + + toxicityEvalModel.fixedResponse( + """ + { + "label" : "non-toxic" + } + """.stripIndent() + ); + + summarizationEvalModel.fixedResponse( + """ + { + "label" : "good" + } + """.stripIndent() + ); } private void setupUpdatedModelResponsesForPreference() { @@ -182,7 +204,7 @@ private void setupUpdatedModelResponsesForPreference() { .reply( """ { - "evaluation": "Incorrect", + "label": "Incorrect", "feedback": "The previous suggestion conflicts with user preferences for indoor activities. Outdoor bike tours are not suitable." } """ diff --git a/src/test/java/demo/multiagent/application/ActivityViewTest.java b/src/test/java/demo/multiagent/application/ActivityViewTest.java index bf58362..b6b725f 100644 --- a/src/test/java/demo/multiagent/application/ActivityViewTest.java +++ b/src/test/java/demo/multiagent/application/ActivityViewTest.java @@ -7,6 +7,7 @@ import com.typesafe.config.ConfigFactory; import demo.multiagent.application.ActivityView.ActivityEntries; import demo.multiagent.application.ActivityView.ActivityEntry; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; @@ -19,7 +20,9 @@ protected TestKit.Settings testKitSettings() { // We don't need a real one though. return TestKit.Settings.DEFAULT.withAdditionalConfig( ConfigFactory.parseString("akka.javasdk.agent.openai.api-key=fake-key") - ).withWorkflowIncomingMessages(AgentTeamWorkflow.class); + ) + .withWorkflowIncomingMessages(AgentTeamWorkflow.class) + .withDisabledComponents(Set.of(AgentTeamEvaluatorConsumer.class)); } @Test diff --git a/src/test/java/demo/multiagent/application/AgentTeamWorkflowTest.java b/src/test/java/demo/multiagent/application/AgentTeamWorkflowTest.java index 373c742..9431a99 100644 --- a/src/test/java/demo/multiagent/application/AgentTeamWorkflowTest.java +++ b/src/test/java/demo/multiagent/application/AgentTeamWorkflowTest.java @@ -4,6 +4,8 @@ import static org.assertj.core.api.Assertions.assertThat; import akka.javasdk.JsonSupport; +import akka.javasdk.agent.evaluator.SummarizationEvaluator; +import akka.javasdk.agent.evaluator.ToxicityEvaluator; import akka.javasdk.testkit.TestKit; import akka.javasdk.testkit.TestKitSupport; import akka.javasdk.testkit.TestModelProvider; @@ -22,6 +24,8 @@ public class AgentTeamWorkflowTest extends TestKitSupport { // <1> private final TestModelProvider activitiesModel = new TestModelProvider(); private final TestModelProvider weatherModel = new TestModelProvider(); private final TestModelProvider summaryModel = new TestModelProvider(); + private final TestModelProvider toxicityEvalModel = new TestModelProvider(); + private final TestModelProvider summarizationEvalModel = new TestModelProvider(); @Override protected TestKit.Settings testKitSettings() { @@ -32,7 +36,9 @@ protected TestKit.Settings testKitSettings() { .withModelProvider(PlannerAgent.class, plannerModel) .withModelProvider(ActivityAgent.class, activitiesModel) .withModelProvider(WeatherAgent.class, weatherModel) - .withModelProvider(SummarizerAgent.class, summaryModel); + .withModelProvider(SummarizerAgent.class, summaryModel) + .withModelProvider(ToxicityEvaluator.class, toxicityEvalModel) + .withModelProvider(SummarizationEvaluator.class, summarizationEvalModel); } @Test @@ -68,6 +74,22 @@ public void test() { "visiting the Vasa Museum, exploring Gamla Stan (Old Town)" ); + toxicityEvalModel.fixedResponse( + """ + { + "label" : "non-toxic" + } + """.stripIndent() + ); + + summarizationEvalModel.fixedResponse( + """ + { + "label" : "good" + } + """.stripIndent() + ); + var query = "I am in Stockholm. What should I do? Beware of the weather"; var sessionId = UUID.randomUUID().toString();