feat: updated evaluator selector ui

api/oss/tests/pytest/acceptance/loadables/test_loadable_strategies.py

@@ -881,10 +881,15 @@ def test_edge_a1_query_windowing_limit_1(self, authed_api, mock_data):
                 "include_trace_ids": True,
                 "windowing": {"limit": 1},
             },
-            condition_fn=lambda r: len(
-                r.json().get("query_revision", {}).get("data", {}).get("trace_ids", [])
-            )
-            == 1,
+            condition_fn=lambda r: (
+                len(
+                    r.json()
+                    .get("query_revision", {})
+                    .get("data", {})
+                    .get("trace_ids", [])
+                )
+                == 1
+            ),
         )
         # ---------------------------------------------------------------------
 
@@ -919,10 +924,12 @@ def test_edge_a2_query_request_windowing_overrides_stored(
                 "include_traces": True,
                 "windowing": {"limit": 1},
             },
-            condition_fn=lambda r: len(
-                r.json().get("query_revision", {}).get("data", {}).get("traces", [])
-            )
-            == 1,
+            condition_fn=lambda r: (
+                len(
+                    r.json().get("query_revision", {}).get("data", {}).get("traces", [])
+                )
+                == 1
+            ),
         )
         # ---------------------------------------------------------------------
 

web/oss/src/components/Evaluators/components/EvaluatorTemplateDropdown.tsx

@@ -26,6 +26,10 @@ interface EvaluatorTemplateDropdownProps {
     trigger?: React.ReactNode
     /** Additional class name for the trigger wrapper */
     className?: string
+    /** Controlled open state (optional — when provided, component is controlled) */
+    open?: boolean
+    /** Callback when open state changes (required when using controlled `open`) */
+    onOpenChange?: (open: boolean) => void
 }
 
 /**
@@ -36,9 +40,25 @@ const EvaluatorTemplateDropdown = ({
     onSelect,
     trigger,
     className,
+    open: controlledOpen,
+    onOpenChange: controlledOnOpenChange,
 }: EvaluatorTemplateDropdownProps) => {
     const [activeTab, setActiveTab] = useState<string>(DEFAULT_TAB_KEY)
-    const [open, setOpen] = useState(false)
+    const [internalOpen, setInternalOpen] = useState(false)
+
+    // Support both controlled and uncontrolled modes
+    const isControlled = controlledOpen !== undefined
+    const open = isControlled ? controlledOpen : internalOpen
+    const setOpen = useCallback(
+        (next: boolean) => {
+            if (isControlled) {
+                controlledOnOpenChange?.(next)
+            } else {
+                setInternalOpen(next)
+            }
+        },
+        [isControlled, controlledOnOpenChange],
+    )
     const nonArchivedEvaluators = useAtomValue(evaluatorTemplatesDataAtom)
     const {isPending: isLoadingEvaluators} = useAtomValue(evaluatorTemplatesQueryAtom)
 

web/oss/src/components/Playground/Components/PlaygroundHeader/index.tsx

@@ -1,8 +1,12 @@
-import React, {useCallback, useMemo} from "react"
+import React, {useCallback, useMemo, useState} from "react"
 
 import type {PlaygroundNode} from "@agenta/entities/runnable"
-import {getEvaluatorColor, workflowMolecule} from "@agenta/entities/workflow"
-import type {EvaluatorColor} from "@agenta/entities/workflow"
+import {
+    getEvaluatorColor,
+    workflowMolecule,
+    createEvaluatorFromTemplate,
+} from "@agenta/entities/workflow"
+import type {EvaluatorColor, EvaluatorCatalogTemplate} from "@agenta/entities/workflow"
 import {EntityPicker} from "@agenta/entity-ui"
 import {type WorkflowRevisionSelectionResult} from "@agenta/entity-ui/selection"
 import {useEnrichedEvaluatorOnlyAdapter as useEvaluatorOnlyAdapter} from "@agenta/entity-ui/selection"
@@ -12,14 +16,16 @@ import {bgColors, textColors} from "@agenta/ui"
 import {VersionBadge} from "@agenta/ui/components/presentational"
 import {CloseOutlined, DownOutlined, MoreOutlined} from "@ant-design/icons"
 import {Gavel, PencilSimple, Plus} from "@phosphor-icons/react"
-import {Button, Divider, Dropdown, Space, Tag, Tooltip, Typography} from "antd"
+import {Button, Divider, Dropdown, Space, Tag, Tooltip, Typography, message} from "antd"
 import clsx from "clsx"
 import {useAtomValue, useSetAtom} from "jotai"
 import dynamic from "next/dynamic"
 
+import EvaluatorTemplateDropdown from "@/oss/components/Evaluators/components/EvaluatorTemplateDropdown"
 import useCustomWorkflowConfig from "@/oss/components/pages/app-management/modals/CustomWorkflowModal/hooks/useCustomWorkflowConfig"
 import {currentAppAtom} from "@/oss/state/app"
 import {routerAppIdAtom} from "@/oss/state/app/selectors/app"
+import {openEvaluatorDrawerAtom} from "@/oss/state/evaluator/evaluatorDrawerStore"
 import {writePlaygroundSelectionToQuery} from "@/oss/state/url/playground"
 import {workspaceMemberByIdFamily} from "@/oss/state/workspace/atoms/selectors"
 
@@ -168,24 +174,6 @@ const PlaygroundHeader: React.FC<PlaygroundHeaderProps> = ({className, ...divPro
         [connectedEvaluatorNodes],
     )
 
-    const handleEvaluatorSelect = useCallback(
-        (selection: WorkflowRevisionSelectionResult) => {
-            const rootNode = nodes.find((n) => n.depth === 0)
-            if (!rootNode) return
-
-            connectDownstreamNode({
-                sourceNodeId: rootNode.id,
-                entity: {
-                    type: "workflow",
-                    id: selection.id,
-                    label: selection.label,
-                    metadata: selection.metadata,
-                },
-            })
-        },
-        [nodes, connectDownstreamNode],
-    )
-
     const handleDisconnectAll = useCallback(() => {
         disconnectDownstreamNode("workflow")
     }, [disconnectDownstreamNode])
@@ -200,6 +188,70 @@ const PlaygroundHeader: React.FC<PlaygroundHeaderProps> = ({className, ...divPro
     // Evaluator-only adapter with colored type tags, human filtering, and custom revision labels
     const evaluatorWorkflowAdapter = useEvaluatorOnlyAdapter(renderWorkflowRevisionLabel)
 
+    // Controlled state for EvaluatorTemplateDropdown
+    const [templateDropdownOpen, setTemplateDropdownOpen] = useState(false)
+
+    // Open the evaluator template dropdown (called from EntityPicker's onCreateNew)
+    const handleOpenTemplateDropdown = useCallback(() => {
+        // Small delay to let the EntityPicker popover close first
+        setTimeout(() => {
+            setTemplateDropdownOpen(true)
+        }, 100)
+    }, [])
+
+    const openEvaluatorDrawer = useSetAtom(openEvaluatorDrawerAtom)
+
+    // Handle template selection from EvaluatorTemplateDropdown
+    const handleTemplateSelect = useCallback(
+        async (template: EvaluatorCatalogTemplate) => {
+            const templateKey = template.key
+            if (!templateKey) {
+                message.error("Unable to open evaluator template")
+                return
+            }
+
+            const localId = await createEvaluatorFromTemplate(templateKey)
+            if (!localId) {
+                message.error("Unable to create evaluator from template")
+                return
+            }
+
+            openEvaluatorDrawer({
+                entityId: localId,
+                mode: "create",
+            })
+        },
+        [openEvaluatorDrawer],
+    )
+
+    // Multi-select: toggle evaluator connection/disconnection
+    const handleEvaluatorToggle = useCallback(
+        (selection: WorkflowRevisionSelectionResult) => {
+            const rootNode = nodes.find((n) => n.depth === 0)
+            if (!rootNode) return
+
+            // Check if this revision is already connected
+            const existingNode = connectedEvaluatorNodes.find((n) => n.entityId === selection.id)
+
+            if (existingNode) {
+                // Disconnect
+                disconnectSingleDownstreamNode(existingNode.id)
+            } else {
+                // Connect
+                connectDownstreamNode({
+                    sourceNodeId: rootNode.id,
+                    entity: {
+                        type: "workflow",
+                        id: selection.id,
+                        label: selection.label,
+                        metadata: selection.metadata,
+                    },
+                })
+            }
+        },
+        [nodes, connectedEvaluatorNodes, connectDownstreamNode, disconnectSingleDownstreamNode],
+    )
+
     // Simplified refresh function - atoms will handle the data updates automatically
     const handleUpdate = useCallback(async () => {
         // For now, use a simple page reload since atoms auto-refresh on mount
@@ -298,15 +350,21 @@ const PlaygroundHeader: React.FC<PlaygroundHeaderProps> = ({className, ...divPro
                             <EntityPicker<WorkflowRevisionSelectionResult>
                                 variant="popover-cascader"
                                 adapter={evaluatorWorkflowAdapter}
-                                onSelect={handleEvaluatorSelect}
+                                onSelect={handleEvaluatorToggle}
                                 size="small"
                                 placeholder="Evaluator"
                                 icon={<Gavel size={14} />}
                                 disabled={!hasRootNode}
-                                disabledChildIds={connectedRevisionIds}
+                                multiSelect
+                                selectedChildIds={connectedRevisionIds}
+                                selectionSummary
+                                childItemLabelMode="simple"
+                                panelWidth={280}
+                                onCreateNew={handleOpenTemplateDropdown}
+                                createNewLabel="New evaluator"
                                 popupFooter={
                                     connectedEvaluatorNodes.length > 0 ? (
-                                        <div className="border-t border-solid border-[rgba(5,23,41,0.06)] p-2">
+                                        <div className="border-0 border-t border-solid border-[rgba(5,23,41,0.06)] p-2">
                                             <Button
                                                 size="small"
                                                 danger
@@ -321,6 +379,12 @@ const PlaygroundHeader: React.FC<PlaygroundHeaderProps> = ({className, ...divPro
                             />
                         </span>
                     </Tooltip>
+                    <EvaluatorTemplateDropdown
+                        onSelect={handleTemplateSelect}
+                        open={templateDropdownOpen}
+                        onOpenChange={setTemplateDropdownOpen}
+                        trigger={<span className="w-0 h-0 inline-block" />}
+                    />
                     <TestsetDropdown />
                     {isProjectLevelPlayground ? (
                         <Tooltip title="Compare mode is unavailable in project-level playground">

web/oss/src/pages/w/[workspace_id]/p/[project_id]/annotations/index.tsx

@@ -3,15 +3,18 @@ import {useMemo} from "react"
 import {AnnotationUIProvider, type AnnotationUINavigation} from "@agenta/annotation-ui/context"
 import AnnotationQueuesView from "@agenta/annotation-ui/queue-list"
 import {PageLayout} from "@agenta/ui"
+import {useSetAtom} from "jotai"
 import {useRouter} from "next/router"
 
+import {openHumanEvaluatorDrawerAtom} from "@/oss/components/Evaluators/Drawers/HumanEvaluatorDrawer/store"
 import {useProjectPermissions} from "@/oss/hooks/useProjectPermissions"
 import useURL from "@/oss/hooks/useURL"
 
 const AnnotationQueuesPage = () => {
     const router = useRouter()
     const {projectURL} = useURL()
     const {canExportData} = useProjectPermissions()
+    const openHumanEvaluatorDrawer = useSetAtom(openHumanEvaluatorDrawerAtom)
 
     const navigation = useMemo<AnnotationUINavigation>(
         () => ({
@@ -30,7 +33,11 @@ const AnnotationQueuesPage = () => {
                 title={<span className="inline-flex items-center gap-2">Queues</span>}
                 className="h-full min-h-0"
             >
-                <AnnotationQueuesView canExportData={canExportData} />
+                <AnnotationQueuesView
+                    canExportData={canExportData}
+                    feedbackOnCreate={() => openHumanEvaluatorDrawer({mode: "create"})}
+                    feedbackCreateLabel="Create evaluator"
+                />
             </PageLayout>
         </AnnotationUIProvider>
     )

web/packages/agenta-annotation-ui/src/components/AnnotationQueuesView/index.tsx

@@ -222,9 +222,15 @@ const QueuesHeaderFilters = () => {
 
 export interface AnnotationQueuesViewProps {
     canExportData?: boolean
+    feedbackOnCreate?: () => void
+    feedbackCreateLabel?: string
 }
 
-const AnnotationQueuesView = ({canExportData = true}: AnnotationQueuesViewProps) => {
+const AnnotationQueuesView = ({
+    canExportData = true,
+    feedbackOnCreate,
+    feedbackCreateLabel,
+}: AnnotationQueuesViewProps) => {
     const navigation = useAnnotationNavigation()
     const searchTerm = useAtomValue(simpleQueueSearchTermAtom)
     const kindFilter = useAtomValue(simpleQueueKindFilterAtom)
@@ -514,7 +520,10 @@ const AnnotationQueuesView = ({canExportData = true}: AnnotationQueuesViewProps)
                 autoHeight
                 store={getDefaultStore()}
             />
-            <CreateQueueDrawer />
+            <CreateQueueDrawer
+                feedbackOnCreate={feedbackOnCreate}
+                feedbackCreateLabel={feedbackCreateLabel}
+            />
         </div>
     )
 }

web/packages/agenta-annotation-ui/src/components/CreateQueueDrawer/EntityEvaluatorSelector.tsx

@@ -25,9 +25,12 @@ export interface EntityEvaluatorSelectorProps {
     onSelect: (selection: WorkflowRevisionSelectionResult) => void
     instanceId: string
     buttonLabel?: string
+    onCreate?: () => void
+    createLabel?: string
     disabledRevisionIds?: Set<string>
     disabledRevisionTooltip?: string
     panelMinWidth?: number
+    panelWidth?: number
     disabled?: boolean
     selectedEvaluatorId?: string | null
     selectedRevisionId?: string | null
@@ -124,9 +127,12 @@ export function EntityEvaluatorSelector({
     onSelect,
     instanceId,
     buttonLabel = "Add evaluator",
+    onCreate,
+    createLabel = "Create evaluator",
     disabledRevisionIds,
     disabledRevisionTooltip = "Already added",
     panelMinWidth = 280,
+    panelWidth,
     disabled = false,
     selectedEvaluatorId,
     selectedRevisionId,
@@ -172,13 +178,16 @@ export function EntityEvaluatorSelector({
                 icon={<Plus size={14} />}
                 showDropdownIcon={false}
                 panelMinWidth={panelMinWidth}
+                panelWidth={panelWidth}
                 disabled={disabled}
                 selectedParentId={selectedEvaluatorId}
                 selectedChildId={selectedRevisionId}
                 disabledChildIds={disabledRevisionIds}
                 disabledChildTooltip={disabledRevisionTooltip}
                 openChildOnHover={openVersionOnHover}
                 size="middle"
+                onCreateNew={onCreate}
+                createNewLabel={createLabel}
             />
         </div>
     )

web/packages/agenta-annotation-ui/src/components/CreateQueueDrawer/index.tsx

@@ -50,6 +50,8 @@ interface CreateQueueDrawerContentProps {
     selection: {itemType: "traces" | "testcases"; itemIds: string[]} | null
     onClearSelection: () => void
     onItemsAdded?: () => void
+    feedbackOnCreate?: () => void
+    feedbackCreateLabel?: string
 }
 
 const INITIAL_FORM_VALUES: Pick<FormValues, "kind"> = {
@@ -94,6 +96,8 @@ function CreateQueueDrawerContent({
     selection,
     onClearSelection,
     onItemsAdded,
+    feedbackOnCreate,
+    feedbackCreateLabel,
 }: CreateQueueDrawerContentProps) {
     const projectId = useAtomValue(projectIdAtom)
     const createQueue = useSetAtom(createSimpleQueueAtom)
@@ -346,6 +350,8 @@ function CreateQueueDrawerContent({
                                         <EntityEvaluatorSelector
                                             onSelect={handleEvaluatorSelect}
                                             instanceId="queue-evaluator"
+                                            onCreate={feedbackOnCreate}
+                                            createLabel={feedbackCreateLabel}
                                             disabled={isSubmitting}
                                             disabledRevisionIds={selectedRevisionIds}
                                             selectedEvaluatorId={
@@ -378,9 +384,15 @@ function CreateQueueDrawerContent({
 
 interface CreateQueueDrawerProps {
     onItemsAdded?: () => void
+    feedbackOnCreate?: () => void
+    feedbackCreateLabel?: string
 }
 
-const CreateQueueDrawer = ({onItemsAdded}: CreateQueueDrawerProps) => {
+const CreateQueueDrawer = ({
+    onItemsAdded,
+    feedbackOnCreate,
+    feedbackCreateLabel,
+}: CreateQueueDrawerProps) => {
     const [open, setOpen] = useAtom(createQueueDrawerOpenAtom)
     const defaultKind = useAtomValue(createQueueDrawerDefaultKindAtom)
     const [selection, setSelection] = useAtom(createQueueDrawerSelectionAtom)
@@ -455,6 +467,8 @@ const CreateQueueDrawer = ({onItemsAdded}: CreateQueueDrawerProps) => {
                     selection={selection}
                     onClearSelection={handleClearSelection}
                     onItemsAdded={onItemsAdded}
+                    feedbackOnCreate={feedbackOnCreate}
+                    feedbackCreateLabel={feedbackCreateLabel}
                 />
             ) : null}
         </Drawer>