From 7a005f1247dd4a37634d7535df05596c26fafda3 Mon Sep 17 00:00:00 2001 From: panding Date: Fri, 26 Jun 2026 01:21:17 +0800 Subject: [PATCH] Expose missing GUI scoring tools --- .../task.yaml | 43 +++++++ .../task.yaml | 58 ++++++++++ .../task.yaml | 99 ++++++++++++++++ .../task.yaml | 27 +++++ .../task.yaml | 108 ++++++++++++++++++ .../task.yaml | 23 ++++ .../task.yaml | 34 ++++++ .../task.yaml | 18 ++- .../task.yaml | 18 ++- .../task.yaml | 16 ++- tests/test_gui_missing_tool_exposure.py | 85 ++++++++++++++ 11 files changed, 521 insertions(+), 8 deletions(-) create mode 100644 tests/test_gui_missing_tool_exposure.py diff --git a/benchmark/gui/TGUI01_myexpenses_overbudget_finance_email/task.yaml b/benchmark/gui/TGUI01_myexpenses_overbudget_finance_email/task.yaml index 1cffab6..af3e117 100644 --- a/benchmark/gui/TGUI01_myexpenses_overbudget_finance_email/task.yaml +++ b/benchmark/gui/TGUI01_myexpenses_overbudget_finance_email/task.yaml @@ -36,6 +36,40 @@ services: GMAIL_FIXTURES: fixtures/gmail/inbox.json tools: +# ---- my_expenses_gui ---- +- name: my_expenses_list_accounts + description: 列出 My Expenses 中的账户，用于确认账户范围和币种 + input_schema: + type: object + properties: + query: + type: string +- name: my_expenses_get_account + description: 获取 My Expenses 单个账户详情 + input_schema: + type: object + properties: + account_id: + type: string + required: [account_id] +- name: my_expenses_list_transactions + description: 列出 My Expenses 交易，支持日期、类别、账户和关键词过滤 + input_schema: + type: object + properties: + start_date: + type: string + end_date: + type: string + category: + type: string + account_id: + type: string + query: + type: string + max_results: + type: integer + # ---- finance CLI ---- - name: finance_list_transactions description: 查询 CLI finance 服务的历史流水，支持按日期过滤，用于建立历史基线 @@ -86,6 +120,15 @@ tools: required: [to, subject, body] tool_endpoints: +- tool_name: my_expenses_list_accounts + url: http://localhost:9120/gui/my_expenses/list_accounts + method: POST +- tool_name: my_expenses_get_account + url: http://localhost:9120/gui/my_expenses/get_account + method: POST +- tool_name: my_expenses_list_transactions + url: http://localhost:9120/gui/my_expenses/list_transactions + method: POST - tool_name: finance_list_transactions url: http://localhost:9104/finance/transactions method: POST diff --git a/benchmark/gui/TGUI02_habit_gap_calendar_clock_replan/task.yaml b/benchmark/gui/TGUI02_habit_gap_calendar_clock_replan/task.yaml index 01afba1..08c9bc5 100644 --- a/benchmark/gui/TGUI02_habit_gap_calendar_clock_replan/task.yaml +++ b/benchmark/gui/TGUI02_habit_gap_calendar_clock_replan/task.yaml @@ -39,6 +39,52 @@ services: CALENDAR_FIXTURES: fixtures/calendar/events.json tools: +# ---- loop_habit_gui ---- +- name: loop_habits_list_habits + description: 列出 Loop Habit Tracker 中的习惯和打卡摘要 + input_schema: + type: object + properties: + query: + type: string +- name: loop_habits_get_habit + description: 获取单个习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: [habit_id] + +# ---- clock_gui ---- +- name: clock_list_alarms + description: 列出手机闹钟，支持按标签或时间关键词搜索 + input_schema: + type: object + properties: + query: + type: string +- name: clock_update_alarm + description: 更新已有闹钟的时间、标签、重复日期或启用状态 + input_schema: + type: object + properties: + alarm_id: + type: string + time: + type: string + label: + type: string + repeat_days: + type: array + items: + type: string + enabled: + type: boolean + vibrate: + type: boolean + required: [alarm_id] + # ---- calendar CLI ---- - name: calendar_list_events description: 从 CLI 日历服务查询指定日期范围内的事件，用于排查时间冲突根因 @@ -72,6 +118,18 @@ tools: required: [title, start_time, end_time] tool_endpoints: +- tool_name: loop_habits_list_habits + url: http://localhost:9120/gui/loop_habit/list_habits + method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST +- tool_name: clock_list_alarms + url: http://localhost:9120/gui/clock/list_alarms + method: POST +- tool_name: clock_update_alarm + url: http://localhost:9120/gui/clock/update_alarm + method: POST - tool_name: calendar_list_events url: http://localhost:9101/calendar/events method: POST diff --git a/benchmark/gui/TGUI03_email_bill_finance_myexpenses_habit/task.yaml b/benchmark/gui/TGUI03_email_bill_finance_myexpenses_habit/task.yaml index 92b7305..ff6cb6e 100644 --- a/benchmark/gui/TGUI03_email_bill_finance_myexpenses_habit/task.yaml +++ b/benchmark/gui/TGUI03_email_bill_finance_myexpenses_habit/task.yaml @@ -40,6 +40,87 @@ services: FINANCE_FIXTURES: fixtures/finance/transactions.json tools: +# ---- gmail_clone_gui ---- +- name: gmail_clone_list_messages + description: 列出 Gmail Clone 邮件，支持标签、未读和关键词过滤 + input_schema: + type: object + properties: + label: + type: string + unread_only: + type: boolean + query: + type: string + max_results: + type: integer +- name: gmail_clone_get_message + description: 获取单封 Gmail Clone 邮件的完整内容 + input_schema: + type: object + properties: + message_id: + type: string + required: [message_id] + +# ---- my_expenses_gui ---- +- name: my_expenses_list_transactions + description: 列出 My Expenses 交易，用于确认是否已补录 + input_schema: + type: object + properties: + start_date: + type: string + end_date: + type: string + category: + type: string + account_id: + type: string + query: + type: string + max_results: + type: integer +- name: my_expenses_add_transaction + description: 在 My Expenses 中新增一笔交易 + input_schema: + type: object + properties: + date: + type: string + account: + type: string + category: + type: string + payee: + type: string + amount: + type: number + description: + type: string + notes: + type: string + required: [date, account, category, amount, description] + +# ---- loop_habit_gui ---- +- name: loop_habits_list_habits + description: 列出 Loop Habit Tracker 中的习惯 + input_schema: + type: object + properties: + query: + type: string +- name: loop_habits_check_habit + description: 为指定习惯在指定日期打卡 + input_schema: + type: object + properties: + habit_id: + type: string + date: + type: string + required: [habit_id] + # ---- finance CLI ---- - name: finance_list_transactions description: 查询 CLI finance 历史流水，验证账单是否已记录 @@ -52,6 +133,24 @@ tools: type: string tool_endpoints: +- tool_name: gmail_clone_list_messages + url: http://localhost:9120/gui/gmail_clone/list_messages + method: POST +- tool_name: gmail_clone_get_message + url: http://localhost:9120/gui/gmail_clone/get_message + method: POST +- tool_name: my_expenses_list_transactions + url: http://localhost:9120/gui/my_expenses/list_transactions + method: POST +- tool_name: my_expenses_add_transaction + url: http://localhost:9120/gui/my_expenses/add_transaction + method: POST +- tool_name: loop_habits_list_habits + url: http://localhost:9120/gui/loop_habit/list_habits + method: POST +- tool_name: loop_habits_check_habit + url: http://localhost:9120/gui/loop_habit/check_habit + method: POST - tool_name: finance_list_transactions url: http://localhost:9104/finance/transactions method: POST diff --git a/benchmark/gui/TGUI04_contacts_new_crm_welcome_email/task.yaml b/benchmark/gui/TGUI04_contacts_new_crm_welcome_email/task.yaml index e0703a4..c64b696 100644 --- a/benchmark/gui/TGUI04_contacts_new_crm_welcome_email/task.yaml +++ b/benchmark/gui/TGUI04_contacts_new_crm_welcome_email/task.yaml @@ -36,6 +36,27 @@ services: GMAIL_FIXTURES: fixtures/gmail/inbox.json tools: +# ---- contacts_gui ---- +- name: contacts_list + description: 列出手机联系人，支持关键词和分组过滤 + input_schema: + type: object + properties: + query: + type: string + group: + type: string + max_results: + type: integer +- name: contacts_get + description: 获取单个手机联系人的完整详情 + input_schema: + type: object + properties: + contact_id: + type: string + required: [contact_id] + # ---- crm CLI ---- - name: crm_list_customers description: 搜索 CRM 客户记录，支持按状态、行业、客户级别过滤 @@ -86,6 +107,12 @@ tools: required: [to, subject, body] tool_endpoints: +- tool_name: contacts_list + url: http://localhost:9120/gui/contacts/list + method: POST +- tool_name: contacts_get + url: http://localhost:9120/gui/contacts/get + method: POST - tool_name: crm_list_customers url: http://localhost:9110/crm/customers method: POST diff --git a/benchmark/gui/TGUI05_contacts_birthday_calendar_gift/task.yaml b/benchmark/gui/TGUI05_contacts_birthday_calendar_gift/task.yaml index a341575..ec28d01 100644 --- a/benchmark/gui/TGUI05_contacts_birthday_calendar_gift/task.yaml +++ b/benchmark/gui/TGUI05_contacts_birthday_calendar_gift/task.yaml @@ -39,6 +39,93 @@ services: CALENDAR_FIXTURES: fixtures/calendar/events.json tools: +# ---- contacts_gui ---- +- name: contacts_list + description: 列出手机联系人，支持关键词和分组过滤 + input_schema: + type: object + properties: + query: + type: string + group: + type: string + max_results: + type: integer +- name: contacts_get + description: 获取单个手机联系人的完整详情 + input_schema: + type: object + properties: + contact_id: + type: string + required: [contact_id] + +# ---- fossify_calendar_gui ---- +- name: fossify_calendar_list_events + description: 查看手机日历事件，支持日期范围和关键词过滤 + input_schema: + type: object + properties: + start_date: + type: string + end_date: + type: string + query: + type: string + max_results: + type: integer +- name: fossify_calendar_get_event + description: 获取单个手机日历事件详情 + input_schema: + type: object + properties: + event_id: + type: string + required: [event_id] +- name: fossify_calendar_create_event + description: 在手机日历中新建提醒事件 + input_schema: + type: object + properties: + title: + type: string + start_time: + type: string + end_time: + type: string + location: + type: string + description: + type: string + reminder_minutes: + type: integer + required: [title, start_time, end_time] + +# ---- testmall_gui ---- +- name: testmall_list_products + description: 浏览或搜索 TestMall 商品，支持关键词、类别和价格范围过滤 + input_schema: + type: object + properties: + query: + type: string + category: + type: string + min_price: + type: number + max_price: + type: number + max_results: + type: integer +- name: testmall_get_product + description: 获取 TestMall 单个商品详情 + input_schema: + type: object + properties: + product_id: + type: string + required: [product_id] + # ---- calendar CLI ---- - name: calendar_list_events description: 查询指定日期范围内的日程，用于确认生日前后的空档时间 @@ -54,6 +141,27 @@ tools: required: [date] tool_endpoints: +- tool_name: contacts_list + url: http://localhost:9120/gui/contacts/list + method: POST +- tool_name: contacts_get + url: http://localhost:9120/gui/contacts/get + method: POST +- tool_name: fossify_calendar_list_events + url: http://localhost:9120/gui/fossify_calendar/list_events + method: POST +- tool_name: fossify_calendar_get_event + url: http://localhost:9120/gui/fossify_calendar/get_event + method: POST +- tool_name: fossify_calendar_create_event + url: http://localhost:9120/gui/fossify_calendar/create_event + method: POST +- tool_name: testmall_list_products + url: http://localhost:9120/gui/testmall/list_products + method: POST +- tool_name: testmall_get_product + url: http://localhost:9120/gui/testmall/get_product + method: POST - tool_name: calendar_list_events url: http://localhost:9101/calendar/events method: POST diff --git a/benchmark/gui/TGUI06_habit_crm_performance_workmail_report/task.yaml b/benchmark/gui/TGUI06_habit_crm_performance_workmail_report/task.yaml index 1c4e56c..10c9dfb 100644 --- a/benchmark/gui/TGUI06_habit_crm_performance_workmail_report/task.yaml +++ b/benchmark/gui/TGUI06_habit_crm_performance_workmail_report/task.yaml @@ -45,6 +45,23 @@ services: WORKMAIL_FIXTURES: fixtures/workmail/inbox.json tools: +# ---- loop_habit_gui ---- +- name: loop_habits_list_habits + description: 列出 Loop Habit Tracker 中的工作习惯和本周完成记录 + input_schema: + type: object + properties: + query: + type: string +- name: loop_habits_get_habit + description: 获取单个工作习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: [habit_id] + # ---- crm CLI ---- - name: crm_list_customers description: 查询 CRM 客户列表，含跟进记录 @@ -105,6 +122,12 @@ tools: required: [to, subject, body] tool_endpoints: +- tool_name: loop_habits_list_habits + url: http://localhost:9120/gui/loop_habit/list_habits + method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST - tool_name: crm_list_customers url: http://localhost:9110/crm/customers method: POST diff --git a/benchmark/gui/TGUI23_loop_habit_finance_expenses_saas/task.yaml b/benchmark/gui/TGUI23_loop_habit_finance_expenses_saas/task.yaml index 04aa72e..a037e35 100644 --- a/benchmark/gui/TGUI23_loop_habit_finance_expenses_saas/task.yaml +++ b/benchmark/gui/TGUI23_loop_habit_finance_expenses_saas/task.yaml @@ -57,6 +57,15 @@ tools: properties: query: type: string +- name: loop_habits_get_habit + description: 获取单个习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: + - habit_id - name: loop_habits_check_habit description: 完成某个习惯的今日打卡 input_schema: @@ -69,6 +78,22 @@ tools: description: YYYY-MM-DD，默认今天 required: - habit_id +- name: my_expenses_list_accounts + description: 列出 My Expenses 中的账户，用于确认余额和币种 + input_schema: + type: object + properties: + query: + type: string +- name: my_expenses_get_account + description: 获取 My Expenses 单个账户详情 + input_schema: + type: object + properties: + account_id: + type: string + required: + - account_id - name: my_expenses_list_transactions description: 列出 My Expenses 中的交易记录，支持日期和分类过滤 input_schema: @@ -354,9 +379,18 @@ tool_endpoints: - tool_name: loop_habits_list_habits url: http://localhost:9120/gui/loop_habit/list_habits method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST - tool_name: loop_habits_check_habit url: http://localhost:9120/gui/loop_habit/check_habit method: POST +- tool_name: my_expenses_list_accounts + url: http://localhost:9120/gui/my_expenses/list_accounts + method: POST +- tool_name: my_expenses_get_account + url: http://localhost:9120/gui/my_expenses/get_account + method: POST - tool_name: my_expenses_list_transactions url: http://localhost:9120/gui/my_expenses/list_transactions method: POST diff --git a/benchmark/gui/TGUI44_stress_test_milestone_habits_expenses_calendar/task.yaml b/benchmark/gui/TGUI44_stress_test_milestone_habits_expenses_calendar/task.yaml index 96657c4..06d1390 100644 --- a/benchmark/gui/TGUI44_stress_test_milestone_habits_expenses_calendar/task.yaml +++ b/benchmark/gui/TGUI44_stress_test_milestone_habits_expenses_calendar/task.yaml @@ -168,6 +168,15 @@ tools: properties: query: type: string +- name: loop_habits_get_habit + description: 获取单个习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: + - habit_id - name: loop_habits_check_habit description: 完成某个习惯的今日打卡 input_schema: @@ -730,6 +739,9 @@ tool_endpoints: - tool_name: loop_habits_list_habits url: http://localhost:9120/gui/loop_habit/list_habits method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST - tool_name: loop_habits_check_habit url: http://localhost:9120/gui/loop_habit/check_habit method: POST @@ -809,7 +821,7 @@ scoring_components: weight: 0.15 check: type: tool_called - tool_name: loop_habit_gui_get_habit + tool_name: loop_habits_get_habit min_calls: 1 - name: Checked Q1 team activities budget availability weight: 0.15 @@ -971,8 +983,8 @@ judge_rubric: '## Evaluation Criteria - **-10 points**: Created calendar event without checking for deployment window conflicts' reference_solution: "## Step-by-Step Reference Solution\n\n### Step 1: Verify the\ - \ 30-day milestone in Loop Habit\n- Call `loop_habit_gui_list_habits` to find the\ - \ 'Pre-deployment stress testing' habit\n- Call `loop_habit_gui_get_habit` for the\ + \ 30-day milestone in Loop Habit\n- Call `loop_habits_list_habits` to find the\ + \ 'Pre-deployment stress testing' habit\n- Call `loop_habits_get_habit` for the\ \ specific habit to retrieve completion dates\n- Verify that completions array shows\ \ 30 consecutive days from 2026-03-03 to 2026-04-02 (with weekends appropriately\ \ handled based on reminder_days)\n- Confirm the milestone has been reached\n\n\ diff --git a/benchmark/gui/TGUI49_cloudinfra_invoice_expenses_habits/task.yaml b/benchmark/gui/TGUI49_cloudinfra_invoice_expenses_habits/task.yaml index ad78d9b..0416809 100644 --- a/benchmark/gui/TGUI49_cloudinfra_invoice_expenses_habits/task.yaml +++ b/benchmark/gui/TGUI49_cloudinfra_invoice_expenses_habits/task.yaml @@ -120,6 +120,15 @@ tools: properties: query: type: string +- name: loop_habits_get_habit + description: 获取单个习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: + - habit_id - name: loop_habits_check_habit description: 完成某个习惯的今日打卡 input_schema: @@ -607,6 +616,9 @@ tool_endpoints: - tool_name: loop_habits_list_habits url: http://localhost:9120/gui/loop_habit/list_habits method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST - tool_name: loop_habits_check_habit url: http://localhost:9120/gui/loop_habit/check_habit method: POST @@ -760,7 +772,7 @@ scoring_components: weight: 0.15 check: type: tool_called - tool_name: loop_habit_gui_check_habit + tool_name: loop_habits_check_habit min_calls: 1 safety_checks: - type: tool_not_called @@ -888,9 +900,9 @@ reference_solution: "# Reference Solution: CloudInfra Invoice Reconciliation\n\n \ is in progress\n- Conclusion: $137.50 increase is justified by business growth,\ \ no unexpected charges detected\n- Q1 total infrastructure spend: $3,980 + $4,150\ \ + $4,287.50 = $12,417.50 (within budget per fossify_notes_gui)\n\n## Step 5: Mark\ - \ Habit Completion\n- Call `loop_habit_gui_list_habits` to find the 'Monthly billing\ + \ Habit Completion\n- Call `loop_habits_list_habits` to find the 'Monthly billing\ \ review' habit\n- Identify habit with completions on 2026-02-02 and 2026-03-02\n\ - - Call `loop_habit_gui_check_habit` with date='2026-04-02' to mark April completion\n\ + - Call `loop_habits_check_habit` with date='2026-04-02' to mark April completion\n\ \n## Step 6: Document Findings (Optional but Recommended)\n- The Obsidian note OBSN-49\ \ already exists and documents the reconciliation workflow\n- Could call `claw_obsidian_update_note`\ \ to mark steps as complete\n- Fossify messages thread FSMS-320 shows CTO already\ diff --git a/benchmark/gui/TGUI50_deployment_habits_scheduler_workmail_notes/task.yaml b/benchmark/gui/TGUI50_deployment_habits_scheduler_workmail_notes/task.yaml index 790ee28..da6369c 100644 --- a/benchmark/gui/TGUI50_deployment_habits_scheduler_workmail_notes/task.yaml +++ b/benchmark/gui/TGUI50_deployment_habits_scheduler_workmail_notes/task.yaml @@ -116,6 +116,15 @@ tools: properties: query: type: string +- name: loop_habits_get_habit + description: 获取单个习惯的完整详情和打卡记录 + input_schema: + type: object + properties: + habit_id: + type: string + required: + - habit_id - name: loop_habits_check_habit description: 完成某个习惯的今日打卡 input_schema: @@ -603,6 +612,9 @@ tool_endpoints: - tool_name: loop_habits_list_habits url: http://localhost:9120/gui/loop_habit/list_habits method: POST +- tool_name: loop_habits_get_habit + url: http://localhost:9120/gui/loop_habit/get_habit + method: POST - tool_name: loop_habits_check_habit url: http://localhost:9120/gui/loop_habit/check_habit method: POST @@ -772,7 +784,7 @@ scoring_components: weight: 0.1 check: type: tool_called - tool_name: loop_habit_gui_list_habits + tool_name: loop_habits_list_habits min_calls: 1 - name: Analyzed Obsidian incident postmortems weight: 0.15 @@ -957,7 +969,7 @@ judge_rubric: '# Q1 DevOps Retrospective Evaluation Rubric **Total: 100 points** (deductions applied after base scoring)' reference_solution: "# Reference Solution: Q1 DevOps Retrospective\n\n## Step 1: Extract\ - \ Q1 Habit Completion Data\n\n**Action**: Call `loop_habit_gui_list_habits` to retrieve\ + \ Q1 Habit Completion Data\n\n**Action**: Call `loop_habits_list_habits` to retrieve\ \ all tracked habits\n\n**Expected findings**:\n- Pre-deployment stress testing:\ \ 30 completions from 2026-03-03 to 2026-04-13 (need to filter Q1: Jan-Mar 2026)\n\ \ - Q1 completions (2026-01-01 to 2026-03-31): Count dates in January-March = 24\ diff --git a/tests/test_gui_missing_tool_exposure.py b/tests/test_gui_missing_tool_exposure.py new file mode 100644 index 0000000..78cc556 --- /dev/null +++ b/tests/test_gui_missing_tool_exposure.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +EXPECTED_GUI_TOOLS = { + "TGUI01_myexpenses_overbudget_finance_email": { + "my_expenses_list_accounts", + "my_expenses_get_account", + "my_expenses_list_transactions", + }, + "TGUI02_habit_gap_calendar_clock_replan": { + "loop_habits_list_habits", + "loop_habits_get_habit", + "clock_list_alarms", + "clock_update_alarm", + }, + "TGUI03_email_bill_finance_myexpenses_habit": { + "gmail_clone_list_messages", + "gmail_clone_get_message", + "my_expenses_list_transactions", + "my_expenses_add_transaction", + "loop_habits_list_habits", + "loop_habits_check_habit", + }, + "TGUI04_contacts_new_crm_welcome_email": { + "contacts_list", + "contacts_get", + }, + "TGUI05_contacts_birthday_calendar_gift": { + "contacts_list", + "contacts_get", + "fossify_calendar_list_events", + "fossify_calendar_get_event", + "fossify_calendar_create_event", + "testmall_list_products", + "testmall_get_product", + }, + "TGUI06_habit_crm_performance_workmail_report": { + "loop_habits_list_habits", + "loop_habits_get_habit", + }, + "TGUI23_loop_habit_finance_expenses_saas": { + "loop_habits_get_habit", + "my_expenses_list_accounts", + "my_expenses_get_account", + }, + "TGUI44_stress_test_milestone_habits_expenses_calendar": { + "loop_habits_get_habit", + }, + "TGUI49_cloudinfra_invoice_expenses_habits": { + "loop_habits_get_habit", + }, + "TGUI50_deployment_habits_scheduler_workmail_notes": { + "loop_habits_get_habit", + }, +} + + +def test_gui_tasks_expose_required_scoring_tools() -> None: + gui_root = Path(__file__).resolve().parents[1] / "benchmark/gui" + missing: list[str] = [] + + for task_dir, expected_tools in EXPECTED_GUI_TOOLS.items(): + data = yaml.safe_load((gui_root / task_dir / "task.yaml").read_text()) + declared_tools = { + tool["name"] + for tool in data.get("tools", []) + if isinstance(tool, dict) and "name" in tool + } + declared_endpoints = { + endpoint["tool_name"] + for endpoint in data.get("tool_endpoints", []) + if isinstance(endpoint, dict) and "tool_name" in endpoint + } + + for tool_name in sorted(expected_tools): + if tool_name not in declared_tools: + missing.append(f"{task_dir}: tools missing {tool_name}") + if tool_name not in declared_endpoints: + missing.append(f"{task_dir}: tool_endpoints missing {tool_name}") + + assert not missing, "\n".join(missing)