diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py index a4e7ddb..1051d9d 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py @@ -2,10 +2,12 @@ # Licensed under the MIT License. """ -ESS FlightCheck — Workday Deep Validation (WD-ENV-xxx, WD-CONN-xxx, WD-FLOW-xxx, WD-WF-xxx) +ESS FlightCheck — Workday Deep Validation (WD-ENV-xxx, WD-CONN-xxx, WD-FLOW-xxx, WD-WF-xxx, WD-WF-CAT-xxx) Validates Workday environment variables, connection references, flow status, -and tests all 17 ESS SOAP workflows against the actual Workday API. +tests all 17 ESS SOAP workflows against the actual Workday API, and surfaces +a manual checklist for any Workday scenario referenced in customer topics +that is outside the OOTB scenario catalog (WD-WF-CAT-001). The SOAP tests reuse the Kit's Workday MCP client (src/mcp/workday/client.py) or, when running standalone, build SOAP envelopes directly with httpx. @@ -16,6 +18,7 @@ import os import re import sys +from pathlib import Path from xml.sax.saxutils import escape as xml_escape # Use defusedxml everywhere we parse SOAP responses. Workday talks to us over @@ -222,6 +225,16 @@ def run_workday_checks(runner) -> list[CheckResult]: runs at the end of the workday block when WD-PKG-001 detected a known flavor, so its diagnostic can list which Microsoft-shipped refs are bound vs. unbound for the detected flavor. + + WD-WF-CAT-001 (custom-workflow inventory checklist) runs after the + SOAP tests. It walks `workspace/agents/*/topics/*.mcs.yml` for + Workday scenario references that are NOT in the live OOTB catalog + resolved from the customer's Dataverse + (`msdyn_employeeselfservicetemplateconfigs` where `ismanaged=true`) + and surfaces them as a MANUAL row with a 4-item operator checklist. + WD-WF-CAT-LINK is the corresponding cross-link trailer emitted + from inside `_check_workflows` so admins reading a green 17-row + SOAP report don't miss the manual row. """ results: list[CheckResult] = [] @@ -275,6 +288,16 @@ def run_workday_checks(runner) -> list[CheckResult]: # connection checks above. results.extend(_check_package_connection_completeness(runner)) + # WD-WF-CAT-001 — Workday custom-workflow inventory checklist. + # Runs after the SOAP tests so the WD-WF-CAT-LINK trailer that + # `_check_workflows` emits inside its own returns has already + # populated `runner._workday_unknown_scenarios` (the trailer + # triggers the lazy discovery walk via _get_unknown_workday_scenarios). + # Emitting WD-WF-CAT-001 here ensures the full manual checklist + # appears in the per-Workday-block output even if there were no + # SOAP tests run (e.g. credentials unavailable). + results.extend(_check_custom_workflow_inventory(runner)) + return results @@ -1908,11 +1931,13 @@ def _check_workflows(runner) -> list[CheckResult]: """ flavor = getattr(runner, "_workday_package_flavor", None) if flavor == "simplified": - return [_simplified_install_skip( + results = [_simplified_install_skip( checkpoint_id="WD-WF-000", description="Workday SOAP workflow tests", category="Workday Workflows", )] + _append_wd_wf_cat_link_trailer(runner, results) + return results results = [] @@ -1931,6 +1956,7 @@ def _check_workflows(runner) -> list[CheckResult]: result="Workday not configured - skipping 17 workflow tests", remediation="Run /connect workday first, then re-run /flightcheck.", )) + _append_wd_wf_cat_link_trailer(runner, results) return results if not test_employee: @@ -1941,6 +1967,7 @@ def _check_workflows(runner) -> list[CheckResult]: result="No test employee ID provided - skipping workflow tests", remediation="Re-run flightcheck and enter a test employee ID when prompted.", )) + _append_wd_wf_cat_link_trailer(runner, results) return results # Safe to log here - tenant is from the metadata-only resolver, but we @@ -1958,6 +1985,7 @@ def _check_workflows(runner) -> list[CheckResult]: result="httpx not installed - skipping", remediation="pip install httpx", )) + _append_wd_wf_cat_link_trailer(runner, results) return results # --- Now resolve credentials. From this point on the local scope holds @@ -1975,6 +2003,7 @@ def _check_workflows(runner) -> list[CheckResult]: "username and password to test the 17 workflows." ), )) + _append_wd_wf_cat_link_trailer(runner, results) return results import datetime @@ -2064,9 +2093,638 @@ def _check_workflows(runner) -> list[CheckResult]: description=desc, result=f"Error: {error[:100]}", )) + # Cross-link trailer (WD-WF-CAT-LINK) — surfaces the WD-WF-CAT-001 + # manual checklist from inside the SOAP-test output so admins + # reading a clean 17-row pass don't miss the custom-scenario + # inventory row. See `_append_wd_wf_cat_link_trailer` for why this + # also fires on the credential-missing skip path. + _append_wd_wf_cat_link_trailer(runner, results) + return results +def _append_wd_wf_cat_link_trailer(runner, results: list[CheckResult]) -> None: + """Append the WD-WF-CAT-LINK row to `results` if there are unknown + Workday scenario references in customer topics. + + Called from EVERY exit path of `_check_workflows` (not just the + SOAP-test-loop-completed path) so the cross-link surfaces even + when SOAP tests are skipped for credential-missing reasons. + AC2 ("Linked from Test-WorkdayWorkflows output when an unknown + workflow name is referenced in customer topics") needs the link + to appear regardless of whether the operator has configured ISU + creds yet — the inventory of custom scenarios doesn't depend on + Workday connectivity. + + The result text is deliberately worded "above" without naming + "17 SOAP tests" because some exit paths emit zero SOAP rows. + """ + unknown = _get_unknown_workday_scenarios(runner) + if not unknown: + return + results.append(CheckResult( + checkpoint_id="WD-WF-CAT-LINK", category="Workday Workflows", + priority=Priority.MEDIUM.value, status=Status.MANUAL.value, + description="Custom Workday scenarios outside the SOAP-test catalog", + result=( + f"{len(unknown)} Workday scenario reference(s) in customer " + "topics are NOT covered by the automated SOAP tests in this " + "Workday Workflows block." + ), + remediation=( + "See WD-WF-CAT-001 for the per-scenario list and the " + "manual verification checklist (ISU account, payload " + "shape vs. Workday WSDL, evaluation test prompt, " + "connection-ref auth health)." + ), + )) + + +# ───────────────────────────────────────────────────────────────────────── +# WD-WF-CAT-001 — Workday custom-workflow inventory checklist (MANUAL) +# ───────────────────────────────────────────────────────────────────────── +# +# The kit hard-codes a 17-workflow SOAP-test catalog (`WORKFLOWS` +# above). Customers wire up additional Workday scenarios two ways: +# +# 1. Template-config + topic — a topic calls +# `dialog: msdyn_copilotforemployeeselfservicehr.topic.WorkdaySystemGetCommonExecution` +# with a `scenarioName: msdyn_...` value. The shared flow reads +# the matching template config row from Dataverse by ScenarioName +# and executes the SOAP request. +# 2. Standalone topic + cloud flow — `kind: InvokeFlowAction` +# pointing at a Power Automate flow bound to `shared_workdaysoap`. +# +# Both paths exit FlightCheck's automated validation surface. The +# kit cannot: +# * Parse tenant-specific Workday WSDL to validate payload shape. +# * Read template config XML field-by-field against the WSDL. +# * Confirm the ISU used by the custom scenario has the right +# Workday security domain. +# * Tell whether an evaluation test exists for the custom scenario. +# +# Per AGENTS.md principle #2, this is the canonical MANUAL pattern: +# the kit observes one side (topic references scenario X), the +# operator verifies the other side (Workday-tenant configuration). +# +# Data sources: +# * Local YAML walk (`workspace/agents/*/topics/*.mcs.yml`) — +# no API call, no cassette required. +# * Live Dataverse query against +# `msdyn_employeeselfservicetemplateconfigs` filtered by +# `ismanaged=true` — the tenant-accurate OOTB scenario set. +# * Cached `runner._workday_package_flavor` (from WD-PKG-001) for +# the simplified-install gate per principle #11. +# +# Outputs: +# * One PASSED row if every discovered scenario is in the live +# Dataverse-resolved OOTB catalog. +# * One MANUAL row enumerating unknowns + the 4-item checklist +# (Principle #7 — bucketed, not per-resource). +# * One SKIPPED row on simplified install (no ISU/RaaS), missing +# workspace, zero Workday references, or missing Dataverse token +# (cannot resolve OOTB catalog → cannot make a PASS/FAIL claim +# per AGENTS.md principle #1). +# * One WARNING row if the Dataverse query errored — surfaces the +# error per principle #3 instead of silently passing. +# Caches discovered + unknown lists on the runner so the WD-WF-CAT-LINK +# trailer inside _check_workflows can reference them without re-walking. + +# Marker for the shared Workday execution topic (Pattern A). The +# `dialog:` field is fully-qualified and always contains this suffix +# regardless of agent publisher prefix. +_WORKDAY_SYSTEM_DIALOG_SUFFIX = ".topic.WorkdaySystemGetCommonExecution" + + +def _load_workday_ootb_catalog_from_dataverse( + runner, +) -> tuple[set[str] | None, str]: + """Query the customer's Dataverse tenant for OOTB Workday scenarios. + + Reads `msdyn_employeeselfservicetemplateconfigs` (the table the + Workday extension pack writes to during installation) and returns + the set of scenario names whose records are `ismanaged=true` — + i.e. shipped by Microsoft's managed solution, not added by the + customer. This is the tenant-accurate, drift-free source of truth + for "what counts as OOTB Workday." + + Returns (catalog, status): + catalog: set[str] of scenario names on success, or None when + the catalog could not be resolved. + status: "ok" — query succeeded. + "no_token" — no Dataverse token / env URL + available; caller must SKIP + per AGENTS.md principle #1. + "query_error: " — Dataverse query errored; + caller must WARN per + principle #3 (fail loudly on + API errors). + """ + env_url = getattr(runner, "env_url", None) + dv_token = getattr(runner, "dv_token", None) + if not env_url or not dv_token: + return (None, "no_token") + try: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + from auth import query_all + rows = query_all( + env_url, dv_token, + "msdyn_employeeselfservicetemplateconfigs", + "msdyn_name,ismanaged", + ) + except Exception as exc: # noqa: BLE001 — surface the error verbatim + return (None, f"query_error: {exc}") + catalog = { + r.get("msdyn_name", "") + for r in rows + if r.get("ismanaged") is True and r.get("msdyn_name") + } + return (catalog, "ok") + + +def _get_workday_ootb_catalog( + runner, +) -> tuple[set[str] | None, str]: + """Resolve the Workday OOTB scenarioName catalog from Dataverse. + + There is no fallback: the customer's own Dataverse tenant is the + only authoritative source for what the installed Workday extension + pack ships. If we can't reach it, the caller must SKIP or WARN per + AGENTS.md principle #1 (never return PASS when the check cannot + actually validate what it claims to validate). + + Returns (catalog, status). See + `_load_workday_ootb_catalog_from_dataverse` for the status values. + + Caches the resolution on `runner._workday_ootb_catalog_cache` so + repeated reads within one flightcheck invocation don't re-query + Dataverse. + """ + cached = getattr(runner, "_workday_ootb_catalog_cache", None) + if cached is not None: + return cached + result = _load_workday_ootb_catalog_from_dataverse(runner) + runner._workday_ootb_catalog_cache = result + return result + + +def _is_workday_bound_workflow_json(workflow_json_path: Path) -> bool: + """Return True if a Power Automate workflow.json references the + Workday SOAP connector (`shared_workdaysoap`) anywhere in its + `connectionReferences` block. + + Used to qualify Pattern B (InvokeFlowAction) references: a flow + is "Workday-bound" if any of its connection references binds to + the Workday SOAP connector. Conservative — a flow that touches + both Workday and another connector still counts as Workday-bound + so the operator gets surfaced for review. + """ + if not workflow_json_path.exists(): + return False + try: + data = json.loads(workflow_json_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return False + conn_refs = data.get("properties", {}).get("connectionReferences", {}) + for ref in conn_refs.values(): + api_obj = ref.get("api", {}) + if api_obj.get("name", "") == "shared_workdaysoap": + return True + # Fallback for older flow JSON shapes that put it directly: + if ref.get("apiId", "").lower().endswith(WORKDAY_SOAP_CONNECTOR_SUFFIX): + return True + return False + + +def _build_workflow_id_to_path_index(agent_dir: Path) -> dict[str, Path]: + """Walk `agent_dir/workflows/*/metadata.yml`, parse the workflowId + out of each, and return {workflowId(lowercase): workflow.json path}. + + Built once per agent so Pattern-B lookups are O(1) per topic + reference. Metadata YAML format is documented in + workspace/agents/{slug}/workflows//metadata.yml — single key + `workflowId:` plus `jsonFileName:` (always `workflow.json`). + """ + index: dict[str, Path] = {} + workflows_dir = agent_dir / "workflows" + if not workflows_dir.exists(): + return index + for sub in workflows_dir.iterdir(): + if not sub.is_dir(): + continue + metadata = sub / "metadata.yml" + if not metadata.exists(): + continue + try: + content = metadata.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + m = re.search(r"^\s*workflowId\s*:\s*([0-9a-fA-F-]{36})\s*$", content, re.MULTILINE) + if m: + workflow_json = sub / "workflow.json" + index[m.group(1).lower()] = workflow_json + return index + + +def _scan_topic_for_workday_refs( + topic_path: Path, agent_slug: str, workflow_index: dict[str, Path], +) -> list[dict]: + """Scan a single topic YAML for Workday scenario references. + + Returns a list of dicts, one per reference found. Each dict has: + agent: the agent folder slug + topic: topic filename + line: 1-based line number of the trigger line + scenarioName: str | None (None for Pattern B) + pattern: "system-common-execution" | "invoke-flow-action" + flowId: str | None (set only for Pattern B) + """ + try: + lines = topic_path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return [] + + refs: list[dict] = [] + + # Pattern A: line ending in WorkdaySystemGetCommonExecution; walk + # back to find the nearest scenarioName: above it (within ~30 + # lines — the typical BeginDialog block size). + for i, line in enumerate(lines): + stripped = line.strip() + if "dialog:" in stripped and _WORKDAY_SYSTEM_DIALOG_SUFFIX in stripped: + scenario_line = None + scenario_value = None + for j in range(i - 1, max(-1, i - 31), -1): + back = lines[j].strip() + m = re.match(r"^scenarioName\s*:\s*=?\s*\"?([\w]+)\"?\s*$", back) + if m: + scenario_line = j + 1 + scenario_value = m.group(1) + break + refs.append({ + "agent": agent_slug, + "topic": topic_path.name, + "line": scenario_line if scenario_line else i + 1, + "scenarioName": scenario_value, + "pattern": "system-common-execution", + "flowId": None, + }) + + # Pattern B: InvokeFlowAction → flowId → workflow.json bound to + # shared_workdaysoap. Pull every (kind:InvokeFlowAction, flowId) + # pair, then qualify against the workflow index. + in_invoke_block = False + invoke_start_line = -1 + for i, line in enumerate(lines): + stripped = line.strip() + if re.match(r"^-?\s*kind\s*:\s*InvokeFlowAction\s*$", stripped): + in_invoke_block = True + invoke_start_line = i + 1 + continue + if in_invoke_block: + m = re.match(r"^flowId\s*:\s*([0-9a-fA-F-]{36})\s*$", stripped) + if m: + flow_id = m.group(1).lower() + workflow_json = workflow_index.get(flow_id) + if workflow_json and _is_workday_bound_workflow_json(workflow_json): + refs.append({ + "agent": agent_slug, + "topic": topic_path.name, + "line": invoke_start_line, + "scenarioName": None, + "pattern": "invoke-flow-action", + "flowId": flow_id, + }) + in_invoke_block = False + invoke_start_line = -1 + # Heuristic close: a new top-level `- kind:` line ends + # the current InvokeFlowAction block without a flowId. + elif re.match(r"^-\s*kind\s*:", stripped) and i > invoke_start_line - 1: + in_invoke_block = False + invoke_start_line = -1 + return refs + + +def _discover_customer_workday_scenarios( + workspace_root: Path = Path("workspace/agents"), +) -> list[dict]: + """Walk every agent under workspace_root and return all Workday + scenario references (Pattern A + Pattern B). Returns [] when the + workspace doesn't exist (callers should treat that as SKIPPED, not + PASSED — see _check_custom_workflow_inventory). + """ + if not workspace_root.exists(): + return [] + discovered: list[dict] = [] + for agent_dir in sorted(workspace_root.iterdir()): + if not agent_dir.is_dir() or agent_dir.name.startswith("."): + continue + topics_dir = agent_dir / "topics" + if not topics_dir.exists(): + continue + workflow_index = _build_workflow_id_to_path_index(agent_dir) + for topic_file in sorted(topics_dir.glob("*.mcs.yml")): + discovered.extend( + _scan_topic_for_workday_refs( + topic_file, agent_dir.name, workflow_index, + ) + ) + return discovered + + +def _get_unknown_workday_scenarios(runner) -> list[dict]: + """Return the cached list of unknown Workday refs, computing and + caching it on first read. + + Used by both _check_custom_workflow_inventory and the + WD-WF-CAT-LINK trailer inside _check_workflows so the topic walk + runs at most once per flightcheck invocation regardless of which + check the runner schedules first. + + The OOTB catalog this diff'd against comes from + `_get_workday_ootb_catalog` (live Dataverse). When the catalog + cannot be resolved (no token / query error), this function returns + an empty list — the WD-WF-CAT-001 main check is the single source + of truth for surfacing that condition (SKIPPED / WARNING). The + trailer self-suppresses in that case so we don't claim "0 unknowns" + when the truth is "we couldn't tell." + """ + cached = getattr(runner, "_workday_unknown_scenarios", None) + if cached is not None: + return cached + workspace_root_str = "workspace/agents" + discovered = _discover_customer_workday_scenarios(Path(workspace_root_str)) + runner._workday_discovered_scenarios = discovered + if not discovered: + runner._workday_unknown_scenarios = [] + return [] + catalog, _status = _get_workday_ootb_catalog(runner) + if catalog is None: + runner._workday_unknown_scenarios = [] + return [] + unknown = [ + ref for ref in discovered + if ref["pattern"] == "invoke-flow-action" + or (ref.get("scenarioName") and ref["scenarioName"] not in catalog) + ] + runner._workday_unknown_scenarios = unknown + return unknown + + +def _format_unknown_scenarios(unknown: list[dict]) -> str: + """Format the unknown-refs list for the result field. One block per + reference, agent + topic + line cited verbatim per AGENTS.md + principle #8 (result = what the kit observed). + """ + lines: list[str] = [] + for ref in unknown: + if ref["pattern"] == "system-common-execution": + name = ref.get("scenarioName") or "(scenarioName not found in topic)" + lines.append(f" • {name}") + lines.append( + f" Topic: topics/{ref['topic']}:{ref['line']}" + ) + lines.append(" Pattern: WorkdaySystemGetCommonExecution + scenarioName") + else: # invoke-flow-action + lines.append(f" • ({ref.get('flowId', '')})") + lines.append( + f" Topic: topics/{ref['topic']}:{ref['line']}" + ) + lines.append(" Pattern: InvokeFlowAction → flow bound to shared_workdaysoap") + lines.append(f" Agent: {ref['agent']}") + lines.append("") + return "\n".join(lines).rstrip() + + +_WD_WF_CAT_CHECKLIST = ( + "Manual verification required — the kit cannot validate custom " + "Workday scenarios end-to-end. For EACH scenario listed above:\n" + "\n" + " 1. ISU account: Confirm which ISU registered in Workday is used " + "by this scenario. Default is the account in environment variable " + "EmployeeContextRequestAccountName (see WD-ENV-001 output). Custom " + "scenarios may use a different ISU — verify in the template config " + "XML in Dataverse (Power Platform Maker → Tables → " + "msdyn_employeeselfservicetemplateconfigs → search ScenarioName).\n" + "\n" + " 2. Payload shape: Open the template config XML and confirm the " + "SOAP request body matches the Workday WSDL for the named service. " + "Field names, reference types, and required vs. optional elements " + "MUST match the Workday contract. Mismatches surface at runtime as " + "the 'Workflow Contract/Payload Mismatch' failure mode.\n" + "\n" + " 3. Test prompt: Add at least one evaluation test case to the " + "agent's evaluations/ folder that exercises the scenario end-to-end " + "with a known-good employee. Use /create-eval and tag the test set " + "with the scenario name.\n" + "\n" + " 4. Auth health: Re-check the WD-CONN-* connection token health " + "output for the connection reference this scenario uses. " + "Intermittent auth failures usually trace to a stale OAuth token " + "on one of the ISU refs — reauthenticate in Power Platform Maker " + "→ Connections.\n" + "\n" + "Note: the OOTB Workday catalog is resolved live from the " + "customer's own Dataverse " + "(msdyn_employeeselfservicetemplateconfigs, filtered by " + "ismanaged=true), which auto-detects every scenario the installed " + "Workday extension pack ships. A scenario surfacing as MANUAL " + "means it is NOT a managed row in the customer's tenant — either " + "it is genuinely custom (work through the checklist above) or the " + "extension pack is not installed.\n" + "\n" + "Found a new pattern? Log it back to the gap-discovery process. " + "File an issue at " + "https://github.com/microsoft/Employee-Self-Service-Agent-Developer-Kit/issues/new " + "with title 'Workday gap-discovery: ' if any of " + "the following apply:\n" + " • This MANUAL row surfaced a scenario that you believe should " + "ship OOTB in the Workday extension pack (forward to Microsoft " + "ESS so the next pack revision can include it).\n" + " • A Workday-bound topic in your agent did NOT surface here but " + "should have (the detection walker missed a new wiring pattern — " + "Pattern C or beyond; attach the topic YAML snippet so a new " + "detection rule can be added to _scan_topic_for_workday_refs).\n" + " • The 4-item checklist above was insufficient for diagnosing " + "your scenario (propose the additional verification step).\n" + "Include the topic YAML snippet, the scenarioName / flowId, and " + "the ADO incident number if any. Closing the loop here is how " + "WD-WF-CAT-001 gets better over time." +) + + +def _check_custom_workflow_inventory(runner) -> list[CheckResult]: + """WD-WF-CAT-001 — Manual checklist for custom Workday workflows. + + Gates (in order): + * `runner._workday_package_flavor == "simplified"` → SKIPPED + (ISU/scenario inventory doesn't apply on the simplified + install per AGENTS.md principle #11). + * Missing `workspace/agents/` → SKIPPED. + * Zero Workday references discovered in any topic → SKIPPED. + * No Dataverse token / env URL → SKIPPED (we cannot resolve the + live OOTB catalog and must not return PASS per principle #1). + * Dataverse query errored → WARNING (surface the error per + principle #3 rather than silently swallowing it). + + Otherwise: + * Every discovered scenario is a managed row in Dataverse → PASSED. + * Any unknown / flow-bound reference → MANUAL with bucketed + listing in `result` and the 4-item checklist in `remediation`. + + Caches discovered list on `runner._workday_discovered_scenarios` + and the (possibly empty) unknown list on + `runner._workday_unknown_scenarios` so the WD-WF-CAT-LINK trailer + inside _check_workflows can read them without re-walking. + """ + cp_id = "WD-WF-CAT-001" + category = "Workday Workflows" + description = "Workday custom-workflow inventory checklist" + doc_link = f"{DOC_BASE}/workday-extensibility" + + flavor = getattr(runner, "_workday_package_flavor", None) + if flavor == "simplified": + return [_simplified_install_skip( + checkpoint_id=cp_id, + description=description, + category=category, + )] + + workspace_root = Path("workspace/agents") + if not workspace_root.exists(): + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.SKIPPED.value, + description=description, + result="workspace/agents/ directory not found.", + remediation=( + "Run /setup to extract agent files before this check " + "can enumerate Workday scenario references in topics." + ), + doc_link=doc_link, + )] + + # Discover Workday refs from topics first — no catalog needed for + # this step. If there are none, we can SKIP cleanly without even + # touching Dataverse. + discovered = _discover_customer_workday_scenarios(workspace_root) + runner._workday_discovered_scenarios = discovered + + if not discovered: + runner._workday_unknown_scenarios = [] + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.SKIPPED.value, + description=description, + result=( + "No Workday scenario references found in any agent topic. " + "Either Workday is not wired into the customer's agent yet, " + "or its topics are not yet extracted to " + "workspace/agents/*/topics/." + ), + remediation=( + "If the customer intends to use Workday, run /create to add " + "a Workday scenario topic, or /setup to re-extract topics " + "if you expected references to be present." + ), + doc_link=doc_link, + )] + + # We have Workday refs in topics — we need the live Dataverse + # OOTB catalog to know which are custom. No fallback: if Dataverse + # is unreachable, we cannot make a PASS/FAIL claim (principle #1). + catalog, status_code = _get_workday_ootb_catalog(runner) + + if catalog is None and status_code == "no_token": + # Suppress the trailer too — without the catalog we cannot + # legitimately list "unknowns." + runner._workday_unknown_scenarios = [] + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.SKIPPED.value, + description=description, + result=( + f"Found {len(discovered)} Workday scenario reference(s) " + "in customer topics, but no Dataverse credentials are " + "available to resolve the live OOTB scenario catalog " + "(msdyn_employeeselfservicetemplateconfigs where " + "ismanaged=true). Cannot determine which references are " + "custom vs. OOTB without that lookup." + ), + remediation=( + "Re-run flightcheck after running /setup so Dataverse " + "credentials are cached on the runner, or run it in an " + "environment where the Dataverse MCP server is " + "authenticated." + ), + doc_link=doc_link, + )] + + if catalog is None: + # query_error: — surface verbatim per principle #3. + err_msg = status_code.removeprefix("query_error: ") or "unknown error" + runner._workday_unknown_scenarios = [] + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.WARNING.value, + description=description, + result=( + f"Found {len(discovered)} Workday scenario reference(s) " + "in customer topics, but the Dataverse query for the " + "live OOTB scenario catalog " + "(msdyn_employeeselfservicetemplateconfigs where " + f"ismanaged=true) failed: {err_msg}. Cannot determine " + "which references are custom vs. OOTB until the query " + "succeeds." + ), + remediation=( + "Investigate the Dataverse error above. Common causes: " + "expired Dataverse token (re-run /setup), table not " + "present in this environment (ESS solution not " + "installed), or transient service outage (retry)." + ), + doc_link=doc_link, + )] + + # Catalog resolved cleanly — compute unknowns and cache them. + unknown = [ + ref for ref in discovered + if ref["pattern"] == "invoke-flow-action" + or (ref.get("scenarioName") and ref["scenarioName"] not in catalog) + ] + runner._workday_unknown_scenarios = unknown + + if not unknown: + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.PASSED.value, + description=description, + result=( + f"Found {len(discovered)} Workday scenario reference(s) " + "in customer topics. All are managed rows in the " + "customer's Dataverse " + "(msdyn_employeeselfservicetemplateconfigs where " + "ismanaged=true) and require no manual review." + ), + doc_link=doc_link, + )] + + body = _format_unknown_scenarios(unknown) + return [CheckResult( + checkpoint_id=cp_id, category=category, + priority=Priority.HIGH.value, status=Status.MANUAL.value, + description=description, + result=( + f"Found {len(unknown)} Workday scenario reference(s) in " + "customer topics that the kit cannot validate end-to-end " + "(not in the OOTB catalog):\n\n" + f"{body}" + ), + remediation=_WD_WF_CAT_CHECKLIST, + doc_link=doc_link, + )] + + # ---- Credential Resolution ---- def _resolve_workday_metadata(runner) -> tuple[str, str, str]: diff --git a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md index 8c9396e..11a3a89 100644 --- a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md +++ b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md @@ -94,6 +94,50 @@ Tests all 17 ESS pre-configured workflows against the Workday API. Requires ISU | WD-WF-016 | Update Email | Human_Resources | Write | | Contact Information | | WD-WF-017 | Update Phone | Human_Resources | Write | | Contact Information | +### Workday Custom-Workflow Inventory (WD-WF-CAT-xxx) — Manual + +The 17 SOAP tests above cover only the OOTB workflows the kit ships +SOAP envelopes for. Customers routinely wire up additional Workday +scenarios via two patterns: + +- **Pattern A** — Topic that calls + `WorkdaySystemGetCommonExecution` with a `scenarioName` of a + template-config record in Dataverse. +- **Pattern B** — Standalone topic that calls a customer-built cloud + flow bound to the `shared_workdaysoap` connector via + `InvokeFlowAction`. + +Both patterns exit the automated validation surface (the kit doesn't +ship a Workday WSDL parser; per-tenant security domain / ISU config +varies). WD-WF-CAT-001 walks `workspace/agents/*/topics/*.mcs.yml` +for these patterns and emits a MANUAL row enumerating any scenarios +that aren't in the OOTB catalog. The OOTB catalog is resolved live +from the customer's own Dataverse: the check queries +`msdyn_employeeselfservicetemplateconfigs` and treats every +`ismanaged=true` row as OOTB (auto-detects every scenario shipped by +the installed Workday extension pack, with no kit-side curation +needed). There is no fallback — if Dataverse credentials are missing +the check returns SKIPPED, and if the Dataverse query errors the +check returns WARNING surfacing the error verbatim. Returning PASSED +without a tenant-accurate catalog would violate FlightCheck design +principle #1 ("never return PASSED when the check cannot actually +validate what it claims to validate"). The MANUAL remediation +carries a 4-item checklist (ISU account, payload shape vs. Workday +WSDL, evaluation test prompt, connection-ref auth health). + +| ID | Check | Priority | Method | Doc Link | +|----|-------|----------|--------|----------| +| WD-WF-CAT-001 | Workday custom-workflow inventory checklist (MANUAL) | High | Local file walk + Dataverse managed-template-config diff (SKIP on no token, WARN on query error) | [workday-extensibility](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday-extensibility) | +| WD-WF-CAT-LINK | Cross-link trailer surfacing WD-WF-CAT-001 from inside the SOAP-test block | Medium | Computed from WD-WF-CAT-001 cache | [workday-extensibility](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday-extensibility) | + +MANUAL rows do not fail readiness (per FlightCheck design principle #2) +— they direct the operator to verify what the kit cannot. Address +each scenario by confirming it against the 4-item checklist in the +customer's environment. A scenario surfacing as MANUAL means it is +NOT a managed row in the customer's tenant — either it is genuinely +custom or the Workday extension pack is not installed in this +environment. + ## 6. Local Agent File Validation (Kit-exclusive) These checks parse the extracted agent files on disk — a capability the diff --git a/tests/flightcheck/checks/test_workday_custom_inventory.py b/tests/flightcheck/checks/test_workday_custom_inventory.py new file mode 100644 index 0000000..f535818 --- /dev/null +++ b/tests/flightcheck/checks/test_workday_custom_inventory.py @@ -0,0 +1,937 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Tests for WD-WF-CAT-001 (Workday custom-workflow inventory checklist) +and its WD-WF-CAT-LINK cross-link trailer emitted from inside +`_check_workflows`. + +These are PURE-LOGIC tests: the production check makes NO external API +calls — it reads `workspace/agents/*/topics/*.mcs.yml` and +`workspace/agents/*/workflows/*/workflow.json` from the local +filesystem. Per `tests/AGENTS.md` "The cardinal rule does not apply +to: Tests of the kit's pure-logic helpers (no network)." — no +cassettes or mock-tier enforcement needed. + +The check covers the WD-001 acceptance criteria (ADO 7392277, +incidents 760098889 / 783902203): + + AC1: Checklist enumerates: which custom Workday workflows are wired + up, ISU account used, expected payload shape, test prompt. + AC2: Linked from Test-WorkdayWorkflows output when an unknown + workflow name is referenced in customer topics. + AC3: Includes a "found a new pattern? log it here" loop back to + the gap-discovery process. + +Tests below pin each AC explicitly so a future drive-by refactor that +weakens the checklist text fails CI rather than silently ships a +weaker remediation. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +import responses + +from tests.conftest import ( + FAKE_DATAVERSE_URL, + FAKE_TOKEN, + require_validated_mock, +) +from tests.mocks import dataverse as dv + +require_validated_mock(dv) + + +# ───────────────────────────────────────────────────────────────────────── +# Minimal runner — _check_custom_workflow_inventory only reads +# `_workday_package_flavor` and writes the cached `_workday_*` lists. +# ───────────────────────────────────────────────────────────────────────── + + +@dataclass +class _MinimalRunner: + config: dict[str, Any] = field(default_factory=dict) + + +def _result_by_id(results: list, checkpoint_id: str): + matches = [r for r in results if r.checkpoint_id == checkpoint_id] + assert len(matches) == 1, ( + f"Expected exactly one result for {checkpoint_id}, got {len(matches)}: " + f"{[r.checkpoint_id for r in results]}" + ) + return matches[0] + + +# ───────────────────────────────────────────────────────────────────────── +# Workspace fixtures — small helpers that lay down realistic +# `workspace/agents//topics/` + `workspace/agents//workflows/` +# trees under a tmp_path, matching the on-disk shape exactly so the +# production walkers don't need to be parameterized for testing. +# ───────────────────────────────────────────────────────────────────────── + + +def _write_topic_system_common_execution( + topic_path: Path, *, scenario_name: str +) -> None: + """Pattern A: scenarioName + WorkdaySystemGetCommonExecution dialog. + + Layout mirrors `src/examples/ess-samples/Workday/ManagerScenarios/ + WorkdayManagersdirect-CompanyCode/topic.yaml` lines 38-46. + """ + topic_path.parent.mkdir(parents=True, exist_ok=True) + topic_path.write_text( + "kind: AdaptiveDialog\n" + "beginDialog:\n" + " kind: OnRecognizedIntent\n" + " id: test-topic\n" + " actions:\n" + " - kind: BeginDialog\n" + " id: Gt044B\n" + " displayName: Redirect to Workday Get Common Execution\n" + " input:\n" + " binding:\n" + ' parameters: ="{\\"params\\":[]}"\n' + f" scenarioName: {scenario_name}\n" + "\n" + " dialog: msdyn_copilotforemployeeselfservicehr.topic.WorkdaySystemGetCommonExecution\n" + " output:\n" + " binding:\n" + " errorResponse: Topic.errorResponse\n", + encoding="utf-8", + ) + + +def _write_topic_invoke_flow_action( + topic_path: Path, *, flow_id: str +) -> None: + """Pattern B: kind: InvokeFlowAction with a flowId pointing at a + Workday-bound flow. Layout mirrors the canonical InvokeFlowAction + shape in `src/examples/ess-samples/Facilities/.../topic.yaml`.""" + topic_path.parent.mkdir(parents=True, exist_ok=True) + topic_path.write_text( + "kind: AdaptiveDialog\n" + "beginDialog:\n" + " kind: OnRecognizedIntent\n" + " actions:\n" + " - kind: InvokeFlowAction\n" + " id: invoke-1\n" + f" flowId: {flow_id}\n" + " input:\n" + " binding:\n" + " text: =Topic.UserQuery\n" + " output:\n" + " binding:\n" + " response: Topic.Response\n", + encoding="utf-8", + ) + + +def _write_workflow( + agent_dir: Path, + *, + workflow_id: str, + workflow_slug: str, + api_name: str, +) -> None: + """Write a minimal workflow folder with `metadata.yml` + + `workflow.json`. `api_name` controls whether the flow is + Workday-bound — set to `shared_workdaysoap` to make this a Workday + flow, anything else (e.g. `shared_servicenow`) makes it + non-Workday.""" + wf_dir = agent_dir / "workflows" / workflow_slug + wf_dir.mkdir(parents=True, exist_ok=True) + (wf_dir / "metadata.yml").write_text( + f"workflowId: {workflow_id}\n" + "jsonFileName: workflow.json\n", + encoding="utf-8", + ) + (wf_dir / "workflow.json").write_text( + json.dumps({ + "properties": { + "connectionReferences": { + "primary": { + "api": {"name": api_name} + } + } + } + }), + encoding="utf-8", + ) + + +def _write_catalog_marker_file(tmp_path: Path) -> None: + """Make sure `Path("workspace/agents")` resolves relative to + `tmp_path` (the test chdir'd here in the autouse fixture). The + discovery walker uses `Path("workspace/agents")` directly — a + relative path resolved against CWD — so no marker file is needed, + but a `workspace/` dir must exist or the walker returns [] (which + triggers the "directory not found" SKIP path rather than the + "no Workday refs" SKIP path).""" + (tmp_path / "workspace" / "agents").mkdir(parents=True, exist_ok=True) + + +# ───────────────────────────────────────────────────────────────────────── +# Class — keeps the autouse env-isolation fixture from leaking into +# unrelated test files. Mirrors the structure of +# `test_workday_workflows_gate.py::TestSimplifiedInstallGate`. +# ───────────────────────────────────────────────────────────────────────── + + +class TestCustomWorkflowInventory: + @pytest.fixture(autouse=True) + def _isolate_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """The production check uses `Path("workspace/agents")` relative + to CWD. Tests must chdir into tmp_path so each test sees the + workspace it just built and is isolated from any sibling test + AND from the developer's repo (which has a real + `workspace/agents/` for `employee-self-service-it`).""" + monkeypatch.chdir(tmp_path) + + # ------------------------------------------------------------------ + # Gates + # ------------------------------------------------------------------ + + def test_simplified_install_skips(self) -> None: + """Principle #11: simplified install (no ISU) has no concept of + ISU/scenario inventory — the check must short-circuit via the + canonical `_simplified_install_skip` helper. Pin that the SKIP + carries WD-PKG-001 reasoning so the operator knows WHY this + check skipped (vs e.g. a credential-missing skip).""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + runner = _MinimalRunner() + runner._workday_package_flavor = "simplified" + + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Skipped" + assert r.category == "Workday Workflows" + # `_simplified_install_skip` produces a result that names + # WD-PKG-001 as the gating check — pin that contract so a + # future refactor that bypasses the shared helper fails. + assert "WD-PKG-001" in r.result + assert "simplified" in r.result.lower() + + def test_no_workspace_directory_skips(self) -> None: + """If `/setup` hasn't run, there's no `workspace/agents/` and the + discovery walker has nothing to scan. SKIP with a result that + names the missing directory verbatim and a remediation that + directs the operator to `/setup` (so this isn't mistaken for + a "no Workday in customer's agent" PASS).""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + runner = _MinimalRunner() + # No workspace/agents/ — _isolate_env chdir'd to a clean tmp_path. + + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Skipped" + assert "workspace/agents/" in r.result + assert "/setup" in r.remediation + + def test_workspace_exists_but_zero_refs_skips(self, tmp_path: Path) -> None: + """A workspace with topics but ZERO Workday references is not a + PASS — the customer may simply not use Workday. SKIP with a + message that names both possibilities (not wired up vs. not + extracted) so the operator can pick the right next step.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + _write_catalog_marker_file(tmp_path) + agent_dir = tmp_path / "workspace" / "agents" / "test-agent" + (agent_dir / "topics").mkdir(parents=True, exist_ok=True) + # Topic that mentions neither WorkdaySystemGetCommonExecution + # nor an InvokeFlowAction bound to shared_workdaysoap. + (agent_dir / "topics" / "greeting.mcs.yml").write_text( + "kind: AdaptiveDialog\n" + "beginDialog:\n" + " kind: OnRecognizedIntent\n" + " actions:\n" + " - kind: SendActivity\n" + " activity: Hello!\n", + encoding="utf-8", + ) + + runner = _MinimalRunner() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Skipped" + # Pin both possibilities so the message stays operator-actionable. + assert "No Workday scenario references" in r.result + assert "not wired into" in r.result + assert "/create" in r.remediation or "/setup" in r.remediation + + # ------------------------------------------------------------------ + # Catalog matching — MANUAL path (the core acceptance criterion) + # ------------------------------------------------------------------ + + @responses.activate + def test_unknown_scenario_emits_manual_with_full_checklist( + self, tmp_path: Path + ) -> None: + """The primary acceptance criterion (AC1): an unknown scenario + surfaces as one MANUAL row with the scenario name in `result` + and the 4-item checklist (ISU / payload / test prompt / auth) + in `remediation`. Pins every checklist item literally so a + drive-by edit that drops one fails CI.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "custom.mcs.yml", + # A clearly-not-shipped name. If a managed row ever has this + # exact name the test breaks loudly — that's the intended + # behaviour. + scenario_name="msdyn_HRCustomNotInCatalogXYZ_TestOnly", + ) + # Empty managed-row response → every discovered scenario is + # treated as custom and surfaces as MANUAL. + _register_template_configs_response(rows=[]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + + # Per AGENTS.md principle #2: MANUAL = "Manual" — must NOT be + # "Failed" or "Warning". MANUAL does not fail readiness. + assert r.status == "Manual", ( + f"Custom scenario must surface as MANUAL (per AGENTS.md " + f"principle #2), got {r.status!r}" + ) + assert r.priority == "High" + + # `result` (AGENTS.md principle #8: what the kit observed): + # must name the scenario verbatim + cite the topic file + line. + assert "msdyn_HRCustomNotInCatalogXYZ_TestOnly" in r.result + assert "topics/custom.mcs.yml" in r.result + assert "WorkdaySystemGetCommonExecution" in r.result + assert "ess-hr" in r.result # agent slug + + # `remediation` (AGENTS.md principle #8: action only). All four + # checklist items (AC1 verbatim from the ticket) must appear. + assert "ISU account" in r.remediation + assert "Payload shape" in r.remediation + assert "Test prompt" in r.remediation + assert "Auth health" in r.remediation + # Pin specific actionable phrases the operator needs to act: + assert "msdyn_employeeselfservicetemplateconfigs" in r.remediation + assert "/create-eval" in r.remediation + # AC3 ("found a new pattern? log it here" loop-back) is pinned + # separately by test_checklist_includes_gap_discovery_loopback + # below — keep that test in lockstep with the AC3 paragraph in + # _WD_WF_CAT_CHECKLIST. + + @responses.activate + def test_checklist_includes_gap_discovery_loopback( + self, tmp_path: Path + ) -> None: + """AC3: the MANUAL remediation MUST include a "found a new + pattern? log it here" paragraph that closes the loop back to + the kit's gap-discovery process. Without it, customers who hit + a detection gap (e.g. a Pattern C wiring the walker doesn't + catch) or a scenario they believe should ship OOTB have no + canonical channel to forward that signal, and WD-WF-CAT-001 + can't improve over time. + + The original commit eb02d32 included this loop-back text; the + Dataverse-API refactor (0fb2383) dropped it on the rationale + that kit-side PRs aren't the remediation for the catalog + anymore. But AC3 is broader than the catalog — it covers + detection-pattern gaps and OOTB-promotion feedback too. Pin + the restored paragraph here so a future drive-by edit that + re-strips it fails CI, not silently ships a weaker checklist. + """ + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "custom.mcs.yml", + scenario_name="msdyn_HRCustomAC3Test_Unknown", + ) + # Empty managed-row response → scenario surfaces as MANUAL so + # the full _WD_WF_CAT_CHECKLIST renders (the loop-back text + # is part of the same checklist, not a separate row). + _register_template_configs_response(rows=[]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Manual" + + # The verbatim AC3 framing phrase from the ticket. If this + # disappears the next time someone refactors the checklist, + # CI must catch it. + assert "Found a new pattern" in r.remediation, ( + "AC3 loop-back framing dropped from checklist — operators " + "have no canonical channel to log gap-discovery feedback" + ) + assert "gap-discovery process" in r.remediation, ( + "AC3 must explicitly name the gap-discovery process so " + "operators understand where the loop closes" + ) + # The actionable channel: the kit repo's issues page. Pin the + # exact URL — a typo'd link is worse than no link (operator + # files an issue against a 404 and the signal is lost). + assert ( + "https://github.com/microsoft/" + "Employee-Self-Service-Agent-Developer-Kit/issues/new" + ) in r.remediation, ( + "AC3 loop-back must link to the kit repo issues page so " + "feedback reaches the team that owns WD-WF-CAT-001" + ) + # The three gap categories the loop-back exists to capture — + # if any are dropped, the loop closes on a narrower set of + # signals than AC3 requires. + assert "should ship OOTB" in r.remediation, ( + "AC3 must invite OOTB-promotion feedback (scenarios " + "customers routinely build custom that Microsoft should " + "ship in the extension pack)" + ) + assert "detection walker" in r.remediation, ( + "AC3 must invite detection-pattern feedback (a topic " + "wiring shape the walker missed)" + ) + assert "checklist above was insufficient" in r.remediation, ( + "AC3 must invite checklist-completeness feedback (the " + "4-item checklist itself can grow as new failure modes " + "are discovered)" + ) + + # ------------------------------------------------------------------ + # Pattern B (InvokeFlowAction → Workday-bound flow) + # ------------------------------------------------------------------ + + @responses.activate + def test_invoke_flow_action_workday_bound_emits_manual( + self, tmp_path: Path + ) -> None: + """Pattern B: a topic that calls a custom cloud flow bound to + `shared_workdaysoap` is ALWAYS unknown (Dataverse template + configs key by scenarioName; customer-built flow GUIDs don't + appear there) — surface MANUAL with the flow GUID + topic + location named verbatim.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + flow_id = "9f1b2c3d-aaaa-bbbb-cccc-111111111111" + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_invoke_flow_action( + agent_dir / "topics" / "custom-flow.mcs.yml", + flow_id=flow_id, + ) + _write_workflow( + agent_dir, + workflow_id=flow_id, + workflow_slug="ess-hr-workday-9f1b2c3d-xxxx", + api_name="shared_workdaysoap", + ) + # Dataverse returns some managed rows — but flow-bound refs + # are ALWAYS unknown regardless of catalog contents, so this + # test still gets MANUAL. + _register_template_configs_response(rows=[ + _template_config_row(name="msdyn_SomeOtherScenario", ismanaged=True), + ]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Manual" + assert "topics/custom-flow.mcs.yml" in r.result + assert flow_id in r.result + assert "InvokeFlowAction" in r.result + assert "shared_workdaysoap" in r.result + # `flow-bound, no scenarioName` is the literal label + # _format_unknown_scenarios uses for Pattern B refs — pin it + # so an operator scanning the output can tell at a glance + # this is a flow ref, not a scenarioName ref. + assert "flow-bound" in r.result + + def test_invoke_flow_action_non_workday_is_ignored( + self, tmp_path: Path + ) -> None: + """Conservative qualification (Pattern B): a flow bound to e.g. + `shared_servicenow` must NOT surface in this check — only + Workday-connected flows do. Otherwise this check would emit + false positives for every customer's ServiceNow integration.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + flow_id = "abcdef01-2222-3333-4444-555555555555" + agent_dir = tmp_path / "workspace" / "agents" / "ess-it" + _write_topic_invoke_flow_action( + agent_dir / "topics" / "create-ticket.mcs.yml", + flow_id=flow_id, + ) + _write_workflow( + agent_dir, + workflow_id=flow_id, + workflow_slug="ess-it-servicenow-aaaa", + api_name="shared_servicenow", + ) + + runner = _MinimalRunner() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + # Zero Workday refs found → SKIPPED with the "not wired in" + # message, NOT a MANUAL row mentioning the ServiceNow flow. + assert r.status == "Skipped" + assert "No Workday scenario references" in r.result + assert flow_id not in r.result, ( + "ServiceNow-bound flow leaked into Workday inventory output — " + "_is_workday_bound_workflow_json qualification regressed" + ) + + # ------------------------------------------------------------------ + # Bucketing (AGENTS.md principle #7) + # ------------------------------------------------------------------ + + @responses.activate + def test_multiple_unknowns_bucket_into_single_row( + self, tmp_path: Path + ) -> None: + """Principle #7: N unknown scenarios collapse to ONE MANUAL row + listing all N in `result`, with the de-duplicated 4-item + checklist as the SINGLE `remediation`. If a future refactor + emits one row per scenario, the operator sees the checklist + N times — exactly what bucketing exists to prevent.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + for n in range(3): + _write_topic_system_common_execution( + agent_dir / "topics" / f"custom-{n}.mcs.yml", + scenario_name=f"msdyn_HRCustomBucketTest_{n}", + ) + # Empty managed-row response → all 3 surface as unknown. + _register_template_configs_response(rows=[]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + # Exactly one row, NOT three. + cat_rows = [r for r in results if r.checkpoint_id == "WD-WF-CAT-001"] + assert len(cat_rows) == 1, ( + f"Bucketing regressed: got {len(cat_rows)} WD-WF-CAT-001 rows, " + f"expected 1 (per AGENTS.md principle #7)" + ) + + r = cat_rows[0] + assert r.status == "Manual" + # All three scenario names appear in `result`. + for n in range(3): + assert f"msdyn_HRCustomBucketTest_{n}" in r.result, ( + f"Scenario {n} dropped from bucketed result" + ) + # The checklist appears ONCE in remediation, not three times. + assert r.remediation.count("ISU account") == 1, ( + "Checklist appears multiple times in remediation — bucketing " + "should emit a single de-duplicated checklist" + ) + + # ------------------------------------------------------------------ + # Caching contract (the check + the cross-link trailer share state) + # ------------------------------------------------------------------ + + @responses.activate + def test_discovery_results_cached_on_runner(self, tmp_path: Path) -> None: + """`_check_custom_workflow_inventory` caches discovery on the + runner so the topic walk runs at most once per flightcheck. + Without this, `_check_workflows` (trailer) + the main check + would walk every topic twice. Pin that both caches populate + on first read.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "custom.mcs.yml", + scenario_name="msdyn_HRCustomCacheTest", + ) + # Empty managed-row response so the scenario surfaces as + # unknown (which is what the cache needs to capture). + _register_template_configs_response(rows=[]) + + runner = _RunnerWithDataverse() + # Pre-condition: cache attributes absent. + assert not hasattr(runner, "_workday_unknown_scenarios") + assert not hasattr(runner, "_workday_discovered_scenarios") + + _check_custom_workflow_inventory(runner) + + # Post-condition: both caches populated. + assert hasattr(runner, "_workday_unknown_scenarios") + assert hasattr(runner, "_workday_discovered_scenarios") + assert len(runner._workday_unknown_scenarios) == 1 + assert ( + runner._workday_unknown_scenarios[0]["scenarioName"] + == "msdyn_HRCustomCacheTest" + ) + + +# ───────────────────────────────────────────────────────────────────────── +# Cross-link trailer (WD-WF-CAT-LINK) — emitted from inside +# `_check_workflows`, satisfies AC2 ("Linked from Test-WorkdayWorkflows +# output"). Lives in its own class because it exercises the +# simplified-install gate of `_check_workflows`, not +# `_check_custom_workflow_inventory`. +# ───────────────────────────────────────────────────────────────────────── + + +class TestCrossLinkTrailer: + @pytest.fixture(autouse=True) + def _isolate_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # Mirror the isolation in test_workday_workflows_gate.py — when + # the simplified gate does NOT fire, `_check_workflows` reaches + # `_resolve_workday_metadata` which reads env vars + mcp.json. + # Strip ambient state so tests are deterministic. + monkeypatch.delenv("WORKDAY_BASE_URL", raising=False) + monkeypatch.delenv("WORKDAY_TENANT", raising=False) + monkeypatch.delenv("WORKDAY_TEST_EMPLOYEE_ID", raising=False) + monkeypatch.chdir(tmp_path) + + @responses.activate + def test_trailer_emitted_when_unknowns_exist(self, tmp_path: Path) -> None: + """AC2: when an unknown Workday workflow name is referenced in + customer topics, the SOAP-test output MUST emit a cross-link + row pointing at WD-WF-CAT-001 so an admin reading a clean + 17-row pass doesn't miss the manual row below it. + + Note: the trailer relies on `_get_unknown_workday_scenarios` + which in turn needs the live Dataverse catalog. Without + credentials the inventory check SKIPs and the trailer + self-suppresses (the main SKIPPED row is the single source of + truth). So this test wires up env_url/dv_token + a mocked + empty managed-row response so the unknown surfaces.""" + from flightcheck.checks.workday import _check_workflows + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "custom.mcs.yml", + scenario_name="msdyn_HRCustomTrailerTest", + ) + # Empty managed-row response → topic scenario surfaces as + # unknown → trailer fires. + _register_template_configs_response(rows=[]) + + @dataclass + class R: + env_url: str = FAKE_DATAVERSE_URL + dv_token: str = FAKE_TOKEN + config: dict = field(default_factory=dict) + + runner = R() + # Don't set _workday_package_flavor at all → gate doesn't fire + # (falls through to credential-missing path which emits the + # WD-WF-000 SKIP row, then the trailer code runs). + + results = _check_workflows(runner) + + # The SKIP row for credentials missing must exist (sanity). + wd_wf_000 = [r for r in results if r.checkpoint_id == "WD-WF-000"] + assert len(wd_wf_000) == 1 + assert wd_wf_000[0].status == "Skipped" + + # AC2: the trailer row exists and references the main check. + trailer = [r for r in results if r.checkpoint_id == "WD-WF-CAT-LINK"] + assert len(trailer) == 1, ( + f"Trailer WD-WF-CAT-LINK missing — AC2 regression. " + f"Got rows: {[r.checkpoint_id for r in results]}" + ) + t = trailer[0] + assert t.status == "Manual" + assert t.category == "Workday Workflows" + assert "WD-WF-CAT-001" in t.remediation, ( + "Trailer must cross-link to WD-WF-CAT-001 — operators read " + "the trailer text to find the full checklist" + ) + assert "1 Workday scenario reference" in t.result, ( + f"Trailer must name the count + the bucket — got {t.result!r}" + ) + + def test_trailer_absent_when_clean(self, tmp_path: Path) -> None: + """When there are zero unknown Workday refs, the trailer MUST + NOT fire — adding noise to a clean SOAP-test report would + defeat the purpose. Pin that the absence is intentional, not + an accident of test setup.""" + from flightcheck.checks.workday import _check_workflows + + # Empty workspace → discovery returns [] → no trailer. + (tmp_path / "workspace" / "agents").mkdir(parents=True, exist_ok=True) + + @dataclass + class R: + config: dict = field(default_factory=dict) + + runner = R() + results = _check_workflows(runner) + + trailer = [r for r in results if r.checkpoint_id == "WD-WF-CAT-LINK"] + assert len(trailer) == 0, ( + f"Trailer fired on clean workspace — should only fire when " + f"unknowns exist. Got: {[(r.checkpoint_id, r.result) for r in trailer]}" + ) + + +# ───────────────────────────────────────────────────────────────────────── +# Catalog source resolution (Dataverse-only) +# +# The OOTB catalog is resolved by `_get_workday_ootb_catalog(runner)`: +# a single Dataverse query against +# `msdyn_employeeselfservicetemplateconfigs` filtered by +# `ismanaged=true`. There is NO fallback — when the catalog cannot be +# resolved, AGENTS.md principle #1 requires SKIPPED (no token) or +# WARNING (query error) instead of a misleading PASSED. These tests +# pin every leg of that decision matrix so a future refactor that +# re-introduces a silent fallback fails CI. +# +# Per tests/AGENTS.md: Dataverse is the `documented` tier — no cassette +# required, mock is built from MS Learn-documented response shape via +# `tests.mocks.dataverse`. `require_validated_mock(dv)` at module top +# enforces this can never silently downgrade to placeholder. +# ───────────────────────────────────────────────────────────────────────── + + +@dataclass +class _RunnerWithDataverse: + """Runner that exposes env_url + dv_token so `_get_workday_ootb_catalog` + follows the Dataverse leg (default-skipped on `_MinimalRunner`).""" + env_url: str = FAKE_DATAVERSE_URL + dv_token: str = FAKE_TOKEN + config: dict[str, Any] = field(default_factory=dict) + + +def _template_config_row(*, name: str, ismanaged: bool) -> dict[str, Any]: + """Build one `msdyn_employeeselfservicetemplateconfigs` record matching + the `msdyn_name,ismanaged` select projection the production helper + requests. Shape sourced from MS Learn Web API reference (the + `tests.mocks.dataverse` module is `documented` tier — see its + `MOCK_STATUS`).""" + return { + "@odata.etag": 'W/"1"', + "msdyn_name": name, + "ismanaged": ismanaged, + } + + +def _register_template_configs_response( + *, + base_url: str = FAKE_DATAVERSE_URL, + rows: list[dict[str, Any]] | None = None, + status: int = 200, +) -> None: + """Register a `responses` mock for the template-configs query. URL is + path-only (no query string) so it matches regardless of the exact + $select / $filter / paging params the production code builds.""" + payload = dv.collection(rows or []) if status == 200 else {"error": {"message": "mock failure"}} + responses.add( + method="GET", + url=f"{base_url}/api/data/v9.2/msdyn_employeeselfservicetemplateconfigs", + json=payload, + status=status, + ) + + +class TestCatalogSource: + """Pin Dataverse-only semantics of `_get_workday_ootb_catalog`. + The four tests below cover every leg of the resolver's decision + matrix: (a) Dataverse-success with managed rows → catalog used, + (b) Dataverse-success with NO managed rows → check still runs (no + implicit PASS) and falls through to MANUAL because the + topic-referenced scenario is not in the managed set, (c) no + Dataverse token → SKIPPED (cannot validate without the catalog, + per AGENTS.md principle #1), (d) Dataverse query errors → + WARNING with the error message surfaced verbatim (per principle + #3 — fail loudly on API errors).""" + + @pytest.fixture(autouse=True) + def _isolate_env( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.chdir(tmp_path) + + @responses.activate + def test_dataverse_managed_rows_used_as_catalog( + self, tmp_path: Path + ) -> None: + """When Dataverse is reachable and returns a managed + (ismanaged=true) row whose `msdyn_name` matches a topic's + scenarioName, the scenario is treated as OOTB and the check + PASSES. The result text MUST name Dataverse + ismanaged=true + as the source so an operator inspecting a green report knows + the catalog was tenant-accurate.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + custom_scenario = "msdyn_HRWorkdayTenantSpecificScenario_XYZ" + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "tenant-scenario.mcs.yml", + scenario_name=custom_scenario, + ) + + # Tenant has this scenario installed as a managed template + # config — i.e. it ships in the customer's Workday extension + # pack. Without the live-Dataverse leg, this would surface + # as MANUAL (no JSON seed exists to fall back to). + _register_template_configs_response(rows=[ + _template_config_row(name=custom_scenario, ismanaged=True), + ]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Passed", ( + f"Dataverse-managed scenario must surface as PASSED — " + f"the Dataverse-resolution leg regressed. " + f"Got {r.status!r}: {r.result!r}" + ) + # Source attribution pinned per principle #8 (`result` = + # observed). Operators must know the catalog was tenant-live. + assert "Dataverse" in r.result + assert "ismanaged=true" in r.result + # Cache populated with status "ok" for re-reads by the trailer. + assert runner._workday_ootb_catalog_cache[1] == "ok" + assert custom_scenario in runner._workday_ootb_catalog_cache[0] + + @responses.activate + def test_dataverse_unmanaged_rows_excluded_from_catalog( + self, tmp_path: Path + ) -> None: + """`ismanaged=false` rows are customer-added template configs, + NOT shipped by the extension pack. They must NOT count as OOTB + — otherwise a customer who added a custom scenario via /create + would see it falsely treated as "validated by Microsoft" and + skip the MANUAL checklist that exists to catch payload / + auth / test-prompt gaps. Pin the filter explicitly.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + custom_scenario = "msdyn_HRWorkdayCustomerAuthored_ABC" + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "customer-auth.mcs.yml", + scenario_name=custom_scenario, + ) + + # Same name exists in Dataverse but ismanaged=false → customer- + # authored, not OOTB. Must NOT short-circuit the MANUAL row. + _register_template_configs_response(rows=[ + _template_config_row(name=custom_scenario, ismanaged=False), + ]) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Manual", ( + f"ismanaged=false rows must NOT count as OOTB — " + f"the customer-added scenario should surface as MANUAL " + f"for review. Got {r.status!r}: result={r.result!r}" + ) + assert custom_scenario in r.result + assert "ISU account" in r.remediation # full checklist still emitted + # Cache: Dataverse WAS reached (status "ok"), it just returned + # no managed rows. The empty catalog correctly excludes the + # unmanaged scenario. + assert runner._workday_ootb_catalog_cache[1] == "ok" + assert custom_scenario not in runner._workday_ootb_catalog_cache[0] + + @responses.activate + def test_dataverse_query_failure_emits_warning( + self, tmp_path: Path + ) -> None: + """When the Dataverse query errors (500, network issue, expired + token, etc.), the check MUST emit a WARNING that surfaces the + error rather than silently passing — per AGENTS.md principle + #3 ("fail loudly on API errors, never silently swallow them + as PASS"). There is no JSON fallback: a tenant-accurate + catalog is the only valid source of truth, and a query error + means we don't have one.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "some-scenario.mcs.yml", + scenario_name="msdyn_AnyScenario_DoesntMatter", + ) + + # Dataverse query 500s — must surface as WARNING, not PASSED + # and not MANUAL (we cannot tell if it's custom). + _register_template_configs_response(status=500) + + runner = _RunnerWithDataverse() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Warning", ( + f"Dataverse query failure must emit WARNING per " + f"AGENTS.md principle #3, got {r.status!r}: {r.result!r}" + ) + # Result must name the failure mode so the operator can + # diagnose. Don't pin the exact error string (it comes from + # auth.query_all and may evolve) but pin the structural cues. + assert "Dataverse" in r.result + assert "msdyn_employeeselfservicetemplateconfigs" in r.result + assert "failed" in r.result.lower() or "error" in r.result.lower() + # Remediation must direct the operator to fix the underlying + # Dataverse problem (not work around it). + assert "Dataverse" in r.remediation + # Cache populated with the error status so re-reads don't + # re-query. + catalog, status_code = runner._workday_ootb_catalog_cache + assert catalog is None + assert status_code.startswith("query_error:") + # Unknown cache emptied so the trailer self-suppresses (the + # WARNING row is the single source of truth for this state). + assert runner._workday_unknown_scenarios == [] + + def test_no_dataverse_token_skips(self, tmp_path: Path) -> None: + """The CI / offline / no-auth runner has no env_url + dv_token. + The resolver MUST short-circuit to SKIPPED without attempting + any HTTP call (so this test doesn't even need + @responses.activate — any HTTP attempt would surface as a + connection error against an unrouted host). Per AGENTS.md + principle #1: without the catalog we cannot validate, so we + must not return PASSED.""" + from flightcheck.checks.workday import _check_custom_workflow_inventory + + agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" + _write_topic_system_common_execution( + agent_dir / "topics" / "offline.mcs.yml", + scenario_name="msdyn_AnyScenario_DoesntMatter", + ) + + # `_MinimalRunner` has no env_url / dv_token — Dataverse path + # must short-circuit before any HTTP call. + runner = _MinimalRunner() + results = _check_custom_workflow_inventory(runner) + + r = _result_by_id(results, "WD-WF-CAT-001") + assert r.status == "Skipped", ( + f"No-Dataverse-token runner must SKIP per AGENTS.md " + f"principle #1, got {r.status!r}: {r.result!r}" + ) + # Result must explain WHY this skipped (so the operator + # doesn't conflate it with the "Workday not wired up" SKIP). + assert "Dataverse" in r.result + assert "credentials" in r.result.lower() or "token" in r.result.lower() + # Remediation directs the operator at /setup so credentials get + # cached on the runner. + assert "/setup" in r.remediation or "Dataverse" in r.remediation + # Cache: catalog None with "no_token" status code. + catalog, status_code = runner._workday_ootb_catalog_cache + assert catalog is None + assert status_code == "no_token" + # Unknown cache emptied so the trailer self-suppresses. + assert runner._workday_unknown_scenarios == []