diff --git a/.github/workflows/control-plane.yml b/.github/workflows/control-plane.yml
index 93e9ffb..f7bdab5 100644
--- a/.github/workflows/control-plane.yml
+++ b/.github/workflows/control-plane.yml
@@ -98,7 +98,7 @@ jobs:
 
       - name: Resolve GitHub App token
         id: app-token
-        if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'issues' }}
+        if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.mode != 'alert-test') || github.event_name == 'issues' }}
         env:
           AIO_FLEET_APP_ID: ${{ secrets.AIO_FLEET_APP_ID }}
           AIO_FLEET_APP_INSTALLATION_ID: ${{ secrets.AIO_FLEET_APP_INSTALLATION_ID }}
diff --git a/tests/test_control_plane_workflow.py b/tests/test_control_plane_workflow.py
index 0bab6fa..2b2f664 100644
--- a/tests/test_control_plane_workflow.py
+++ b/tests/test_control_plane_workflow.py
@@ -81,9 +81,11 @@ def test_alert_test_mode_uses_alert_webhook_secret_only() -> None:
     workflow = yaml.safe_load(WORKFLOW.read_text())
     on_config = workflow.get("on", workflow.get(True))
     mode = on_config["workflow_dispatch"]["inputs"]["mode"]
+    app_token = _step(workflow["jobs"]["control-plane"], "Resolve GitHub App token")
     alert_test = _step(workflow["jobs"]["control-plane"], "Test alert webhook")
 
     assert "alert-test" in mode["options"]  # nosec B101
+    assert "inputs.mode != 'alert-test'" in app_token["if"]  # nosec B101
     assert (  # nosec B101
         alert_test["if"]
         == "${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'alert-test' }}"