fix: include stack trace bot output (#1592)

Author: eakmanrq

sqlmesh/integrations/github/cicd/controller.py

@@ -6,6 +6,7 @@
 import os
 import pathlib
 import re
+import traceback
 import typing as t
 import unittest
 from enum import Enum
@@ -23,6 +24,7 @@
 from sqlmesh.core.plan import LoadedSnapshotIntervals, Plan
 from sqlmesh.core.user import User
 from sqlmesh.integrations.github.cicd.config import GithubCICDBotConfig
+from sqlmesh.utils.concurrency import NodeExecutionFailedError
 from sqlmesh.utils.date import now
 from sqlmesh.utils.errors import (
     CICDBotError,
@@ -727,9 +729,14 @@ def conclusion_handler(
                     skip_reason = "Unit Test(s) Failed so skipping PR creation"
                 else:
                     skip_reason = "A prior stage failed resulting in skipping PR creation."
+
+                if isinstance(exception, NodeExecutionFailedError):
+                    failure_msg = f"Node `{exception.node.name}` failed to apply.\n\n**Stack Trace:**\n```\n{traceback.format_exc()}\n```"
+                else:
+                    failure_msg = f"This is an unexpected error.\n\n**Exception:**\n{exception}"
                 conclusion_to_summary = {
                     GithubCheckConclusion.SKIPPED: f":next_track_button: Skipped creating or updating PR Environment `{self.pr_environment_name}`. {skip_reason}",
-                    GithubCheckConclusion.FAILURE: f":x: Failed to create or update PR Environment `{self.pr_environment_name}`. This is an unexpected error.\n**Exception:**\n{exception}",
+                    GithubCheckConclusion.FAILURE: f":x: Failed to create or update PR Environment `{self.pr_environment_name}`.\n{failure_msg}",
                     GithubCheckConclusion.CANCELLED: f":stop_sign: Cancelled creating or updating PR Environment `{self.pr_environment_name}`",
                     GithubCheckConclusion.ACTION_REQUIRED: f":warning: Action Required to create or update PR Environment `{self.pr_environment_name}`. There are likely uncateogrized changes. Run `plan` locally to apply these changes. If you want the bot to automatically categorize changes, then check documentation (https://sqlmesh.readthedocs.io/en/stable/integrations/github/) for more information.",
                 }

tests/integrations/github/cicd/test_integration.py

@@ -1225,3 +1225,145 @@ def test_deploy_comment_pre_categorized(
 
 """
     )
+
+
+@freeze_time("2023-01-01 15:00:00")
+def test_error_msg_when_applying_plan_with_bug(
+    github_client,
+    make_controller,
+    make_mock_check_run,
+    make_mock_issue_comment,
+    make_pull_request_review,
+    mocker: MockerFixture,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """
+    PR with auto-categorization but has a mistake in the model so apply fails
+
+    Scenario:
+    - PR is not merged
+    - PR has been approved by a required reviewer
+    - Tests passed
+    - PR Merge Method defined
+    - Delete environment is disabled
+    - Bugged change made in PR with auto-categorization
+    """
+    mock_repo = github_client.get_repo()
+    mock_repo.create_check_run = mocker.MagicMock(
+        side_effect=lambda **kwargs: make_mock_check_run(**kwargs)
+    )
+
+    created_comments: t.List[MockIssueComment] = []
+    mock_issue = mock_repo.get_issue()
+    mock_issue.create_comment = mocker.MagicMock(
+        side_effect=lambda comment: make_mock_issue_comment(
+            comment=comment, created_comments=created_comments
+        )
+    )
+    mock_issue.get_comments = mocker.MagicMock(side_effect=lambda: created_comments)
+
+    mock_pull_request = mock_repo.get_pull()
+    mock_pull_request.get_reviews = mocker.MagicMock(
+        side_effect=lambda: [make_pull_request_review(username="test_github", state="APPROVED")]
+    )
+    mock_pull_request.merged = False
+    mock_pull_request.merge = mocker.MagicMock()
+
+    controller = make_controller(
+        "tests/fixtures/github/pull_request_synchronized.json",
+        github_client,
+        bot_config=GithubCICDBotConfig(
+            merge_method=MergeMethod.MERGE,
+            auto_categorize_changes=CategorizerConfig.all_full(),
+            invalidate_environment_after_deploy=False,
+        ),
+        mock_out_context=False,
+    )
+    controller._context.plan("prod", no_prompts=True, auto_apply=True)
+    controller._context.users = [
+        User(username="test", github_username="test_github", roles=[UserRole.REQUIRED_APPROVER])
+    ]
+    # Make an error by adding a column that doesn't exist
+    model = controller._context.get_model("sushi.waiter_revenue_by_day").copy()
+    model.query.expressions.append(exp.alias_("non_existing_col", "new_col"))
+    controller._context.upsert_model(model)
+
+    command._run_all(controller)
+
+    assert "SQLMesh - Run Unit Tests" in controller._check_run_mapping
+    test_checks_runs = controller._check_run_mapping["SQLMesh - Run Unit Tests"].all_kwargs
+    assert len(test_checks_runs) == 3
+    assert GithubCheckStatus(test_checks_runs[0]["status"]).is_queued
+    assert GithubCheckStatus(test_checks_runs[1]["status"]).is_in_progress
+    assert GithubCheckStatus(test_checks_runs[2]["status"]).is_completed
+    assert GithubCheckConclusion(test_checks_runs[2]["conclusion"]).is_success
+    assert test_checks_runs[2]["output"]["title"] == "Tests Passed"
+    assert (
+        test_checks_runs[2]["output"]["summary"].strip()
+        == "**Successfully Ran `2` Tests Against `duckdb`**"
+    )
+
+    assert "SQLMesh - PR Environment Synced" in controller._check_run_mapping
+    pr_checks_runs = controller._check_run_mapping["SQLMesh - PR Environment Synced"].all_kwargs
+    assert len(pr_checks_runs) == 3
+    assert GithubCheckStatus(pr_checks_runs[0]["status"]).is_queued
+    assert GithubCheckStatus(pr_checks_runs[1]["status"]).is_in_progress
+    assert GithubCheckStatus(pr_checks_runs[2]["status"]).is_completed
+    assert GithubCheckConclusion(pr_checks_runs[2]["conclusion"]).is_failure
+    assert pr_checks_runs[2]["output"]["title"] == "PR Virtual Data Environment: hello_world_2"
+    assert (
+        'Binder Error: Referenced column "non_existing_col" not found in FROM clause!'
+        in pr_checks_runs[2]["output"]["summary"]
+    )
+
+    assert "SQLMesh - Prod Plan Preview" in controller._check_run_mapping
+    prod_plan_preview_checks_runs = controller._check_run_mapping[
+        "SQLMesh - Prod Plan Preview"
+    ].all_kwargs
+    assert len(prod_plan_preview_checks_runs) == 2
+    assert GithubCheckStatus(prod_plan_preview_checks_runs[0]["status"]).is_queued
+    assert GithubCheckStatus(prod_plan_preview_checks_runs[1]["status"]).is_completed
+    assert GithubCheckConclusion(prod_plan_preview_checks_runs[1]["conclusion"]).is_skipped
+    assert (
+        prod_plan_preview_checks_runs[1]["output"]["title"]
+        == "Skipped generating prod plan preview since PR was not synchronized"
+    )
+    assert (
+        prod_plan_preview_checks_runs[1]["output"]["summary"]
+        == "Skipped generating prod plan preview since PR was not synchronized"
+    )
+
+    assert "SQLMesh - Prod Environment Synced" in controller._check_run_mapping
+    prod_checks_runs = controller._check_run_mapping["SQLMesh - Prod Environment Synced"].all_kwargs
+    assert len(prod_checks_runs) == 2
+    assert GithubCheckStatus(prod_checks_runs[0]["status"]).is_queued
+    assert GithubCheckStatus(prod_checks_runs[1]["status"]).is_completed
+    assert GithubCheckConclusion(prod_checks_runs[1]["conclusion"]).is_skipped
+    skip_reason = "Skipped Deploying to Production because the PR environment was not updated"
+    assert prod_checks_runs[1]["output"]["title"] == skip_reason
+    assert prod_checks_runs[1]["output"]["summary"] == skip_reason
+
+    assert "SQLMesh - Has Required Approval" in controller._check_run_mapping
+    approval_checks_runs = controller._check_run_mapping[
+        "SQLMesh - Has Required Approval"
+    ].all_kwargs
+    assert len(approval_checks_runs) == 3
+    assert GithubCheckStatus(approval_checks_runs[0]["status"]).is_queued
+    assert GithubCheckStatus(approval_checks_runs[1]["status"]).is_in_progress
+    assert GithubCheckStatus(approval_checks_runs[2]["status"]).is_completed
+    assert GithubCheckConclusion(approval_checks_runs[2]["conclusion"]).is_success
+    assert (
+        approval_checks_runs[2]["output"]["title"]
+        == "Obtained approval from required approvers: test_github"
+    )
+    assert (
+        approval_checks_runs[2]["output"]["summary"]
+        == """**List of possible required approvers:**
+- `test_github`
+"""
+    )
+
+    assert len(get_environment_objects(controller, "hello_world_2")) == 0
+    assert not mock_pull_request.merge.called
+
+    assert len(created_comments) == 0