Merge branch 'main' into feat-bigquery-reservations

Author: scottypate

.circleci/continue_config.yml

@@ -93,9 +93,13 @@ jobs:
       - run:
           name: Run linters and code style checks
           command: make py-style
-      # - run:
-      #     name: Exercise the benchmarks
-      #     command: make benchmark-ci
+      - unless:
+          condition:
+            equal: ["3.9", << parameters.python_version >>]
+          steps:
+            - run:
+                name: Exercise the benchmarks
+                command: make benchmark-ci
       - run:
           name: Run cicd tests
           command: make cicd-test
@@ -305,7 +309,7 @@ workflows:
           matrix:
             parameters:
               engine:
-                - snowflake
+                #- snowflake
                 - databricks
                 - redshift
                 - bigquery

.github/workflows/pr.yaml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v6
         with:
           node-version: '22'
       - uses: pnpm/action-setup@v4
@@ -32,7 +32,7 @@ jobs:
       labels: [ubuntu-2204-8]
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v6
         with:
           node-version: '22'
       - uses: pnpm/action-setup@v4
@@ -41,11 +41,11 @@ jobs:
       - name: Install dependencies
         run: pnpm install
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.12'
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
       - name: Install python dependencies
         run: |
           python -m venv .venv
@@ -62,7 +62,7 @@ jobs:
         run: |
           source ../../.venv/bin/activate
           pnpm run test:e2e
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         if: ${{ !cancelled() }}
         with:
           name: playwright-report
@@ -77,11 +77,11 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
       - name: Install SQLMesh dev dependencies
         run: |
           uv venv .venv

.github/workflows/private-repo-test.yaml

@@ -22,13 +22,13 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.12'
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
       - name: Set up Node.js for UI build
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '20'
       - name: Install pnpm

.github/workflows/release_extension.yaml

@@ -28,7 +28,7 @@ jobs:
           fi
           echo "Version format is valid: $version"
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '20'
       - name: Install pnpm

.github/workflows/release_shared_js.yaml

@@ -31,7 +31,7 @@ jobs:
           fi
           echo "Version format is valid: $version"
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '20'
           registry-url: 'https://registry.npmjs.org'

docs/guides/model_selection.md

@@ -242,8 +242,9 @@ Models:
 #### Select with git changes
 
 The git-based selector allows you to select models whose files have changed compared to a target branch (default: main). This includes:
+
 - Untracked files (new files not in git)
-- Uncommitted changes in working directory
+- Uncommitted changes in working directory (both staged and unstaged)
 - Committed changes different from the target branch
 
 For example:

docs/integrations/engines/duckdb.md

@@ -81,8 +81,9 @@ SQLMesh will place models with the explicit catalog "ephemeral", such as `epheme
               data_path: data/ducklake
               encrypted: True
               data_inlining_row_limit: 10
+              metadata_schema: main
     ```
-    
+
 === "Python"
 
     ```python linenums="1"
@@ -106,6 +107,7 @@ SQLMesh will place models with the explicit catalog "ephemeral", such as `epheme
                             data_path="data/ducklake",
                             encrypted=True,
                             data_inlining_row_limit=10,
+                            metadata_schema="main",
                         ),
                     }
                 )
@@ -114,6 +116,14 @@ SQLMesh will place models with the explicit catalog "ephemeral", such as `epheme
     )
     ```
 
+**DuckLake Configuration Options:**
+
+- `path`: Path to the DuckLake catalog file
+- `data_path`: Path where DuckLake data files are stored
+- `encrypted`: Whether to enable encryption for the catalog (default: `False`)
+- `data_inlining_row_limit`: Maximum number of rows to inline in the catalog (default: `0`)
+- `metadata_schema`: The schema in the catalog server in which to store the DuckLake metadata tables (default: `main`)
+
 #### Other Connection Catalogs Example
 
 Catalogs can also be defined to connect to anything that [DuckDB can be attached to](https://duckdb.org/docs/sql/statements/attach.html).

docs/reference/model_configuration.md

@@ -282,7 +282,7 @@ Configuration options for [`SCD_TYPE_2` models](../concepts/models/model_kinds.m
 | `unique_key`              | The model column(s) containing each row's unique key                                                                                                                                        | array[str] |    Y     |
 | `valid_from_name`         | The model column containing each row's valid from date. (Default: `valid_from`)                                                                                                             |    str     |    N     |
 | `valid_to_name`           | The model column containing each row's valid to date. (Default: `valid_to`)                                                                                                                 |    str     |    N     |
-| `invalidate_hard_deletes` | If set to true, when a record is missing from the source table it will be marked as invalid - see [here](../concepts/models/model_kinds.md#deletes) for more information. (Default: `True`) |    bool    |    N     |
+| `invalidate_hard_deletes` | If set to true, when a record is missing from the source table it will be marked as invalid - see [here](../concepts/models/model_kinds.md#deletes) for more information. (Default: `False`) |    bool    |    N     |
 
 ##### SCD Type 2 By Time
 

examples/sushi_dbt/models/schema.yml

@@ -42,8 +42,6 @@ models:
     columns:
       - name: waiter_id
         description: Waiter id
-        tests:
-          - not_null
       - name: ds
         description: Date
   - name: waiter_as_customer_by_day

sqlmesh/core/config/connection.py

@@ -17,6 +17,7 @@
 from packaging import version
 from sqlglot import exp
 from sqlglot.helper import subclasses
+from sqlglot.errors import ParseError
 
 from sqlmesh.core import engine_adapter
 from sqlmesh.core.config.base import BaseConfig
@@ -238,6 +239,7 @@ class DuckDBAttachOptions(BaseConfig):
     data_path: t.Optional[str] = None
     encrypted: bool = False
     data_inlining_row_limit: t.Optional[int] = None
+    metadata_schema: t.Optional[str] = None
 
     def to_sql(self, alias: str) -> str:
         options = []
@@ -259,6 +261,8 @@ def to_sql(self, alias: str) -> str:
                 options.append("ENCRYPTED")
             if self.data_inlining_row_limit is not None:
                 options.append(f"DATA_INLINING_ROW_LIMIT {self.data_inlining_row_limit}")
+            if self.metadata_schema is not None:
+                options.append(f"METADATA_SCHEMA '{self.metadata_schema}'")
 
         options_sql = f" ({', '.join(options)})" if options else ""
         alias_sql = ""
@@ -1889,6 +1893,7 @@ class TrinoConnectionConfig(ConnectionConfig):
 
     # SQLMesh options
     schema_location_mapping: t.Optional[dict[re.Pattern, str]] = None
+    timestamp_mapping: t.Optional[dict[exp.DataType, exp.DataType]] = None
     concurrent_tasks: int = 4
     register_comments: bool = True
     pre_ping: t.Literal[False] = False
@@ -1913,6 +1918,34 @@ def _validate_regex_keys(
                 )
         return compiled
 
+    @field_validator("timestamp_mapping", mode="before")
+    @classmethod
+    def _validate_timestamp_mapping(
+        cls, value: t.Optional[dict[str, str]]
+    ) -> t.Optional[dict[exp.DataType, exp.DataType]]:
+        if value is None:
+            return value
+
+        result: dict[exp.DataType, exp.DataType] = {}
+        for source_type, target_type in value.items():
+            try:
+                source_datatype = exp.DataType.build(source_type)
+            except ParseError:
+                raise ConfigError(
+                    f"Invalid SQL type string in timestamp_mapping: "
+                    f"'{source_type}' is not a valid SQL data type."
+                )
+            try:
+                target_datatype = exp.DataType.build(target_type)
+            except ParseError:
+                raise ConfigError(
+                    f"Invalid SQL type string in timestamp_mapping: "
+                    f"'{target_type}' is not a valid SQL data type."
+                )
+            result[source_datatype] = target_datatype
+
+        return result
+
     @model_validator(mode="after")
     def _root_validator(self) -> Self:
         port = self.port
@@ -2015,7 +2048,10 @@ def _static_connection_kwargs(self) -> t.Dict[str, t.Any]:
 
     @property
     def _extra_engine_config(self) -> t.Dict[str, t.Any]:
-        return {"schema_location_mapping": self.schema_location_mapping}
+        return {
+            "schema_location_mapping": self.schema_location_mapping,
+            "timestamp_mapping": self.timestamp_mapping,
+        }
 
 
 class ClickhouseConnectionConfig(ConnectionConfig):