Fix(motherduck): Attach multiple catalogs in MotherDuck (#4178)

Author: themisvaltinos

docs/integrations/engines/motherduck.md

@@ -1,32 +1,32 @@
 # MotherDuck
 
-This page provides information about how to use SQLMesh with MotherDuck. 
+This page provides information about how to use SQLMesh with MotherDuck.
 
 It begins with a [Connection Quickstart](#connection-quickstart) that demonstrates how to connect to MotherDuck, or you can skip directly to information about using MotherDuck with the built-in scheduler.
 
 ## Connection quickstart
 
 Connecting to cloud warehouses involves a few steps, so this connection quickstart provides the info you need to get up and running with MotherDuck.
 
-It demonstrates connecting to MotherDuck with the `duckdb` library bundled with SQLMesh. 
+It demonstrates connecting to MotherDuck with the `duckdb` library bundled with SQLMesh.
 
 MotherDuck provides a single way to authorize a connection. This quickstart demonstrates authenticating with a token.
 
-!!! tip  
+!!! tip
     This quick start assumes you are familiar with basic SQLMesh commands and functionality.
 
-    If you’re not familiar, work through the [SQLMesh Quickstart](../../quick_start.md) before continuing.  
+    If you’re not familiar, work through the [SQLMesh Quickstart](../../quick_start.md) before continuing.
 
 ### Prerequisites
 
 Before working through this quickstart guide, ensure that:
 
-1. You have a motherduck account and an access token.  
-2. Your computer has SQLMesh installed with the DuckDB extra available.  
-   1. Install from command line with the command `pip install “sqlmesh[duckdb]”`  
-3. You have initialized a SQLMesh example project on your computer  
-   1. Open a command line interface and navigate to the directory where the project files should go.  
-   2. Initialize the project with the command `sqlmesh init motherduck`
+1. You have a motherduck account and an access token.
+2. Your computer has SQLMesh installed with the DuckDB extra available.
+   1. Install from command line with the command `pip install “sqlmesh[duckdb]”`
+3. You have initialized a SQLMesh example project on your computer
+   1. Open a command line interface and navigate to the directory where the project files should go.
+   2. Initialize the project with the command `sqlmesh init duckdb`, since `duckdb` is the dialect.
 
 #### Access control permissions
 
@@ -38,30 +38,32 @@ We now have what is required to configure SQLMesh’s connection to MotherDuck.
 
 We start the configuration by adding a gateway named `motherduck` to our example project’s config.yaml file and making it our `default gateway`, as well as adding our token, persistent, and ephemeral catalogs.
 
-```yaml  
-gateways:  
-    motherduck:  
-        connection:   
-            type: motherduck  
-            catalogs:  
-                persistent: ‘md:’  
-                ephemeral: ‘:memory:’  
-            token: \<your\_token\>
-
-default\_gateway: motherduck  
+```yaml
+gateways:
+  motherduck:
+    connection:
+      type: motherduck
+        catalogs:
+          persistent: "md:"
+          ephemeral: ":memory:"
+      token: <your_token>
+
+default_gateway: motherduck
 ```
 
+Catalogs can be defined to connect to anything that [DuckDB can be attached to](./duckdb.md#other-connection-catalogs-example).
+
 !!! warning
     Best practice for storing secrets like tokens is placing them in [environment variables that the configuration file loads dynamically](../../guides/configuration.md#environment-variables). For simplicity, this guide instead places the value directly in the configuration file.
 
     This code demonstrates how to use the environment variable `MOTHERDUCK_TOKEN` for the configuration's `token` parameter:
 
     ```yaml linenums="1" hl_lines="5"
-    gateways:  
-        motherduck:  
-            connection:   
-                type: motherduck  
-            token: {{ env_var('MOTHERDUCK_TOKEN') }}
+    gateways:
+      motherduck:
+        connection:
+          type: motherduck
+          token: {{ env_var('MOTHERDUCK_TOKEN') }}
     ```
 
 ### Check connection

sqlmesh/core/config/connection.py

@@ -150,16 +150,12 @@ def to_sql(self, alias: str) -> str:
         options_sql = f" ({', '.join(options)})" if options else ""
         alias_sql = ""
         # TODO: Add support for Postgres schema. Currently adding it blocks access to the information_schema
-        if self.type == "motherduck":
-            # MotherDuck does not support aliasing
-            md_db = self.path.replace("md:", "")
-            if md_db != alias.replace('"', ""):
-                raise ConfigError(
-                    f"MotherDuck does not support assigning an alias different from the database name {md_db}."
-                )
-        else:
-            alias_sql += f" AS {alias}"
-        return f"ATTACH '{self.path}'{alias_sql}{options_sql}"
+
+        # MotherDuck does not support aliasing
+        alias_sql = (
+            f" AS {alias}" if not (self.type == "motherduck" or self.path.startswith("md:")) else ""
+        )
+        return f"ATTACH IF NOT EXISTS '{self.path}'{alias_sql}{options_sql}"
 
 
 class BaseDuckDBConnectionConfig(ConnectionConfig):
@@ -264,7 +260,7 @@ def init(cursor: duckdb.DuckDBPyConnection) -> None:
                     if isinstance(path_options, DuckDBAttachOptions):
                         query = path_options.to_sql(alias)
                     else:
-                        query = f"ATTACH '{path_options}'"
+                        query = f"ATTACH IF NOT EXISTS '{path_options}'"
                         if not path_options.startswith("md:"):
                             query += f" AS {alias}"
                     cursor.execute(query)

tests/core/test_connection_config.py

@@ -597,12 +597,47 @@ def test_duckdb_attach_options():
 
     assert (
         options.to_sql(alias="db")
-        == "ATTACH 'dbname=postgres user=postgres host=127.0.0.1' AS db (TYPE POSTGRES, READ_ONLY)"
+        == "ATTACH IF NOT EXISTS 'dbname=postgres user=postgres host=127.0.0.1' AS db (TYPE POSTGRES, READ_ONLY)"
     )
 
     options = DuckDBAttachOptions(type="duckdb", path="test.db", read_only=False)
 
-    assert options.to_sql(alias="db") == "ATTACH 'test.db' AS db"
+    assert options.to_sql(alias="db") == "ATTACH IF NOT EXISTS 'test.db' AS db"
+
+
+def test_motherduck_attach_catalog(make_config):
+    config = make_config(
+        type="motherduck",
+        catalogs={
+            "test1": "md:test1",
+            "test2": DuckDBAttachOptions(
+                type="motherduck",
+                path="md:test2",
+            ),
+        },
+    )
+    assert isinstance(config, MotherDuckConnectionConfig)
+    assert config.get_catalog() == "test1"
+
+    assert config.catalogs.get("test2").read_only is False
+    assert config.catalogs.get("test2").path == "md:test2"
+    assert not config.is_recommended_for_state_sync
+
+
+def test_motherduck_attach_options():
+    options = DuckDBAttachOptions(
+        type="postgres", path="dbname=postgres user=postgres host=127.0.0.1", read_only=True
+    )
+
+    assert (
+        options.to_sql(alias="db")
+        == "ATTACH IF NOT EXISTS 'dbname=postgres user=postgres host=127.0.0.1' AS db (TYPE POSTGRES, READ_ONLY)"
+    )
+
+    options = DuckDBAttachOptions(type="motherduck", path="md:test.db", read_only=False)
+
+    # Here the alias should be ignored compared to duckdb
+    assert options.to_sql(alias="db") == "ATTACH IF NOT EXISTS 'md:test.db'"
 
 
 def test_duckdb_multithreaded_connection_factory(make_config):