Merge branch 'main' into ignore-view-cluster-by

Author: crericha

.circleci/manage-test-db.sh

@@ -80,7 +80,11 @@ redshift_down() {
     EXIT_CODE=1
     ATTEMPTS=0
     while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
-        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'"
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges 
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+        
+        # perform drop
         redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
         if [ $EXIT_CODE -ne 0 ]; then
             echo "Unable to drop database; retrying..."

.github/workflows/pr.yaml

@@ -58,7 +58,7 @@ jobs:
         run: pnpm exec playwright install
       - name: Run e2e tests
         working-directory: ./vscode/extension
-        timeout-minutes: 90
+        timeout-minutes: 30
         run: |
           source ../../.venv/bin/activate
           pnpm run test:e2e
@@ -92,6 +92,44 @@ jobs:
         # of DuckDB require a version of DuckDB we no longer support
         run: |
           source .venv/bin/activate
+
+          # Remove semantic_models and metrics sections for DBT versions < 1.6.0
+          # Using explicit list to avoid version comparison issues
+          if [[ "${{ matrix.dbt-version }}" == "1.3" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.4" ]] || \
+             [[ "${{ matrix.dbt-version }}" == "1.5" ]]; then
+            
+            echo "DBT version is ${{ matrix.dbt-version }} (< 1.6.0), removing semantic_models and metrics sections..."
+            
+            schema_file="tests/fixtures/dbt/sushi_test/models/schema.yml"
+            if [[ -f "$schema_file" ]]; then
+              echo "Modifying $schema_file..."
+              
+              # Create a temporary file
+              temp_file=$(mktemp)
+              
+              # Use awk to remove semantic_models and metrics sections
+              awk '
+                /^semantic_models:/ { in_semantic=1; next }
+                /^metrics:/ { in_metrics=1; next }
+                /^[^ ]/ && (in_semantic || in_metrics) { 
+                  in_semantic=0; 
+                  in_metrics=0 
+                }
+                !in_semantic && !in_metrics { print }
+              ' "$schema_file" > "$temp_file"
+              
+              # Move the temp file back
+              mv "$temp_file" "$schema_file"
+              
+              echo "Successfully removed semantic_models and metrics sections"
+            else
+              echo "Schema file not found at $schema_file, skipping..."
+            fi
+          else
+            echo "DBT version is ${{ matrix.dbt-version }} (>= 1.6.0), keeping semantic_models and metrics sections"
+          fi
+
           make dbt-fast-test
       - name: Test SQLMesh info in sushi_dbt
         working-directory: ./examples/sushi_dbt
@@ -104,4 +142,5 @@ jobs:
           else
             echo "DBT version is ${{ matrix.dbt-version }} (>= 1.5.0), keeping version parameters"
           fi
+
           sqlmesh info --skip-connection

Makefile

@@ -43,8 +43,12 @@ install-dev-dbt-%:
 	fi; \
 	$(MAKE) install-dev; \
 	if [ "$$version" = "1.6.0" ]; then \
-		echo "Applying pydantic override for dbt 1.6.0"; \
-		$(PIP) install 'pydantic>=2.0.0' --reinstall; \
+		echo "Applying overrides for dbt 1.6.0"; \
+		$(PIP) install 'pydantic>=2.0.0' 'google-cloud-bigquery==3.30.0' 'databricks-sdk==0.28.0' --reinstall; \
+	fi; \
+	if [ "$$version" = "1.7.0" ]; then \
+		echo "Applying overrides for dbt 1.7.0"; \
+		$(PIP) install 'databricks-sdk==0.28.0' --reinstall; \
 	fi; \
 	mv pyproject.toml.backup pyproject.toml; \
 	echo "Restored original pyproject.toml"