From 8a6f201e2a4b663afb74ae03cc06103e608bc68f Mon Sep 17 00:00:00 2001 From: Siby Abin Thomas <52344507+sibyabin@users.noreply.github.com> Date: Tue, 23 Dec 2025 14:05:09 +0530 Subject: [PATCH] Fix incremental table recreation when defaultProject not set (issue #2053) --- cli/api/dbadapters/bigquery.ts | 13 +++++----- core/actions/incremental_table_test.ts | 35 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index fd2193f7f..e583109e9 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -180,18 +180,17 @@ export class BigQueryDbAdapter implements IDbAdapter { return null; } - const metadataTarget = { - database: metadata.tableReference.projectId, - schema: metadata.tableReference.datasetId, - name: metadata.tableReference.tableId - }; - // Database isn't checked for equality as it's not always presented to the compiled graph, but // IS always available in the warehouse. if ( metadata.tableReference.datasetId !== target.schema || metadata.tableReference.tableId !== target.name ) { + const metadataTarget = { + database: metadata.tableReference.projectId, + schema: metadata.tableReference.datasetId, + name: metadata.tableReference.tableId + }; throw new Error( `Target ${JSON.stringify(metadataTarget)} does not match requested target ${JSON.stringify( target @@ -206,7 +205,7 @@ export class BigQueryDbAdapter implements IDbAdapter { : metadata.type === "VIEW" ? dataform.TableMetadata.Type.VIEW : dataform.TableMetadata.Type.UNKNOWN, - target: metadataTarget, + target, fields: metadata.schema.fields?.map(field => convertField(field)), lastUpdatedMillis: Long.fromString(metadata.lastModifiedTime), description: metadata.description, diff --git a/core/actions/incremental_table_test.ts b/core/actions/incremental_table_test.ts index 9060e19ce..6f36504de 100644 --- a/core/actions/incremental_table_test.ts +++ b/core/actions/incremental_table_test.ts @@ -882,4 +882,39 @@ defaultIcebergConfig: }); }); }); + + test("incremental table without defaultProject in workflow_settings", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // Create workflow_settings without defaultProject (only defaultDataset and defaultLocation) + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + `defaultDataset: dataform +defaultLocation: europe-west2 +` + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/incremental_table_without_default_project.sqlx"), + `config { + type: "incremental", + name: "incremental_table_without_default_project" +} + +select \${incremental()} as is_incremental` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + const compiledTable = result.compile.compiledGraph.tables[0]; + // Verify the table compiles without database field since defaultProject is not set. + // The BigQuery adapter's table() method uses the input target (which doesn't have a database) + // when defaultProject is not specified, ensuring that warehouse state targets match compiled targets. + // This allows incremental appends to work correctly even when defaultProject is not in workflow_settings.yaml. + expect(compiledTable.type).equals("incremental"); + expect(compiledTable.enumType).equals(dataform.TableType.INCREMENTAL); + expect(compiledTable.target.schema).equals("dataform"); + expect(compiledTable.target.name).equals("incremental_table_without_default_project"); + expect(compiledTable.target.database).equals(""); + }); });