From acb79f28b88e307a06182498290eb95d612691bf Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:11:11 +0000 Subject: [PATCH 01/21] feat(cli): add helper utilities to identify 4-part Lakehouse targets --- cli/api/utils/lakehouse.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 cli/api/utils/lakehouse.ts diff --git a/cli/api/utils/lakehouse.ts b/cli/api/utils/lakehouse.ts new file mode 100644 index 000000000..5f6cd672e --- /dev/null +++ b/cli/api/utils/lakehouse.ts @@ -0,0 +1,17 @@ +import { dataform } from "df/protos/ts"; + +/** + * Returns true when `schema` is a Lakehouse `catalog.namespace` pair rather than a plain BigQuery + * dataset. Standard BigQuery dataset names cannot contain `.`, so a dot unambiguously identifies a + * normalized 4-part Lakehouse target (`project.catalog.namespace.table`). + */ +export function isCatalogSchema(schema?: string | null): boolean { + return Boolean(schema && schema.includes(".")); +} + +/** + * Returns true when `target` refers to a 4-part Lakehouse Catalog object. + */ +export function isLakehouseTarget(target?: dataform.ITarget | null): boolean { + return isCatalogSchema(target?.schema); +} From 6c902247e847c2301a57196fbf8eecbd8c85a872 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:15:29 +0000 Subject: [PATCH 02/21] feat(cli): support 4-part Lakehouse targets in BigQueryDbAdapter --- cli/api/dbadapters/bigquery.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index c07f8d35c..d9dc62a69 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -19,6 +19,7 @@ import { OnCancel, } from "df/cli/api/dbadapters/index"; import { parseBigqueryEvalError } from "df/cli/api/utils/error_parsing"; +import { isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { LimitedResultSet } from "df/cli/api/utils/results"; import { coerceAsError } from "df/common/errors/errors"; import { retry } from "df/common/promises"; @@ -376,8 +377,10 @@ export class BigQueryDbAdapter implements IDbAdapter { ? dataform.TableMetadata.Type.VIEW : dataform.TableMetadata.Type.UNKNOWN, target, - fields: metadata.schema.fields?.map((field) => convertField(field)), - lastUpdatedMillis: Long.fromString(metadata.lastModifiedTime), + fields: metadata.schema?.fields?.map((field) => convertField(field)), + lastUpdatedMillis: metadata.lastModifiedTime + ? Long.fromString(metadata.lastModifiedTime) + : Long.ZERO, description: metadata.description, labels: metadata.labels, bigquery: { @@ -387,6 +390,13 @@ export class BigQueryDbAdapter implements IDbAdapter { } public async deleteTable(target: dataform.ITarget): Promise { + if (isLakehouseTarget(target)) { + const database = target.database || this.bigQueryCredentials.projectId; + const prefix = database ? `${database}.` : ""; + await this.execute(`drop table if exists \`${prefix}${target.schema}.${target.name}\``); + return; + } + await this.getClient(target.database) .dataset(target.schema) .table(target.name) @@ -403,6 +413,11 @@ export class BigQueryDbAdapter implements IDbAdapter { public async setMetadata(action: dataform.IExecutionAction): Promise { const { target, actionDescriptor } = action; + if (isLakehouseTarget(target)) { + // BigQuery's tables.patch REST endpoint rejects 4-part Lakehouse Iceberg tables + // ("Connection is required for BigQuery tables for Apache Iceberg"). + return; + } const metadata = await this.getMetadata(target); const schemaWithDescription = addDescriptionToMetadata( @@ -425,7 +440,7 @@ export class BigQueryDbAdapter implements IDbAdapter { .getMetadata(); return table?.[0]; } catch (e) { - if (e?.errors?.[0]?.reason === "notFound") { + if (e?.code === 404 || e?.errors?.[0]?.reason === "notFound") { // if the table can't be found, just return null return null; } From 349350f57ed8e5fcd5c2c31d8f85867cc1c3b5a3 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:17:33 +0000 Subject: [PATCH 03/21] feat(cli): support 4-part Lakehouse targets and onSchemaChange in ExecutionSql --- cli/api/dbadapters/execution_sql.ts | 95 +++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 1be7c452b..bae385b11 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -1,6 +1,7 @@ import * as semver from "semver"; import { Task, Tasks } from "df/cli/api/dbadapters/tasks"; +import { isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { concatenateQueries } from "df/cli/api/utils/query_transforms"; import { ErrorWithCause } from "df/common/errors/errors"; import { CompilationSql } from "df/core/compilation_sql"; @@ -138,7 +139,7 @@ from (${query}) as insertions`; this.preOps(table, runConfig, tableMetadata).forEach((statement) => tasks.add(statement)); const baseTableType = this.baseTableType(table.enumType); - if (tableMetadata && tableMetadata.type !== baseTableType) { + if (!isLakehouseTarget(table.target) && tableMetadata && tableMetadata.type !== baseTableType) { tasks.add( Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType))), ); @@ -153,10 +154,10 @@ from (${query}) as insertions`; case dataform.OnSchemaChange.FAIL: case dataform.OnSchemaChange.EXTEND: case dataform.OnSchemaChange.SYNCHRONIZE: - this.buildIncrementalSchemaChangeTasks(tasks, table); + this.buildIncrementalSchemaChangeTasks(tasks, table, tableMetadata); // Fall through to run the static DML after the procedure alters the schema case dataform.OnSchemaChange.IGNORE: - const columns = tableMetadata?.fields.map((f) => f.name) || []; + const columns = tableMetadata?.fields?.map((f) => f.name) || []; tasks.add(Task.statement(this.getIncrementalDmlStatement(table, columns))); break; } @@ -231,7 +232,16 @@ from (${query}) as insertions`; return `and ${validPredicates.map((p) => `(${p})`).join(" and ")}`; } - private buildIncrementalSchemaChangeTasks(tasks: Tasks, table: dataform.ITable) { + private buildIncrementalSchemaChangeTasks( + tasks: Tasks, + table: dataform.ITable, + tableMetadata?: dataform.ITableMetadata, + ) { + if (isLakehouseTarget(table.target)) { + this.buildLakehouseIncrementalSchemaChangeTasks(tasks, table, tableMetadata); + return; + } + const uniqueId = this.uniqueIdGenerator(); const emptyTempTableTarget = { @@ -260,6 +270,56 @@ END;`; tasks.add(Task.statement(callProcedureSql)); } + private buildLakehouseIncrementalSchemaChangeTasks( + tasks: Tasks, + table: dataform.ITable, + tableMetadata?: dataform.ITableMetadata, + ) { + const uniqueId = this.uniqueIdGenerator(); + const tempTableName = `${table.target.name}_df_temp_${uniqueId}_empty`; + const emptyTempTableTarget = this.resolveScratchTarget(table.target, tempTableName); + const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); + const existingColumns = tableMetadata?.fields?.map((f) => f.name) || []; + const scriptBody = this.incrementalSchemaChangeBody( + table, + this.resolveTarget(table.target), + emptyTempTableTarget, + existingColumns, + ); + const scriptSql = `BEGIN +${scriptBody} +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS ${emptyTempTableName}; + RAISE; +END;`; + tasks.add(Task.statement(scriptSql)); + } + + private resolveScratchTarget( + tableTarget: dataform.ITarget, + tempTableName: string, + ): dataform.ITarget { + const baseDefaultSchema = this.project?.defaultSchema; + if (!baseDefaultSchema) { + throw new Error( + "When defining an incremental Lakehouse table, 'defaultDataset' must be defined in workflow_settings.yaml.", + ); + } + const scratchSchema = this.project?.schemaSuffix + ? `${baseDefaultSchema}_${this.project.schemaSuffix}` + : baseDefaultSchema; + const defaultDatabase = this.project?.defaultDatabase + ? this.project.databaseSuffix + ? `${this.project.defaultDatabase}_${this.project.databaseSuffix}` + : this.project.defaultDatabase + : undefined; + return { + database: tableTarget.database || defaultDatabase, + schema: scratchSchema, + name: tempTableName, + }; + } + private createProcedureName(target: dataform.ITarget, uniqueId: string): string { return this.resolveTarget({ ...target, @@ -304,18 +364,30 @@ CREATE OR REPLACE TABLE ${emptyTempTableName} AS ( private compareSchemasSql( target: dataform.ITarget, emptyTempTableTarget: dataform.ITarget, + existingColumns?: string[], ): string { - return ` --- Compare schemas -SET dataform_columns = ( + const setDataformColumnsSql = + existingColumns !== undefined + ? `SET dataform_columns = ARRAY[${existingColumns + .map((col) => this.CompilationSql.sqlString(col)) + .join(", ")}];` + : `SET dataform_columns = ( SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) FROM \`${target.database}.${target.schema}.INFORMATION_SCHEMA.COLUMNS\` WHERE table_name = '${target.name}' -); +);`; + + const tempInfoSchemaPrefix = emptyTempTableTarget.database + ? `${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}` + : emptyTempTableTarget.schema; + + return ` +-- Compare schemas +${setDataformColumnsSql} SET temp_table_columns = ( SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) - FROM \`${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}.INFORMATION_SCHEMA.COLUMNS\` + FROM \`${tempInfoSchemaPrefix}.INFORMATION_SCHEMA.COLUMNS\` WHERE table_name = '${emptyTempTableTarget.name}' ); @@ -417,6 +489,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; table: dataform.ITable, qualifiedTargetTableName: string, emptyTempTableTarget: dataform.ITarget, + existingColumns?: string[], ): string { const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); const query = this.getIncrementalQuery(table); @@ -424,11 +497,11 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; const statements: string[] = [ this.declareSchemaChangeVariablesSql(onSchemaChange), this.createEmptyTempTableSql(emptyTempTableName, query), - this.compareSchemasSql(table.target, emptyTempTableTarget), + this.compareSchemasSql(table.target, emptyTempTableTarget, existingColumns), this.applySchemaChangeStrategySql(table, qualifiedTargetTableName), this.cleanupSql(emptyTempTableName), ]; - + return statements.join("\n\n"); } From 3d99e051cf53566149fa6d79c06f317d6a52b267 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:18:13 +0000 Subject: [PATCH 04/21] feat(cli): skip schema creation and metadata patching for Lakehouse targets in Runner --- cli/api/commands/run.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index 371760d6a..58ee90059 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -10,6 +10,7 @@ import { dryRunValidationQueryKey, } from "df/cli/api/utils/dry_run_validation"; import { mapErrorPositions } from "df/cli/api/utils/error_parsing"; +import { isCatalogSchema, isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { CREATE_SCHEMA_WRAPPER_LINE_OFFSET, wrapWithCreateSchemaIfNotExists, @@ -358,6 +359,9 @@ export class Runner { (action.tasks.length > 0 || !!action.jitCode) && !!action.target && !!action.target.schema && + // Lakehouse catalog.namespace targets are managed externally via BigLake REST APIs + // and cannot be listed via datasets.list or created via CREATE SCHEMA. + !isCatalogSchema(action.target.schema) && writesToTarget(action), ) .forEach(({ target }) => { @@ -611,6 +615,8 @@ export class Runner { if ( hasTableMetadata(action.actionDescriptor) && + // BigQuery tables.patch rejects 4-part Lakehouse Iceberg tables. + !isLakehouseTarget(action.target) && // Only set metadata if we expect the action to complete in SUCCESSFUL state // (i.e. it must still be RUNNING, and not FAILED). actionResult.status === dataform.ActionResult.ExecutionStatus.RUNNING && @@ -657,7 +663,7 @@ export class Runner { action: dataform.IExecutionAction, task: dataform.IExecutionTask, ): IMissingSchema | null { - if (!action.target?.schema || !writesToTarget(action)) { + if (!action.target?.schema || isLakehouseTarget(action.target) || !writesToTarget(action)) { return null; } // Assertion tasks must stay single statements: their result is read from the job's rows, From 94b7390d2887c50f3705c63c5b6a239012c0ffc4 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:19:25 +0000 Subject: [PATCH 05/21] docs(cli): document 4-part Lakehouse targets in --actions and --output-actions --- cli/README.md | 4 ++-- cli/commands/compile_options.ts | 7 ++++--- cli/common_options.ts | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cli/README.md b/cli/README.md index 9997715bd..fc09c399f 100644 --- a/cli/README.md +++ b/cli/README.md @@ -174,7 +174,7 @@ Compiles the project and prints a summary (or JSON / Graphviz DOT representation | `--quiet` | boolean | `false` | Less verbose compilation output. Mutually exclusive with --verbose. | | `--watch` | boolean | `false` | Watches the project directory and recompiles on changes. | | `--timeout` | string | `'60s'` | Duration to allow project compilation to complete, e.g. '1s', '10m'. | -| `--output-actions` | array | — | Filters the printed output to these actions (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name target. The whole project is still compiled. | +| `--output-actions` | array | — | Filters the printed output to these actions (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target. The whole project is still compiled. | | `--output-tags` | array | — | Filters the printed output to actions with any of these tags (comma-separated or repeatable). | | `--output-include-deps` | boolean | — | Also includes upstream dependencies of the selected actions in the output. Requires --output-actions or --output-tags. | | `--output-include-dependents` | boolean | — | Also includes downstream dependents of the selected actions in the output. Requires --output-actions or --output-tags. | @@ -250,7 +250,7 @@ Compiles and executes the project against BigQuery (or validates it with `--dry- | `--dry-run` | boolean | — | Validates the run SQL with BigQuery without applying any changes to the warehouse. | | `--run-tests` | boolean | — | Requires the project's unit tests to pass before executing; aborts the run if any test fails. | | `--action-retry-limit` | number | `0` | Retries idempotent actions up to this many times on failure. | -| `--actions` | array | — | Actions to run (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name target. | +| `--actions` | array | — | Actions to run (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target. | | `--tags` | array | — | Tags to filter the actions to run (comma-separated or repeatable). | | `--include-deps` | boolean | — | Also runs upstream dependencies of the selected actions. Requires --actions or --tags. | | `--include-dependents` | boolean | — | Also runs downstream dependents of the selected actions. Requires --actions or --tags. | diff --git a/cli/commands/compile_options.ts b/cli/commands/compile_options.ts index 083efc8f4..76917ea19 100644 --- a/cli/commands/compile_options.ts +++ b/cli/commands/compile_options.ts @@ -31,11 +31,12 @@ export const outputActionsOption: INamedOption = { name: "output-actions", option: { // No wildcard support: prune()'s matchPatterns() does exact matching on the - // action name or its fully-qualified `database.schema.name`. + // action name or its fully-qualified `database.schema.name` (or 4-part + // `database.catalog.namespace.name`). describe: "Filters the printed output to these actions (comma-separated or repeatable). Each entry " + - "is an action name or a fully-qualified database.schema.name target. The whole project " + - "is still compiled.", + "is an action name or a fully-qualified database.schema.name (or 4-part " + + "database.catalog.namespace.name) target. The whole project is still compiled.", type: "array", coerce: splitCommas, }, diff --git a/cli/common_options.ts b/cli/common_options.ts index a2b62cc0e..9186dd7d3 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -45,10 +45,10 @@ export const actionsOption: INamedOption = { name: "actions", option: { // No wildcard support: prune()'s matchPatterns() does exact matching on the action name or - // its fully-qualified `database.schema.name`. + // its fully-qualified `database.schema.name` (or 4-part `database.catalog.namespace.name`). describe: "Actions to run (comma-separated or repeatable). Each entry is an action name or a " + - "fully-qualified database.schema.name target.", + "fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target.", type: "array", coerce: splitCommas, }, From 1d69c9139060721cd9492fcbfe433eb07eeb2c48 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:20:00 +0000 Subject: [PATCH 06/21] test(cli): add BigQueryDbAdapter unit tests for 4-part Lakehouse targets --- cli/api/dbadapters/bigquery_test.ts | 172 ++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index 13225171a..27f13b6e8 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -2,6 +2,7 @@ import { Dataset, Table } from "@google-cloud/bigquery"; import { expect } from "chai"; import { EventEmitter } from "events"; import { AuthClient, gaxios, Impersonated } from "google-auth-library"; +import Long from "long"; import { anything, instance, mock, when } from "ts-mockito"; import { @@ -428,4 +429,175 @@ suite("BigQueryDbAdapter", () => { name: tableName, }); }); + + test("table() handles 4-part Lakehouse datasetId including when lastModifiedTime is absent", async () => { + const projectId = "my-gcp-project"; + const lakehouseDatasetId = "ecommerce_catalog.sales_dw"; + const tableName = "customer_orders"; + const mockBigQuery: any = { + dataset: (datasetId: string) => { + expect(datasetId).to.equal(lakehouseDatasetId); + return { + table: (id: string) => { + expect(id).to.equal(tableName); + return { + getMetadata: () => + Promise.resolve([ + { + kind: "bigquery#table", + id: `${projectId}:${lakehouseDatasetId}.${tableName}`, + tableReference: { + projectId, + datasetId: lakehouseDatasetId, + tableId: tableName, + }, + type: "TABLE", + location: "US", + biglakeConfiguration: { + storageUri: "gs://my-lakehouse-bucket/sales_dw/customer_orders/123/", + }, + schema: { + fields: [ + { name: "order_id", type: "INTEGER" }, + { name: "customer_id", type: "INTEGER" }, + ], + }, + // lastModifiedTime intentionally omitted (UIT / Lakehouse response) + }, + ]), + }; + }, + }; + }, + }; + const adapter = new BigQueryDbAdapter(dataform.BigQuery.create({ projectId, location: "US" }), { + clientProvider: () => mockBigQuery, + }); + const target = { database: projectId, schema: lakehouseDatasetId, name: tableName }; + const result = await adapter.table(target); + expect(result).to.not.equal(null); + expect(result.type).to.equal(dataform.TableMetadata.Type.TABLE); + expect(result.target).to.eql(target); + expect(result.fields.map((f) => f.name)).to.eql(["order_id", "customer_id"]); + expect(result.lastUpdatedMillis.equals(Long.ZERO)).to.equal(true); + }); + + test("table() returns null on 404 for nonexistent 4-part Lakehouse table", async () => { + const mockBigQuery: any = { + dataset: () => ({ + table: () => ({ + getMetadata: () => Promise.reject({ code: 404, message: "Not found" }), + }), + }), + }; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "my-gcp-project", location: "US" }), + { clientProvider: () => mockBigQuery }, + ); + const result = await adapter.table({ + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "missing_table", + }); + expect(result).to.equal(null); + }); + + test("setMetadata() skips tables.patch for 4-part Lakehouse targets", async () => { + let datasetCalled = false; + const mockBigQuery: any = { + dataset: () => { + datasetCalled = true; + throw new Error("tables.patch should not be called on 4-part Lakehouse targets"); + }, + }; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "my-gcp-project", location: "US" }), + { concurrencyLimit: 1, clientProvider: () => mockBigQuery }, + ); + await adapter.setMetadata( + dataform.ExecutionAction.create({ + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + actionDescriptor: { + description: "Orders table", + columns: [{ path: ["order_id"], description: "Primary key" }], + }, + }), + ); + expect(datasetCalled).to.equal(false); + }); + + test("deleteTable() executes 4-part DROP TABLE IF EXISTS for Lakehouse targets", async () => { + let restDeleteCalled = false; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "creds-project", location: "US" }), + { + clientProvider: () => + ({ + dataset: () => ({ + table: () => ({ + delete: () => { + restDeleteCalled = true; + return Promise.resolve(); + }, + }), + }), + }) as any, + }, + ); + const executedStatements: string[] = []; + adapter.execute = async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }; + await adapter.deleteTable({ + database: "explicit-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }); + await adapter.deleteTable({ + schema: "ecommerce_catalog.sales_dw", + name: "fallback_project_orders", + }); + expect(restDeleteCalled).to.equal(false); + expect(executedStatements).to.eql([ + "drop table if exists `explicit-project.ecommerce_catalog.sales_dw.customer_orders`", + "drop table if exists `creds-project.ecommerce_catalog.sales_dw.fallback_project_orders`", + ]); + }); + + test("deleteTable() uses REST table.delete for standard 3-part targets", async () => { + let deletedWithIgnoreNotFound = false; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "p", location: "US" }), + { + clientProvider: () => + ({ + dataset: (schema: string) => { + expect(schema).to.equal("standard_dataset"); + return { + table: (name: string) => { + expect(name).to.equal("standard_table"); + return { + delete: (opts: { ignoreNotFound: boolean }) => { + deletedWithIgnoreNotFound = opts.ignoreNotFound; + return Promise.resolve(); + }, + }; + }, + }; + }, + }) as any, + }, + ); + await adapter.deleteTable({ + database: "p", + schema: "standard_dataset", + name: "standard_table", + }); + expect(deletedWithIgnoreNotFound).to.equal(true); + }); }); From 690a6d750596048fc133b71c9d580d141094d068 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:21:30 +0000 Subject: [PATCH 07/21] test(cli): add ExecutionSql unit tests for 4-part Lakehouse targets --- cli/api/execution_sql_test.ts | 250 ++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index f4d4e9520..eb6daee2c 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -278,3 +278,253 @@ suite("ExecutionSql for operations", () => { ).deep.equals([]); }); }); + +suite("ExecutionSql for 4-part Lakehouse targets", () => { + const executionSql = new ExecutionSql( + { + defaultDatabase: "project-id", + defaultSchema: "default_dataset", + }, + "3.1.5", + () => "test_uuid", + ); + + const lakehouseTarget: dataform.ITarget = { + database: "project-id", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }; + + const lakehouseMetadata: dataform.ITableMetadata = { + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + fields: [ + { name: "order_id", primitive: dataform.Field.Primitive.INTEGER }, + { name: "customer_id", primitive: dataform.Field.Primitive.INTEGER }, + ], + }; + + test("emits 4-part CREATE OR REPLACE TABLE and skips incompatible-type DROP VIEW IF EXISTS", () => { + const table: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + bigquery: { + partitionBy: "BUCKET(order_id, 16)", + }, + }; + const incompatibleMetadata: dataform.ITableMetadata = { + type: dataform.TableMetadata.Type.UNKNOWN, + target: lakehouseTarget, + fields: [], + }; + + const tasks = executionSql + .publishTasks(table, { fullRefresh: false }, incompatibleMetadata) + .build(); + + expect(tasks.length).to.equal(1); + expect(tasks[0].statement).to.not.include("drop view"); + expect(tasks[0].statement).to.equal( + "create or replace table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "partition by BUCKET(order_id, 16) as SELECT 1 AS order_id, 2 AS customer_id", + ); + }); + + test("emits CREATE OR REPLACE TABLE on first run and fullRefresh of 4-part incremental table", () => { + const incTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 3 AS order_id, 4 AS customer_id", + }; + + const firstRunSql = executionSql + .publishTasks(incTable, { fullRefresh: false }) + .build()[0].statement; + const fullRefreshSql = executionSql + .publishTasks(incTable, { fullRefresh: true }, lakehouseMetadata) + .build()[0].statement; + + const expectedDdl = + "create or replace table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id"; + expect(firstRunSql).to.equal(expectedDdl); + expect(fullRefreshSql).to.equal(expectedDdl); + }); + + test("emits valid 4-part DML for incremental INSERT, MERGE, and INSERT_OVERWRITE", () => { + const baseIncTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 3 AS order_id, 4 AS customer_id", + }; + + const insertSql = executionSql + .publishTasks(baseIncTable, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + expect(insertSql).to.include( + "insert into `project-id.ecommerce_catalog.sales_dw.customer_orders`", + ); + expect(insertSql).to.include("(`order_id`,`customer_id`)"); + + const mergeSql = executionSql + .publishTasks( + { ...baseIncTable, uniqueKey: ["order_id"] }, + { fullRefresh: false }, + lakehouseMetadata, + ) + .build()[0].statement; + expect(mergeSql).to.include( + "merge `project-id.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + + const insertOverwriteSql = executionSql + .publishTasks( + { + ...baseIncTable, + incrementalStrategy: dataform.IncrementalStrategy.INSERT_OVERWRITE, + bigquery: { partitionBy: "DATE(ts)" }, + }, + { fullRefresh: false }, + lakehouseMetadata, + ) + .build()[0].statement; + expect(insertOverwriteSql).to.include( + "CREATE OR REPLACE TEMP TABLE `staging_table_temp_test_uuid`", + ); + expect(insertOverwriteSql).to.include( + "MERGE `project-id.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + }); + + test("handles onSchemaChange = FAIL on 4-part target without procedure in catalog.namespace or 5-part INFORMATION_SCHEMA", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 2 AS customer_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.FAIL, + }; + + const sql = executionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.df_osc_"); + expect(sql).to.include( + "CREATE OR REPLACE TABLE `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include("SET dataform_columns = ARRAY['order_id', 'customer_id'];"); + expect(sql).to.include("FROM `project-id.default_dataset.INFORMATION_SCHEMA.COLUMNS`"); + expect(sql).to.include("Schema mismatch defined by on_schema_change = 'FAIL'."); + expect(sql).to.include( + "EXCEPTION WHEN ERROR THEN\n DROP TABLE IF EXISTS `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`;\n RAISE;\nEND;", + ); + }); + + test("handles onSchemaChange = EXTEND on 4-part target and applies schemaSuffix to scratch dataset", () => { + const suffixedExecutionSql = new ExecutionSql( + { + defaultDatabase: "project-id", + defaultSchema: "default_dataset", + schemaSuffix: "staging", + }, + "3.1.5", + () => "test_uuid", + ); + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 2 AS customer_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.EXTEND, + }; + + const sql = suffixedExecutionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.include( + "CREATE OR REPLACE TABLE `project-id.default_dataset_staging.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include("FROM `project-id.default_dataset_staging.INFORMATION_SCHEMA.COLUMNS`"); + expect(sql).to.include( + '"ALTER TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` "', + ); + expect(sql).to.include("ADD COLUMN IF NOT EXISTS"); + }); + + test("handles onSchemaChange = SYNCHRONIZE on 4-part target with DECLARE statements first in BEGIN block", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + uniqueKey: ["order_id"], + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.SYNCHRONIZE, + }; + + const sql = executionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.include("DECLARE invalid_removed_columns ARRAY;"); + expect(sql).to.include( + "Cannot drop columns %T as they are part of the unique key for table `project-id.ecommerce_catalog.sales_dw.customer_orders`", + ); + expect(sql).to.include("DROP COLUMN IF EXISTS"); + expect(sql).to.include("ADD COLUMN IF NOT EXISTS"); + + const scriptBody = sql.split("BEGIN\n")[1].split("\nEXCEPTION WHEN ERROR THEN")[0]; + const statements = scriptBody + .split(";") + .map((s) => + s + .split("\n") + .filter((line) => !line.trim().startsWith("--")) + .join("\n") + .trim(), + ) + .filter((s) => s.length > 0); + + let seenNonDeclare = false; + for (const stmt of statements) { + if (stmt.toUpperCase().startsWith("DECLARE ")) { + expect(seenNonDeclare).to.equal(false); + } else { + seenNonDeclare = true; + } + } + }); + + test("creates operation tasks for 4-part Lakehouse operation target", () => { + const tasks = executionSql.createOperationTasks({ + target: lakehouseTarget, + hasOutput: true, + queries: [ + "CREATE OR REPLACE TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` (order_id INT64)", + ], + }); + expect(tasks).deep.equals([ + dataform.ExecutionTask.create({ + type: "statement", + statement: + "CREATE OR REPLACE TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` (order_id INT64)", + }), + ]); + }); +}); From fbfd75aef33e1d0ac6b10b8fd3f3ceec7d7e2b3d Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 10:21:44 +0000 Subject: [PATCH 08/21] test(cli): add Runner and build unit tests for 4-part Lakehouse targets --- cli/api/BUILD | 2 + cli/api/commands/run_test.ts | 206 +++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 cli/api/commands/run_test.ts diff --git a/cli/api/BUILD b/cli/api/BUILD index c6387ab27..64cc4d6a8 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -58,6 +58,7 @@ ts_test_suite( "commands/compile_copy_filter_test.ts", "commands/jit/rpc_test.ts", "commands/prune_test.ts", + "commands/run_test.ts", "dbadapters/bigquery_test.ts", "execution_sql_test.ts", ] + glob(["lineage/*_test.ts"]), @@ -65,6 +66,7 @@ ts_test_suite( deps = [ ":api", "//cli/api/lineage/testing", + "//cli/api/utils", "//core", "//protos:ts", "//testing", diff --git a/cli/api/commands/run_test.ts b/cli/api/commands/run_test.ts new file mode 100644 index 000000000..24d2c45c8 --- /dev/null +++ b/cli/api/commands/run_test.ts @@ -0,0 +1,206 @@ +import { expect } from "chai"; +import Long from "long"; +import { anyString, anything, instance, mock, verify, when } from "ts-mockito"; + +import { build } from "df/cli/api/commands/build"; +import { Runner } from "df/cli/api/commands/run"; +import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { wrapWithCreateSchemaIfNotExists } from "df/cli/api/utils/query_transforms"; +import { dataform } from "df/protos/ts"; +import { suite, test } from "df/testing"; + +suite("Runner with 4-part Lakehouse targets", () => { + const projectConfig: dataform.IProjectConfig = { + warehouse: "bigquery", + defaultDatabase: "my-gcp-project", + defaultSchema: "default_dataset", + }; + + test("skips schemas() lookup and CREATE SCHEMA wrapper when all actions are 4-part Lakehouse targets", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const executedStatements: string[] = []; + when(mockedDbAdapter.execute(anyString(), anything())).thenCall(async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }); + + const lakehouseStatement = + "create or replace table `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` as select 1 as id"; + const opStatement = + "insert into `my-gcp-project.ecommerce_catalog.sales_dw.audit_log` values (1)"; + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + tasks: [{ type: "statement", statement: lakehouseStatement }], + dependencyTargets: [], + }, + { + type: "operation", + hasOutput: true, + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "audit_log", + }, + tasks: [{ type: "statement", statement: opStatement }], + dependencyTargets: [], + }, + ], + }); + + const realRunResult = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + expect(realRunResult.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + + const dryRunResult = await new Runner(instance(mockedDbAdapter), graph, { + bigquery: { dryRun: true }, + }) + .execute() + .result(); + expect(dryRunResult.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + + verify(mockedDbAdapter.schemas(anything())).never(); + expect(executedStatements).deep.equals([ + lakehouseStatement, + opStatement, + lakehouseStatement, + opStatement, + ]); + }); + + test("wraps missing 3-part dataset with CREATE SCHEMA while leaving 4-part Lakehouse target unwrapped", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const executedStatements: string[] = []; + when(mockedDbAdapter.schemas("my-gcp-project")).thenResolve([]); + when(mockedDbAdapter.execute(anyString(), anything())).thenCall(async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }); + + const standardStatement = + "create or replace table `my-gcp-project.new_dataset.std_table` as select 1"; + const lakehouseStatement = + "create or replace table `my-gcp-project.my_cat.my_ns.lh_table` as select 1"; + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { database: "my-gcp-project", schema: "new_dataset", name: "std_table" }, + tasks: [{ type: "statement", statement: standardStatement }], + dependencyTargets: [], + }, + { + type: "table", + tableType: "table", + target: { database: "my-gcp-project", schema: "my_cat.my_ns", name: "lh_table" }, + tasks: [{ type: "statement", statement: lakehouseStatement }], + dependencyTargets: [], + }, + ], + }); + + const result = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + + expect(result.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + verify(mockedDbAdapter.schemas("my-gcp-project")).once(); + expect(executedStatements).to.have.members([ + wrapWithCreateSchemaIfNotExists("my-gcp-project", "new_dataset", standardStatement), + lakehouseStatement, + ]); + }); + + test("build() fetches 4-part target warehouse state via dbadapter.table() and builds incremental DML", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const lakehouseTarget: dataform.ITarget = { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }; + when(mockedDbAdapter.table(anything())).thenResolve( + dataform.TableMetadata.create({ + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + fields: [{ name: "order_id", primitive: dataform.Field.Primitive.INTEGER }], + lastUpdatedMillis: Long.ZERO, + }), + ); + + const compiledGraph = dataform.CompiledGraph.create({ + projectConfig, + dataformCoreVersion: "3.1.5", + tables: [ + { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 2 AS order_id", + uniqueKey: ["order_id"], + }, + ], + }); + + const executionGraph = await build( + compiledGraph, + { fullRefresh: false }, + instance(mockedDbAdapter), + ); + + expect(executionGraph.actions.length).equals(1); + expect(executionGraph.actions[0].tasks[0].statement).to.include( + "merge `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + }); + + test("does not call setMetadata() on 4-part Lakehouse tables with descriptions", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + when(mockedDbAdapter.execute(anyString(), anything())).thenResolve({ rows: [], metadata: {} }); + + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + actionDescriptor: { + description: "Lakehouse orders table", + columns: [{ path: ["order_id"], description: "Order ID" }], + }, + tasks: [ + { + type: "statement", + statement: + "create or replace table `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` as select 1 as order_id", + }, + ], + dependencyTargets: [], + }, + ], + }); + + const result = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + + expect(result.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + verify(mockedDbAdapter.setMetadata(anything())).never(); + }); +}); From a98387c104f598eff4a604f1f7b80c6fd786816c Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 12:13:42 +0000 Subject: [PATCH 09/21] test(cli): verify action pruning for 4-part Lakehouse targets --- cli/api/commands/prune_test.ts | 74 ++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/cli/api/commands/prune_test.ts b/cli/api/commands/prune_test.ts index 87d9f6e7b..fa54ebc8b 100644 --- a/cli/api/commands/prune_test.ts +++ b/cli/api/commands/prune_test.ts @@ -167,4 +167,78 @@ suite("prune", () => { }), ); }); + + const LAKEHOUSE_TARGET: dataform.ITarget = { + database: "p", + schema: "cat.ns", + name: "orders", + }; + const LAKEHOUSE_ACTION: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: LAKEHOUSE_TARGET, + canonicalTarget: LAKEHOUSE_TARGET, + query: "select 1 as order_id", + dependencyTargets: [], + }; + const DOWNSTREAM_OF_LAKEHOUSE_TARGET: dataform.ITarget = { + database: "p", + schema: "d", + name: "order_summary", + }; + const DOWNSTREAM_OF_LAKEHOUSE_ACTION: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: DOWNSTREAM_OF_LAKEHOUSE_TARGET, + canonicalTarget: DOWNSTREAM_OF_LAKEHOUSE_TARGET, + query: "select * from `p.cat.ns.orders`", + dependencyTargets: [LAKEHOUSE_TARGET], + }; + + test("--actions matches 4-part Lakehouse target by database.catalog.namespace.name", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect(asPlainObject(prune(compiledGraph, { actions: ["p.cat.ns.orders"] }))).deep.equals( + asPlainObject({ + tables: [LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET], + }), + ); + }); + + test("--actions matches 4-part Lakehouse target by bare name and resolves includeDependents", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect( + asPlainObject(prune(compiledGraph, { actions: ["orders"], includeDependents: true })), + ).deep.equals(asPlainObject(compiledGraph)); + }); + + test("--actions with includeDependencies pulls in upstream 4-part Lakehouse table", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect( + asPlainObject( + prune(compiledGraph, { actions: ["p.d.order_summary"], includeDependencies: true }), + ), + ).deep.equals(asPlainObject(compiledGraph)); + }); }); From 2cd4bd0b3a3e82d4d5bdc3df215d96d3c7f3574f Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Fri, 13 Nov 2026 12:17:42 +0000 Subject: [PATCH 10/21] style(cli): fix trailing whitespace in execution_sql.ts --- cli/api/dbadapters/execution_sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index bae385b11..57ed1f229 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -501,7 +501,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; this.applySchemaChangeStrategySql(table, qualifiedTargetTableName), this.cleanupSql(emptyTempTableName), ]; - + return statements.join("\n\n"); } From e9c10c360d3bd96a6f98d36eb1c8066525814094 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:03:02 +0000 Subject: [PATCH 11/21] refactor(core,cli): export shared Lakehouse target helpers from @dataform/core Move isCatalogSchema() and isLakehouseTarget() from cli/api/utils/lakehouse.ts to core/targets.ts and reuse isCatalogSchema() in Session.finalizeSchema(). Addresses https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386956 --- cli/api/commands/run.ts | 8 ++++++-- cli/api/dbadapters/bigquery.ts | 2 +- cli/api/dbadapters/execution_sql.ts | 2 +- cli/api/utils/lakehouse.ts | 17 ----------------- core/session.ts | 4 ++-- core/targets.ts | 16 ++++++++++++++++ 6 files changed, 26 insertions(+), 23 deletions(-) delete mode 100644 cli/api/utils/lakehouse.ts diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index 58ee90059..be76561de 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -10,7 +10,6 @@ import { dryRunValidationQueryKey, } from "df/cli/api/utils/dry_run_validation"; import { mapErrorPositions } from "df/cli/api/utils/error_parsing"; -import { isCatalogSchema, isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { CREATE_SCHEMA_WRAPPER_LINE_OFFSET, wrapWithCreateSchemaIfNotExists, @@ -19,7 +18,12 @@ import { Timer } from "df/cli/api/utils/timer"; import { Flags } from "df/common/flags"; import { retry } from "df/common/promises"; import { deepClone, equals } from "df/common/protos"; -import { targetAsReadableString, targetStringifier } from "df/core/targets"; +import { + isCatalogSchema, + isLakehouseTarget, + targetAsReadableString, + targetStringifier, +} from "df/core/targets"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index d9dc62a69..5ac2b3c88 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -19,10 +19,10 @@ import { OnCancel, } from "df/cli/api/dbadapters/index"; import { parseBigqueryEvalError } from "df/cli/api/utils/error_parsing"; -import { isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { LimitedResultSet } from "df/cli/api/utils/results"; import { coerceAsError } from "df/common/errors/errors"; import { retry } from "df/common/promises"; +import { isLakehouseTarget } from "df/core/targets"; import { dataform } from "df/protos/ts"; const CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 57ed1f229..f77e45f0a 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -1,10 +1,10 @@ import * as semver from "semver"; import { Task, Tasks } from "df/cli/api/dbadapters/tasks"; -import { isLakehouseTarget } from "df/cli/api/utils/lakehouse"; import { concatenateQueries } from "df/cli/api/utils/query_transforms"; import { ErrorWithCause } from "df/common/errors/errors"; import { CompilationSql } from "df/core/compilation_sql"; +import { isLakehouseTarget } from "df/core/targets"; import { tableTypeEnumToString } from "df/core/utils"; import { dataform } from "df/protos/ts"; diff --git a/cli/api/utils/lakehouse.ts b/cli/api/utils/lakehouse.ts deleted file mode 100644 index 5f6cd672e..000000000 --- a/cli/api/utils/lakehouse.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { dataform } from "df/protos/ts"; - -/** - * Returns true when `schema` is a Lakehouse `catalog.namespace` pair rather than a plain BigQuery - * dataset. Standard BigQuery dataset names cannot contain `.`, so a dot unambiguously identifies a - * normalized 4-part Lakehouse target (`project.catalog.namespace.table`). - */ -export function isCatalogSchema(schema?: string | null): boolean { - return Boolean(schema && schema.includes(".")); -} - -/** - * Returns true when `target` refers to a 4-part Lakehouse Catalog object. - */ -export function isLakehouseTarget(target?: dataform.ITarget | null): boolean { - return isCatalogSchema(target?.schema); -} diff --git a/core/session.ts b/core/session.ts index 75c0474f5..4d39b3a7b 100644 --- a/core/session.ts +++ b/core/session.ts @@ -21,7 +21,7 @@ import { CompilationSql } from "df/core/compilation_sql"; import { validateSqlxActionBlocks } from "df/core/compilers"; import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables"; import * as Path from "df/core/path"; -import { targetAsReadableString, targetStringifier } from "df/core/targets"; +import { isCatalogSchema, targetAsReadableString, targetStringifier } from "df/core/targets"; import * as utils from "df/core/utils"; import { ResolvableMap, toResolvable } from "df/core/utils"; import { version as dataformCoreVersion } from "df/core/version"; @@ -668,7 +668,7 @@ export class Session { public finalizeSchema(schema: string): string { // Normalized 4-part Lakehouse schemas ("${catalog}.${namespace}") do not take datasetSuffix. // Standard dataset names cannot contain '.' (enforced by ActionBuilder.validateTarget). - if (schema.includes(".")) { + if (isCatalogSchema(schema)) { return schema; } return `${schema}${this.getSchemaSuffixWithUnderscore()}`; diff --git a/core/targets.ts b/core/targets.ts index 9d1eef04f..f173263a8 100644 --- a/core/targets.ts +++ b/core/targets.ts @@ -19,3 +19,19 @@ export function targetAsReadableString(target: dataform.ITarget): string { } return nameParts.reverse().join("."); } + +/** + * Returns true when `schema` is a Lakehouse `catalog.namespace` pair rather than a plain BigQuery + * dataset. Standard BigQuery dataset names cannot contain `.`, so a dot unambiguously identifies a + * normalized 4-part Lakehouse target (`project.catalog.namespace.table`). + */ +export function isCatalogSchema(schema?: string | null): boolean { + return Boolean(schema && schema.includes(".")); +} + +/** + * Returns true when `target` refers to a 4-part Lakehouse Catalog object. + */ +export function isLakehouseTarget(target?: dataform.ITarget | null): boolean { + return isCatalogSchema(target?.schema); +} From c5badcd41a6afbba0952305407b3446e93edd4d0 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:18:15 +0000 Subject: [PATCH 12/21] fix(cli): emit plain CREATE TABLE on first run and protected Lakehouse incremental tables Avoid overwriting an existing Lakehouse table if tables.get ever reports no metadata; only emit OR REPLACE when fullRefresh is set and the table is not protected. Addresses https://github.com/dataform-co/dataform/pull/2445#discussion_r4584716513 --- cli/api/dbadapters/execution_sql.ts | 8 +++++--- cli/api/execution_sql_test.ts | 22 +++++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index f77e45f0a..356ee466f 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -147,7 +147,9 @@ from (${query}) as insertions`; if (table.enumType === dataform.TableType.INCREMENTAL) { if (!this.shouldWriteIncrementally(table, runConfig, tableMetadata)) { - tasks.add(Task.statement(this.createOrReplace(table))); + const allowReplace = + !isLakehouseTarget(table.target) || Boolean(runConfig.fullRefresh && !table.protected); + tasks.add(Task.statement(this.createOrReplace(table, allowReplace))); } else { const onSchemaChange = table.onSchemaChange ?? dataform.OnSchemaChange.IGNORE; switch (onSchemaChange) { @@ -505,7 +507,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; return statements.join("\n\n"); } - private createOrReplace(table: dataform.ITable) { + private createOrReplace(table: dataform.ITable, allowReplace = true) { const options = []; if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) { options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`); @@ -519,7 +521,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; } } - return `create or replace ${table.materialized ? "materialized " : ""}${this.tableTypeAsSql( + return `create ${allowReplace ? "or replace " : ""}${table.materialized ? "materialized " : ""}${this.tableTypeAsSql( this.baseTableType(table.enumType), )} ${this.resolveTarget(table.target)} ${ table.bigquery && table.bigquery.partitionBy diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index eb6daee2c..4b16283ea 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -332,7 +332,7 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { ); }); - test("emits CREATE OR REPLACE TABLE on first run and fullRefresh of 4-part incremental table", () => { + test("emits plain CREATE TABLE on first run / protected and CREATE OR REPLACE TABLE on fullRefresh of 4-part incremental table", () => { const incTable: dataform.ITable = { type: "incremental", enumType: dataform.TableType.INCREMENTAL, @@ -340,19 +340,27 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { query: "SELECT 1 AS order_id, 2 AS customer_id", incrementalQuery: "SELECT 3 AS order_id, 4 AS customer_id", }; - const firstRunSql = executionSql .publishTasks(incTable, { fullRefresh: false }) .build()[0].statement; + const protectedFullRefreshSql = executionSql + .publishTasks({ ...incTable, protected: true }, { fullRefresh: true }) + .build()[0].statement; const fullRefreshSql = executionSql .publishTasks(incTable, { fullRefresh: true }, lakehouseMetadata) .build()[0].statement; - - const expectedDdl = + expect(firstRunSql).to.equal( + "create table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id", + ); + expect(protectedFullRefreshSql).to.equal( + "create table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id", + ); + expect(fullRefreshSql).to.equal( "create or replace table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + - "as SELECT 1 AS order_id, 2 AS customer_id"; - expect(firstRunSql).to.equal(expectedDdl); - expect(fullRefreshSql).to.equal(expectedDdl); + "as SELECT 1 AS order_id, 2 AS customer_id", + ); }); test("emits valid 4-part DML for incremental INSERT, MERGE, and INSERT_OVERWRITE", () => { From cdc18651e38bde26775f7f29b52532c3105b8dc7 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:33:15 +0000 Subject: [PATCH 13/21] fix(cli): throw when Lakehouse tableMetadata is missing schema fields on schema change Check !tableMetadata?.fields?.length (accounting for protobufjs empty repeated field initialization) instead of defaulting to [], and document that existingColumns is a build-time snapshot. Addresses: - https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386926 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4584717106 --- cli/api/dbadapters/execution_sql.ts | 10 +++++++++- cli/api/execution_sql_test.ts | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 356ee466f..51162f5d7 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -277,11 +277,19 @@ END;`; table: dataform.ITable, tableMetadata?: dataform.ITableMetadata, ) { + if (!tableMetadata?.fields?.length) { + throw new Error( + `Cannot apply onSchemaChange to Lakehouse table ${this.resolveTarget(table.target)} because warehouse metadata is missing schema fields.`, + ); + } const uniqueId = this.uniqueIdGenerator(); const tempTableName = `${table.target.name}_df_temp_${uniqueId}_empty`; const emptyTempTableTarget = this.resolveScratchTarget(table.target, tempTableName); const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); - const existingColumns = tableMetadata?.fields?.map((f) => f.name) || []; + // Note: `existingColumns` is a build-time snapshot from `tables.get` metadata (whereas the + // standard 3-part path queries INFORMATION_SCHEMA at execution time), so schema changes made + // by `incrementalPreOps` are not visible here. + const existingColumns = tableMetadata.fields.map((f) => f.name); const scriptBody = this.incrementalSchemaChangeBody( table, this.resolveTarget(table.target), diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index 4b16283ea..b7790b71d 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -535,4 +535,26 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { }), ]); }); + + test("throws when applying onSchemaChange to Lakehouse table with missing schema fields", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 1 AS order_id", + onSchemaChange: dataform.OnSchemaChange.FAIL, + }; + + expect(() => + executionSql.publishTasks( + table, + { fullRefresh: false }, + dataform.TableMetadata.create({ + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + }), + ), + ).to.throw(/missing schema fields/); + }); }); From df5b2b0ecf76083b6290b21460a39538380828a0 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:39:40 +0000 Subject: [PATCH 14/21] fix(cli): resolve Lakehouse scratch table in defaultProject and require defaultDatabase Always create the temporary schema-inspection table in defaultDatabase (with databaseSuffix) instead of the catalog's project, and throw when defaultProject is unset. Addresses: - https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386911 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4584716811 --- cli/api/dbadapters/execution_sql.ts | 23 +++++++------- cli/api/execution_sql_test.ts | 48 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 51162f5d7..394984479 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -284,7 +284,7 @@ END;`; } const uniqueId = this.uniqueIdGenerator(); const tempTableName = `${table.target.name}_df_temp_${uniqueId}_empty`; - const emptyTempTableTarget = this.resolveScratchTarget(table.target, tempTableName); + const emptyTempTableTarget = this.resolveScratchTarget(tempTableName); const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); // Note: `existingColumns` is a build-time snapshot from `tables.get` metadata (whereas the // standard 3-part path queries INFORMATION_SCHEMA at execution time), so schema changes made @@ -305,26 +305,27 @@ END;`; tasks.add(Task.statement(scriptSql)); } - private resolveScratchTarget( - tableTarget: dataform.ITarget, - tempTableName: string, - ): dataform.ITarget { + private resolveScratchTarget(tempTableName: string): dataform.ITarget { + const baseDefaultDatabase = this.project?.defaultDatabase; + if (!baseDefaultDatabase) { + throw new Error( + "When defining an incremental Lakehouse table with onSchemaChange, 'defaultProject' must be defined in workflow_settings.yaml.", + ); + } const baseDefaultSchema = this.project?.defaultSchema; if (!baseDefaultSchema) { throw new Error( "When defining an incremental Lakehouse table, 'defaultDataset' must be defined in workflow_settings.yaml.", ); } + const scratchDatabase = this.project?.databaseSuffix + ? `${baseDefaultDatabase}_${this.project.databaseSuffix}` + : baseDefaultDatabase; const scratchSchema = this.project?.schemaSuffix ? `${baseDefaultSchema}_${this.project.schemaSuffix}` : baseDefaultSchema; - const defaultDatabase = this.project?.defaultDatabase - ? this.project.databaseSuffix - ? `${this.project.defaultDatabase}_${this.project.databaseSuffix}` - : this.project.defaultDatabase - : undefined; return { - database: tableTarget.database || defaultDatabase, + database: scratchDatabase, schema: scratchSchema, name: tempTableName, }; diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index b7790b71d..a37dd2739 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -557,4 +557,52 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { ), ).to.throw(/missing schema fields/); }); + + test("places scratch table in defaultDatabase when Lakehouse catalog lives in another project and throws if defaultProject is unset", () => { + const crossProjectExecutionSql = new ExecutionSql( + { + defaultDatabase: "default-project", + databaseSuffix: "dev", + defaultSchema: "default_dataset", + schemaSuffix: "staging", + }, + "3.1.5", + () => "test_uuid", + ); + const crossProjectTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: { + database: "catalog-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 1 AS order_id", + onSchemaChange: dataform.OnSchemaChange.EXTEND, + }; + + const sql = crossProjectExecutionSql + .publishTasks(crossProjectTable, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.include( + "CREATE OR REPLACE TABLE `default-project_dev.default_dataset_staging.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include( + "FROM `default-project_dev.default_dataset_staging.INFORMATION_SCHEMA.COLUMNS`", + ); + expect(sql).to.include( + '"ALTER TABLE `catalog-project.ecommerce_catalog.sales_dw.customer_orders` "', + ); + + const noDefaultDbExecutionSql = new ExecutionSql({ defaultSchema: "default_dataset" }, "3.1.5"); + expect(() => + noDefaultDbExecutionSql.publishTasks( + crossProjectTable, + { fullRefresh: false }, + lakehouseMetadata, + ), + ).to.throw(/'defaultProject' must be defined/); + }); }); From fbb8d2daa3ae64c7206c442ec991cf375b335d43 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:49:03 +0000 Subject: [PATCH 15/21] fix(cli): ensure scratch dataset exists with defaultLocation for Lakehouse onSchemaChange Prepend CREATE SCHEMA IF NOT EXISTS (using project defaultLocation when configured) before the Lakehouse schema-change block so projects with only Lakehouse targets have the scratch dataset created in the catalog's region. Addresses: - https://github.com/dataform-co/dataform/pull/2445#discussion_r4564513640 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4566388892 --- cli/api/dbadapters/execution_sql.ts | 7 ++++++- cli/api/execution_sql_test.ts | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 394984479..b7478ce98 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -296,7 +296,12 @@ END;`; emptyTempTableTarget, existingColumns, ); - const scriptSql = `BEGIN + const locationOption = this.project?.defaultLocation + ? ` OPTIONS(location=${this.CompilationSql.sqlString(this.project.defaultLocation)})` + : ""; + const createScratchSchemaSql = `CREATE SCHEMA IF NOT EXISTS \`${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}\`${locationOption};`; + const scriptSql = `${createScratchSchemaSql} +BEGIN ${scriptBody} EXCEPTION WHEN ERROR THEN DROP TABLE IF EXISTS ${emptyTempTableName}; diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index a37dd2739..57b6bee56 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -436,6 +436,7 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { expect(sql).to.include( "EXCEPTION WHEN ERROR THEN\n DROP TABLE IF EXISTS `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`;\n RAISE;\nEND;", ); + expect(sql).to.include("CREATE SCHEMA IF NOT EXISTS `project-id.default_dataset`;\nBEGIN"); }); test("handles onSchemaChange = EXTEND on 4-part target and applies schemaSuffix to scratch dataset", () => { @@ -562,6 +563,7 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { const crossProjectExecutionSql = new ExecutionSql( { defaultDatabase: "default-project", + defaultLocation: "us-central1", databaseSuffix: "dev", defaultSchema: "default_dataset", schemaSuffix: "staging", @@ -595,6 +597,9 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { expect(sql).to.include( '"ALTER TABLE `catalog-project.ecommerce_catalog.sales_dw.customer_orders` "', ); + expect(sql).to.include( + "CREATE SCHEMA IF NOT EXISTS `default-project_dev.default_dataset_staging` OPTIONS(location='us-central1');\nBEGIN", + ); const noDefaultDbExecutionSql = new ExecutionSql({ defaultSchema: "default_dataset" }, "3.1.5"); expect(() => From 1e2c0d4fb4d06c117472b65b218838cccf5ac3f0 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 14:59:17 +0000 Subject: [PATCH 16/21] fix(cli): set 1-day expiration_timestamp on Lakehouse onSchemaChange scratch tables Ensure orphaned scratch tables in defaultDataset are automatically cleaned up if a run times out or is cancelled before DROP TABLE IF EXISTS executes. Addresses https://github.com/dataform-co/dataform/pull/2445#discussion_r4584716533 --- cli/api/dbadapters/execution_sql.ts | 13 ++++++++++--- cli/api/execution_sql_test.ts | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index b7478ce98..ecf5c8b44 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -369,10 +369,17 @@ DECLARE columns_removed ARRAY;`; return sql; } - private createEmptyTempTableSql(emptyTempTableName: string, query: string): string { + private createEmptyTempTableSql( + emptyTempTableName: string, + query: string, + setExpiration = false, + ): string { + const optionsClause = setExpiration + ? "\nOPTIONS(expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))" + : ""; return ` -- Create empty table to extract schema of new query. -CREATE OR REPLACE TABLE ${emptyTempTableName} AS ( +CREATE OR REPLACE TABLE ${emptyTempTableName}${optionsClause} AS ( SELECT * FROM (${query}) AS insertions LIMIT 0 );`; } @@ -512,7 +519,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; const onSchemaChange = table.onSchemaChange || dataform.OnSchemaChange.IGNORE; const statements: string[] = [ this.declareSchemaChangeVariablesSql(onSchemaChange), - this.createEmptyTempTableSql(emptyTempTableName, query), + this.createEmptyTempTableSql(emptyTempTableName, query, isLakehouseTarget(table.target)), this.compareSchemasSql(table.target, emptyTempTableTarget, existingColumns), this.applySchemaChangeStrategySql(table, qualifiedTargetTableName), this.cleanupSql(emptyTempTableName), diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index 57b6bee56..7bebc945f 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -437,6 +437,9 @@ suite("ExecutionSql for 4-part Lakehouse targets", () => { "EXCEPTION WHEN ERROR THEN\n DROP TABLE IF EXISTS `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`;\n RAISE;\nEND;", ); expect(sql).to.include("CREATE SCHEMA IF NOT EXISTS `project-id.default_dataset`;\nBEGIN"); + expect(sql).to.include( + "OPTIONS(expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))", + ); }); test("handles onSchemaChange = EXTEND on 4-part target and applies schemaSuffix to scratch dataset", () => { From f1584316e009a93921b51cbf1c1c5ac354c26d8d Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 15:05:18 +0000 Subject: [PATCH 17/21] fix(cli): require target.database in BigQueryDbAdapter.deleteTable for Lakehouse targets Throw explicitly when target.database is missing instead of falling back to credentials.projectId, avoiding dropping a table in the wrong project when defaultProject differs from credentials. Addresses https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386940 --- cli/api/dbadapters/bigquery.ts | 11 ++++++++--- cli/api/dbadapters/bigquery_test.ts | 15 ++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index 5ac2b3c88..d06ded8da 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -391,9 +391,14 @@ export class BigQueryDbAdapter implements IDbAdapter { public async deleteTable(target: dataform.ITarget): Promise { if (isLakehouseTarget(target)) { - const database = target.database || this.bigQueryCredentials.projectId; - const prefix = database ? `${database}.` : ""; - await this.execute(`drop table if exists \`${prefix}${target.schema}.${target.name}\``); + if (!target.database) { + throw new Error( + `Cannot delete Lakehouse table "${target.schema}.${target.name}" without a target database (project).`, + ); + } + await this.execute( + `drop table if exists \`${target.database}.${target.schema}.${target.name}\``, + ); return; } diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index 27f13b6e8..c63125dc3 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -558,14 +558,19 @@ suite("BigQueryDbAdapter", () => { schema: "ecommerce_catalog.sales_dw", name: "customer_orders", }); - await adapter.deleteTable({ - schema: "ecommerce_catalog.sales_dw", - name: "fallback_project_orders", - }); + let thrownError: Error | undefined; + try { + await adapter.deleteTable({ + schema: "ecommerce_catalog.sales_dw", + name: "missing_project_orders", + }); + } catch (e) { + thrownError = e as Error; + } + expect(thrownError?.message).to.include("without a target database"); expect(restDeleteCalled).to.equal(false); expect(executedStatements).to.eql([ "drop table if exists `explicit-project.ecommerce_catalog.sales_dw.customer_orders`", - "drop table if exists `creds-project.ecommerce_catalog.sales_dw.fallback_project_orders`", ]); }); From 0c410207aee6430457bbf44d433669c3a478219d Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 16:04:08 +0000 Subject: [PATCH 18/21] feat(cli): apply Lakehouse table and column descriptions via ALTER TABLE SQL - Execute separate ALTER TABLE SET OPTIONS and ALTER COLUMN IF EXISTS statements in BigQueryDbAdapter.setMetadata() for 4-part Lakehouse targets, forwarding execution location. - Keep Lakehouse metadata handling solely in BigQueryDbAdapter instead of duplicating the check in Runner.executeAction(). Addresses: - https://github.com/dataform-co/dataform/pull/2445#discussion_r4564684373 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4584717404 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386930 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4584717629 --- cli/api/commands/run.ts | 6 ++-- cli/api/commands/run_test.ts | 6 ++-- cli/api/dbadapters/bigquery.ts | 50 +++++++++++++++++++++++++++-- cli/api/dbadapters/bigquery_test.ts | 31 ++++++++++++++---- cli/api/dbadapters/index.ts | 13 +++++++- tests/api/run.spec.ts | 8 +++-- tests/api/schemas.spec.ts | 2 +- 7 files changed, 95 insertions(+), 21 deletions(-) diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index be76561de..feddae728 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -619,8 +619,6 @@ export class Runner { if ( hasTableMetadata(action.actionDescriptor) && - // BigQuery tables.patch rejects 4-part Lakehouse Iceberg tables. - !isLakehouseTarget(action.target) && // Only set metadata if we expect the action to complete in SUCCESSFUL state // (i.e. it must still be RUNNING, and not FAILED). actionResult.status === dataform.ActionResult.ExecutionStatus.RUNNING && @@ -630,7 +628,9 @@ export class Runner { action.type === "table" ) { try { - await this.dbadapter.setMetadata(action); + await this.dbadapter.setMetadata(action, { + bigquery: this.getBigQueryExecutionOptions(action), + }); } catch (e) { // TODO: Setting the metadata is not a task itself, so we have nowhere to surface this error cleanly. // For now, we can attach the error to the last task in the action so it gets diff --git a/cli/api/commands/run_test.ts b/cli/api/commands/run_test.ts index 24d2c45c8..09d967f46 100644 --- a/cli/api/commands/run_test.ts +++ b/cli/api/commands/run_test.ts @@ -165,9 +165,10 @@ suite("Runner with 4-part Lakehouse targets", () => { ); }); - test("does not call setMetadata() on 4-part Lakehouse tables with descriptions", async () => { + test("delegates setMetadata() with execution options on 4-part Lakehouse tables with descriptions", async () => { const mockedDbAdapter = mock(BigQueryDbAdapter); when(mockedDbAdapter.execute(anyString(), anything())).thenResolve({ rows: [], metadata: {} }); + when(mockedDbAdapter.setMetadata(anything(), anything())).thenResolve(); const graph = dataform.ExecutionGraph.create({ projectConfig, @@ -199,8 +200,7 @@ suite("Runner with 4-part Lakehouse targets", () => { }); const result = await new Runner(instance(mockedDbAdapter), graph).execute().result(); - expect(result.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); - verify(mockedDbAdapter.setMetadata(anything())).never(); + verify(mockedDbAdapter.setMetadata(anything(), anything())).once(); }); }); diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index d06ded8da..91d6fdce6 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -416,11 +416,13 @@ export class BigQueryDbAdapter implements IDbAdapter { return data[0].map((dataset) => dataset.id); } - public async setMetadata(action: dataform.IExecutionAction): Promise { + public async setMetadata( + action: dataform.IExecutionAction, + options?: { bigquery?: IBigQueryExecutionOptions }, + ): Promise { const { target, actionDescriptor } = action; if (isLakehouseTarget(target)) { - // BigQuery's tables.patch REST endpoint rejects 4-part Lakehouse Iceberg tables - // ("Connection is required for BigQuery tables for Apache Iceberg"). + await this.setLakehouseMetadata(target, actionDescriptor, options); return; } @@ -437,6 +439,40 @@ export class BigQueryDbAdapter implements IDbAdapter { }); } + private async setLakehouseMetadata( + target: dataform.ITarget, + actionDescriptor?: dataform.IActionDescriptor | null, + options?: { bigquery?: IBigQueryExecutionOptions }, + ): Promise { + if (!target.database) { + throw new Error( + `Cannot set metadata on Lakehouse table "${target.schema}.${target.name}" without a target database (project).`, + ); + } + // BigQuery's tables.patch REST endpoint rejects 4-part Lakehouse Iceberg tables, so apply + // table and top-level column descriptions via SQL DDL instead. BigQuery does not allow + // mixing table SET OPTIONS and ALTER COLUMN in a single ALTER TABLE statement. + const qualifiedTable = `\`${target.database}.${target.schema}.${target.name}\``; + const statements: string[] = []; + if (actionDescriptor?.description != null && actionDescriptor.description !== "") { + statements.push( + `ALTER TABLE ${qualifiedTable} SET OPTIONS(description = ${escapeSqlStringLiteral(actionDescriptor.description)});`, + ); + } + const alterColumns = (actionDescriptor?.columns || []) + .filter((col) => col.path?.length === 1 && col.description != null && col.description !== "") + .map( + (col) => + `ALTER COLUMN IF EXISTS \`${col.path[0]}\` SET OPTIONS(description = ${escapeSqlStringLiteral(col.description)})`, + ); + if (alterColumns.length > 0) { + statements.push(`ALTER TABLE ${qualifiedTable} ${alterColumns.join(", ")};`); + } + if (statements.length > 0) { + await this.execute(statements.join("\n"), { bigquery: options?.bigquery }); + } + } + private async getMetadata(target: dataform.ITarget): Promise { try { const table = await this.getClient(target.database) @@ -744,3 +780,11 @@ function createBigQueryError(jobMetadata: any): IBigQueryError { } return error; } + +function escapeSqlStringLiteral(value: string): string { + return `'${value + .replace(/\\/g, "\\\\") + .replace(/'/g, "\\'") + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r")}'`; +} diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index c63125dc3..644f4b034 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -1,4 +1,4 @@ -import { Dataset, Table } from "@google-cloud/bigquery"; +import { BigQuery, Dataset, Table } from "@google-cloud/bigquery"; import { expect } from "chai"; import { EventEmitter } from "events"; import { AuthClient, gaxios, Impersonated } from "google-auth-library"; @@ -502,18 +502,23 @@ suite("BigQueryDbAdapter", () => { expect(result).to.equal(null); }); - test("setMetadata() skips tables.patch for 4-part Lakehouse targets", async () => { + test("setMetadata() applies table and top-level column descriptions via ALTER TABLE SQL for 4-part Lakehouse targets", async () => { let datasetCalled = false; - const mockBigQuery: any = { + const mockBigQuery = { dataset: () => { datasetCalled = true; - throw new Error("tables.patch should not be called on 4-part Lakehouse targets"); + throw new Error("tables.patch must not be called for Lakehouse targets"); }, - }; + } as unknown as BigQuery; const adapter = new BigQueryDbAdapter( dataform.BigQuery.create({ projectId: "my-gcp-project", location: "US" }), { concurrencyLimit: 1, clientProvider: () => mockBigQuery }, ); + const executedCalls: Array<{ statement: string; location?: string }> = []; + adapter.execute = async (statement: string, options?: any) => { + executedCalls.push({ statement, location: options?.bigquery?.location }); + return { rows: [], metadata: {} }; + }; await adapter.setMetadata( dataform.ExecutionAction.create({ target: { @@ -522,12 +527,24 @@ suite("BigQueryDbAdapter", () => { name: "customer_orders", }, actionDescriptor: { - description: "Orders table", - columns: [{ path: ["order_id"], description: "Primary key" }], + description: "Orders' main\ntable", + columns: [ + { path: ["order_id"], description: "Primary key" }, + { path: ["struct_col", "nested"], description: "Ignored nested field" }, + ], }, }), + { bigquery: { location: "us-central1" } }, ); expect(datasetCalled).to.equal(false); + expect(executedCalls).to.eql([ + { + statement: + "ALTER TABLE `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` SET OPTIONS(description = 'Orders\\' main\\ntable');\n" + + "ALTER TABLE `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` ALTER COLUMN IF EXISTS `order_id` SET OPTIONS(description = 'Primary key');", + location: "us-central1", + }, + ]); }); test("deleteTable() executes 4-part DROP TABLE IF EXISTS for Lakehouse targets", async () => { diff --git a/cli/api/dbadapters/index.ts b/cli/api/dbadapters/index.ts index af2f13b85..aa2610be9 100644 --- a/cli/api/dbadapters/index.ts +++ b/cli/api/dbadapters/index.ts @@ -63,5 +63,16 @@ export interface IDbAdapter extends IDbClient { table(target: dataform.ITarget): Promise; deleteTable(target: dataform.ITarget): Promise; - setMetadata(action: dataform.IExecutionAction): Promise; + setMetadata( + action: dataform.IExecutionAction, + options?: { + bigquery?: { + labels?: { [label: string]: string }; + location?: string; + jobPrefix?: string; + dryRun?: boolean; + reservation?: string; + }; + }, + ): Promise; } diff --git a/tests/api/run.spec.ts b/tests/api/run.spec.ts index 45487ebaf..074a5c3e9 100644 --- a/tests/api/run.spec.ts +++ b/tests/api/run.spec.ts @@ -736,7 +736,7 @@ suite("@dataform/api/run", () => { executionOptions.push(options); return Promise.resolve({ rows: [], metadata: {} }); }); - when(mockedDbAdapter.setMetadata(anything())).thenCall( + when(mockedDbAdapter.setMetadata(anything(), anything())).thenCall( (action: dataform.IExecutionAction) => { setMetadataActionDescriptors.push(action.actionDescriptor!); return Promise.resolve(); @@ -786,7 +786,7 @@ suite("@dataform/api/run", () => { const result = await runner.execute().result(); expect(result.status).to.equal(dataform.RunResult.ExecutionStatus.SUCCESSFUL); - verify(mockedDbAdapter.setMetadata(anything())).never(); + verify(mockedDbAdapter.setMetadata(anything(), anything())).never(); }); test("resolveJobLabels falls back to bigqueryLabels when jobLabels is empty, and execution labels override them", () => { @@ -910,7 +910,9 @@ suite("@dataform/api/run", () => { rows: [], metadata: {}, }); - when(mockedDbAdapter.setMetadata(anything())).thenReject(new Error("Error during setMetadata")); + when(mockedDbAdapter.setMetadata(anything(), anything())).thenReject( + new Error("Error during setMetadata"), + ); const mockDbAdapterInstance = instance(mockedDbAdapter); diff --git a/tests/api/schemas.spec.ts b/tests/api/schemas.spec.ts index b23cd173a..df9447755 100644 --- a/tests/api/schemas.spec.ts +++ b/tests/api/schemas.spec.ts @@ -260,7 +260,7 @@ suite("schema creation", () => { errorMessage: "Run cancelled", }, ]); - verify(mockedDbAdapter.setMetadata(anything())).never(); + verify(mockedDbAdapter.setMetadata(anything(), anything())).never(); expect(executedStatements).deep.equals([wrapped("new_ds", "CREATE TABLE a")]); }); From 5d4c6e446a7fb983c91342c6d3a5ba96fe887751 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 18:54:12 +0000 Subject: [PATCH 19/21] chore(cli): remove redundant Lakehouse check and clarify comment in Runner - Remove redundant isLakehouseTarget() guard in missingSchemaToCreate() since findMissingSchemas() already excludes catalog schemas. - Clarify that datasets.list can list BIGLAKE_ICEBERG namespaces, while CREATE SCHEMA cannot create them. Addresses: - https://github.com/dataform-co/dataform/pull/2445#discussion_r4566386950 - https://github.com/dataform-co/dataform/pull/2445#discussion_r4584716545 --- cli/api/commands/run.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index feddae728..c64a5c09e 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -18,12 +18,7 @@ import { Timer } from "df/cli/api/utils/timer"; import { Flags } from "df/common/flags"; import { retry } from "df/common/promises"; import { deepClone, equals } from "df/common/protos"; -import { - isCatalogSchema, - isLakehouseTarget, - targetAsReadableString, - targetStringifier, -} from "df/core/targets"; +import { isCatalogSchema, targetAsReadableString, targetStringifier } from "df/core/targets"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -363,8 +358,7 @@ export class Runner { (action.tasks.length > 0 || !!action.jitCode) && !!action.target && !!action.target.schema && - // Lakehouse catalog.namespace targets are managed externally via BigLake REST APIs - // and cannot be listed via datasets.list or created via CREATE SCHEMA. + // Lakehouse catalog.namespace targets cannot be created via CREATE SCHEMA; they must exist before the run. !isCatalogSchema(action.target.schema) && writesToTarget(action), ) @@ -667,7 +661,7 @@ export class Runner { action: dataform.IExecutionAction, task: dataform.IExecutionTask, ): IMissingSchema | null { - if (!action.target?.schema || isLakehouseTarget(action.target) || !writesToTarget(action)) { + if (!action.target?.schema || !writesToTarget(action)) { return null; } // Assertion tasks must stay single statements: their result is read from the job's rows, From cc8dbceb54b2e2ef5d81074afb2d443922b61229 Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 18:59:53 +0000 Subject: [PATCH 20/21] fix(cli): skip 4-part Lakehouse targets in OpenLineage dataset inputs and outputs Data Lineage parses bigquery namespace dataset names as 3-part .. identifiers. Filter out 4-part Lakehouse targets from inputs and omit inputs/outputs when the action target itself is a Lakehouse target. Addresses https://github.com/dataform-co/dataform/pull/2445#pullrequestreview-5918676679 --- cli/api/lineage/payload_builder.ts | 21 +++++++++++++----- cli/api/lineage/payload_builder_test.ts | 29 +++++++++++++++++++++++++ docs/lineage.md | 2 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/cli/api/lineage/payload_builder.ts b/cli/api/lineage/payload_builder.ts index 05c01d3cc..99936ce1e 100644 --- a/cli/api/lineage/payload_builder.ts +++ b/cli/api/lineage/payload_builder.ts @@ -3,6 +3,7 @@ import Long from "long"; import * as path from "path"; import { toLineageActionType } from "df/cli/api/lineage/action_type"; +import { isLakehouseTarget } from "df/core/targets"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -81,13 +82,21 @@ export class LineagePayloadBuilder { } } - const emitDatasets = eventType === "COMPLETE" && hasLineageOutput(action, actionType); - + // Data Lineage expects 3-part `..
` BigQuery dataset names; a 4-part + // Lakehouse target (`project.catalog.namespace.table`) would be misparsed as project + // `project.catalog`. Skip Lakehouse targets in `inputs`, and omit `inputs`/`outputs` altogether + // when the action's own target is a Lakehouse target. + const emitDatasets = + eventType === "COMPLETE" && + hasLineageOutput(action, actionType) && + !isLakehouseTarget(action.target); const inputs = emitDatasets - ? (action.dependencyTargets || []).map((dep) => ({ - namespace: "bigquery", - name: toFqn(dep, projectId), - })) + ? (action.dependencyTargets || []) + .filter((dep) => !isLakehouseTarget(dep)) + .map((dep) => ({ + namespace: "bigquery", + name: toFqn(dep, projectId), + })) : []; const outputs = emitDatasets diff --git a/cli/api/lineage/payload_builder_test.ts b/cli/api/lineage/payload_builder_test.ts index 728b655b1..d15206983 100644 --- a/cli/api/lineage/payload_builder_test.ts +++ b/cli/api/lineage/payload_builder_test.ts @@ -331,4 +331,33 @@ suite("LineagePayloadBuilder", () => { expect(opWithoutOutputComplete.inputs).to.deep.equal([]); expect(opWithoutOutputComplete.outputs).to.deep.equal([]); }); + + test("omits Lakehouse targets from inputs and omits inputs/outputs when action target is Lakehouse", () => { + const builder = new LineagePayloadBuilder("/tmp/proj"); + + const lakehouseAction = dataform.ExecutionAction.create({ + target: { database: "proj", schema: "my_catalog.my_ns", name: "lh_table" }, + type: "table", + dependencyTargets: [{ database: "proj", schema: "bq_dataset", name: "src_table" }], + }); + const lhComplete = builder.build(lakehouseAction, COMPLETE_RESULT, "proj", "us"); + expect(lhComplete.inputs).to.deep.equal([]); + expect(lhComplete.outputs).to.deep.equal([]); + + const bqDownstreamAction = dataform.ExecutionAction.create({ + target: { database: "proj", schema: "bq_dataset", name: "downstream_table" }, + type: "table", + dependencyTargets: [ + { database: "proj", schema: "my_catalog.my_ns", name: "lh_table" }, + { database: "proj", schema: "bq_dataset", name: "src_table" }, + ], + }); + const bqComplete = builder.build(bqDownstreamAction, COMPLETE_RESULT, "proj", "us"); + expect(bqComplete.inputs).to.deep.equal([ + { namespace: "bigquery", name: "proj.bq_dataset.src_table" }, + ]); + expect(bqComplete.outputs).to.deep.equal([ + { namespace: "bigquery", name: "proj.bq_dataset.downstream_table" }, + ]); + }); }); diff --git a/docs/lineage.md b/docs/lineage.md index 852fd112f..d47b86d24 100644 --- a/docs/lineage.md +++ b/docs/lineage.md @@ -79,6 +79,8 @@ Lineage events are emitted for every Dataform action that executes a BigQuery jo `inputs` and `outputs` are populated only on `COMPLETE` events (`START`, `FAIL`, and `ABORT` events carry empty `inputs` and `outputs` arrays) so failed or cancelled runs do not create false lineage edges. +For **4-part Lakehouse Catalog targets** (`project.catalog.namespace.table`), `Process` and `Run` events are still emitted, but Lakehouse targets are omitted from `inputs`, and both `inputs` and `outputs` are empty (`[]`) when the action's own target is a Lakehouse target (because the Data Lineage `bigquery` namespace expects 3-part `..
` identifiers). + --- ## Where Lineage Appears in Google Cloud From b171e5a37a92357f482d3c381a505f4d856b076b Mon Sep 17 00:00:00 2001 From: Pawbag123 Date: Tue, 17 Nov 2026 19:14:18 +0000 Subject: [PATCH 21/21] docs(cli): document 4-part Lakehouse target requirements in cli/README.md Document catalog/namespace pre-existence, defaultProject/defaultDataset location and permission requirements for onSchemaChange, and SQL description updates. Addresses https://github.com/dataform-co/dataform/pull/2445#pullrequestreview-5918676679 --- cli/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/README.md b/cli/README.md index fc09c399f..1ce9de5ae 100644 --- a/cli/README.md +++ b/cli/README.md @@ -274,6 +274,10 @@ Compiles and executes the project against BigQuery (or validates it with `--dry- - `--timeout` bounds compilation only, not execution. If it's set without `--execution-timeout`, `run` prints a warning reminding you of that. - `--job-labels` override `defaultJobLabels`, action `jobLabels` and table `labels` that use the same key. - `--emit-lineage` requires the Data Lineage API (`datalineage.googleapis.com`) to be enabled and the `roles/datalineage.producer` role; for configuration and troubleshooting, see the [Data Lineage guide](../docs/lineage.md). +- **4-part Lakehouse Catalog targets (`project.catalog.namespace.table`):** + - The target catalog and namespace must already exist before running `dataform run` (Lakehouse namespaces are managed via BigLake APIs and are not created via `CREATE SCHEMA`). + - Incremental Lakehouse tables with `onSchemaChange` (`FAIL`, `EXTEND`, or `SYNCHRONIZE`) inspect the incoming query schema using a short-lived empty table in `defaultDataset`. Both `defaultProject` and `defaultDataset` must be configured in `workflow_settings.yaml` (or via CLI overrides) in the same location as the Lakehouse catalog (`defaultLocation`), and the execution identity needs permissions to create the dataset (if missing) and create/drop tables in `defaultDataset`. + - Table descriptions and top-level column descriptions on Lakehouse tables are applied after action execution via `ALTER TABLE` SQL statements (nested `STRUCT` field descriptions are skipped because BigQuery `ALTER COLUMN` only supports top-level columns). Exit codes: `0` on success, `1` if compilation fails, a BigQuery job would get more than 64 labels (checked before any action or unit test runs), unit tests fail (with `--run-tests`), the run fails, times out, or is cancelled (Ctrl+C).