diff --git a/cli/README.md b/cli/README.md index 9997715bd..1ce9de5ae 100644 --- a/cli/README.md +++ b/cli/README.md @@ -174,7 +174,7 @@ Compiles the project and prints a summary (or JSON / Graphviz DOT representation | `--quiet` | boolean | `false` | Less verbose compilation output. Mutually exclusive with --verbose. | | `--watch` | boolean | `false` | Watches the project directory and recompiles on changes. | | `--timeout` | string | `'60s'` | Duration to allow project compilation to complete, e.g. '1s', '10m'. | -| `--output-actions` | array | — | Filters the printed output to these actions (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name target. The whole project is still compiled. | +| `--output-actions` | array | — | Filters the printed output to these actions (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target. The whole project is still compiled. | | `--output-tags` | array | — | Filters the printed output to actions with any of these tags (comma-separated or repeatable). | | `--output-include-deps` | boolean | — | Also includes upstream dependencies of the selected actions in the output. Requires --output-actions or --output-tags. | | `--output-include-dependents` | boolean | — | Also includes downstream dependents of the selected actions in the output. Requires --output-actions or --output-tags. | @@ -250,7 +250,7 @@ Compiles and executes the project against BigQuery (or validates it with `--dry- | `--dry-run` | boolean | — | Validates the run SQL with BigQuery without applying any changes to the warehouse. | | `--run-tests` | boolean | — | Requires the project's unit tests to pass before executing; aborts the run if any test fails. | | `--action-retry-limit` | number | `0` | Retries idempotent actions up to this many times on failure. | -| `--actions` | array | — | Actions to run (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name target. | +| `--actions` | array | — | Actions to run (comma-separated or repeatable). Each entry is an action name or a fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target. | | `--tags` | array | — | Tags to filter the actions to run (comma-separated or repeatable). | | `--include-deps` | boolean | — | Also runs upstream dependencies of the selected actions. Requires --actions or --tags. | | `--include-dependents` | boolean | — | Also runs downstream dependents of the selected actions. Requires --actions or --tags. | @@ -274,6 +274,10 @@ Compiles and executes the project against BigQuery (or validates it with `--dry- - `--timeout` bounds compilation only, not execution. If it's set without `--execution-timeout`, `run` prints a warning reminding you of that. - `--job-labels` override `defaultJobLabels`, action `jobLabels` and table `labels` that use the same key. - `--emit-lineage` requires the Data Lineage API (`datalineage.googleapis.com`) to be enabled and the `roles/datalineage.producer` role; for configuration and troubleshooting, see the [Data Lineage guide](../docs/lineage.md). +- **4-part Lakehouse Catalog targets (`project.catalog.namespace.table`):** + - The target catalog and namespace must already exist before running `dataform run` (Lakehouse namespaces are managed via BigLake APIs and are not created via `CREATE SCHEMA`). + - Incremental Lakehouse tables with `onSchemaChange` (`FAIL`, `EXTEND`, or `SYNCHRONIZE`) inspect the incoming query schema using a short-lived empty table in `defaultDataset`. Both `defaultProject` and `defaultDataset` must be configured in `workflow_settings.yaml` (or via CLI overrides) in the same location as the Lakehouse catalog (`defaultLocation`), and the execution identity needs permissions to create the dataset (if missing) and create/drop tables in `defaultDataset`. + - Table descriptions and top-level column descriptions on Lakehouse tables are applied after action execution via `ALTER TABLE` SQL statements (nested `STRUCT` field descriptions are skipped because BigQuery `ALTER COLUMN` only supports top-level columns). Exit codes: `0` on success, `1` if compilation fails, a BigQuery job would get more than 64 labels (checked before any action or unit test runs), unit tests fail (with `--run-tests`), the run fails, times out, or is cancelled (Ctrl+C). diff --git a/cli/api/BUILD b/cli/api/BUILD index c6387ab27..64cc4d6a8 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -58,6 +58,7 @@ ts_test_suite( "commands/compile_copy_filter_test.ts", "commands/jit/rpc_test.ts", "commands/prune_test.ts", + "commands/run_test.ts", "dbadapters/bigquery_test.ts", "execution_sql_test.ts", ] + glob(["lineage/*_test.ts"]), @@ -65,6 +66,7 @@ ts_test_suite( deps = [ ":api", "//cli/api/lineage/testing", + "//cli/api/utils", "//core", "//protos:ts", "//testing", diff --git a/cli/api/commands/prune_test.ts b/cli/api/commands/prune_test.ts index 87d9f6e7b..fa54ebc8b 100644 --- a/cli/api/commands/prune_test.ts +++ b/cli/api/commands/prune_test.ts @@ -167,4 +167,78 @@ suite("prune", () => { }), ); }); + + const LAKEHOUSE_TARGET: dataform.ITarget = { + database: "p", + schema: "cat.ns", + name: "orders", + }; + const LAKEHOUSE_ACTION: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: LAKEHOUSE_TARGET, + canonicalTarget: LAKEHOUSE_TARGET, + query: "select 1 as order_id", + dependencyTargets: [], + }; + const DOWNSTREAM_OF_LAKEHOUSE_TARGET: dataform.ITarget = { + database: "p", + schema: "d", + name: "order_summary", + }; + const DOWNSTREAM_OF_LAKEHOUSE_ACTION: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: DOWNSTREAM_OF_LAKEHOUSE_TARGET, + canonicalTarget: DOWNSTREAM_OF_LAKEHOUSE_TARGET, + query: "select * from `p.cat.ns.orders`", + dependencyTargets: [LAKEHOUSE_TARGET], + }; + + test("--actions matches 4-part Lakehouse target by database.catalog.namespace.name", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect(asPlainObject(prune(compiledGraph, { actions: ["p.cat.ns.orders"] }))).deep.equals( + asPlainObject({ + tables: [LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET], + }), + ); + }); + + test("--actions matches 4-part Lakehouse target by bare name and resolves includeDependents", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect( + asPlainObject(prune(compiledGraph, { actions: ["orders"], includeDependents: true })), + ).deep.equals(asPlainObject(compiledGraph)); + }); + + test("--actions with includeDependencies pulls in upstream 4-part Lakehouse table", () => { + const compiledGraph: dataform.ICompiledGraph = { + tables: [LAKEHOUSE_ACTION, DOWNSTREAM_OF_LAKEHOUSE_ACTION], + operations: [], + assertions: [], + propertyGraphs: [], + targets: [LAKEHOUSE_TARGET, DOWNSTREAM_OF_LAKEHOUSE_TARGET], + }; + expect( + asPlainObject( + prune(compiledGraph, { actions: ["p.d.order_summary"], includeDependencies: true }), + ), + ).deep.equals(asPlainObject(compiledGraph)); + }); }); diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts index 371760d6a..c64a5c09e 100644 --- a/cli/api/commands/run.ts +++ b/cli/api/commands/run.ts @@ -18,7 +18,7 @@ import { Timer } from "df/cli/api/utils/timer"; import { Flags } from "df/common/flags"; import { retry } from "df/common/promises"; import { deepClone, equals } from "df/common/protos"; -import { targetAsReadableString, targetStringifier } from "df/core/targets"; +import { isCatalogSchema, targetAsReadableString, targetStringifier } from "df/core/targets"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -358,6 +358,8 @@ export class Runner { (action.tasks.length > 0 || !!action.jitCode) && !!action.target && !!action.target.schema && + // Lakehouse catalog.namespace targets cannot be created via CREATE SCHEMA; they must exist before the run. + !isCatalogSchema(action.target.schema) && writesToTarget(action), ) .forEach(({ target }) => { @@ -620,7 +622,9 @@ export class Runner { action.type === "table" ) { try { - await this.dbadapter.setMetadata(action); + await this.dbadapter.setMetadata(action, { + bigquery: this.getBigQueryExecutionOptions(action), + }); } catch (e) { // TODO: Setting the metadata is not a task itself, so we have nowhere to surface this error cleanly. // For now, we can attach the error to the last task in the action so it gets diff --git a/cli/api/commands/run_test.ts b/cli/api/commands/run_test.ts new file mode 100644 index 000000000..09d967f46 --- /dev/null +++ b/cli/api/commands/run_test.ts @@ -0,0 +1,206 @@ +import { expect } from "chai"; +import Long from "long"; +import { anyString, anything, instance, mock, verify, when } from "ts-mockito"; + +import { build } from "df/cli/api/commands/build"; +import { Runner } from "df/cli/api/commands/run"; +import { BigQueryDbAdapter } from "df/cli/api/dbadapters/bigquery"; +import { wrapWithCreateSchemaIfNotExists } from "df/cli/api/utils/query_transforms"; +import { dataform } from "df/protos/ts"; +import { suite, test } from "df/testing"; + +suite("Runner with 4-part Lakehouse targets", () => { + const projectConfig: dataform.IProjectConfig = { + warehouse: "bigquery", + defaultDatabase: "my-gcp-project", + defaultSchema: "default_dataset", + }; + + test("skips schemas() lookup and CREATE SCHEMA wrapper when all actions are 4-part Lakehouse targets", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const executedStatements: string[] = []; + when(mockedDbAdapter.execute(anyString(), anything())).thenCall(async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }); + + const lakehouseStatement = + "create or replace table `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` as select 1 as id"; + const opStatement = + "insert into `my-gcp-project.ecommerce_catalog.sales_dw.audit_log` values (1)"; + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + tasks: [{ type: "statement", statement: lakehouseStatement }], + dependencyTargets: [], + }, + { + type: "operation", + hasOutput: true, + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "audit_log", + }, + tasks: [{ type: "statement", statement: opStatement }], + dependencyTargets: [], + }, + ], + }); + + const realRunResult = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + expect(realRunResult.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + + const dryRunResult = await new Runner(instance(mockedDbAdapter), graph, { + bigquery: { dryRun: true }, + }) + .execute() + .result(); + expect(dryRunResult.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + + verify(mockedDbAdapter.schemas(anything())).never(); + expect(executedStatements).deep.equals([ + lakehouseStatement, + opStatement, + lakehouseStatement, + opStatement, + ]); + }); + + test("wraps missing 3-part dataset with CREATE SCHEMA while leaving 4-part Lakehouse target unwrapped", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const executedStatements: string[] = []; + when(mockedDbAdapter.schemas("my-gcp-project")).thenResolve([]); + when(mockedDbAdapter.execute(anyString(), anything())).thenCall(async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }); + + const standardStatement = + "create or replace table `my-gcp-project.new_dataset.std_table` as select 1"; + const lakehouseStatement = + "create or replace table `my-gcp-project.my_cat.my_ns.lh_table` as select 1"; + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { database: "my-gcp-project", schema: "new_dataset", name: "std_table" }, + tasks: [{ type: "statement", statement: standardStatement }], + dependencyTargets: [], + }, + { + type: "table", + tableType: "table", + target: { database: "my-gcp-project", schema: "my_cat.my_ns", name: "lh_table" }, + tasks: [{ type: "statement", statement: lakehouseStatement }], + dependencyTargets: [], + }, + ], + }); + + const result = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + + expect(result.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + verify(mockedDbAdapter.schemas("my-gcp-project")).once(); + expect(executedStatements).to.have.members([ + wrapWithCreateSchemaIfNotExists("my-gcp-project", "new_dataset", standardStatement), + lakehouseStatement, + ]); + }); + + test("build() fetches 4-part target warehouse state via dbadapter.table() and builds incremental DML", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + const lakehouseTarget: dataform.ITarget = { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }; + when(mockedDbAdapter.table(anything())).thenResolve( + dataform.TableMetadata.create({ + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + fields: [{ name: "order_id", primitive: dataform.Field.Primitive.INTEGER }], + lastUpdatedMillis: Long.ZERO, + }), + ); + + const compiledGraph = dataform.CompiledGraph.create({ + projectConfig, + dataformCoreVersion: "3.1.5", + tables: [ + { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 2 AS order_id", + uniqueKey: ["order_id"], + }, + ], + }); + + const executionGraph = await build( + compiledGraph, + { fullRefresh: false }, + instance(mockedDbAdapter), + ); + + expect(executionGraph.actions.length).equals(1); + expect(executionGraph.actions[0].tasks[0].statement).to.include( + "merge `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + }); + + test("delegates setMetadata() with execution options on 4-part Lakehouse tables with descriptions", async () => { + const mockedDbAdapter = mock(BigQueryDbAdapter); + when(mockedDbAdapter.execute(anyString(), anything())).thenResolve({ rows: [], metadata: {} }); + when(mockedDbAdapter.setMetadata(anything(), anything())).thenResolve(); + + const graph = dataform.ExecutionGraph.create({ + projectConfig, + runConfig: {}, + warehouseState: {}, + actions: [ + { + type: "table", + tableType: "table", + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + actionDescriptor: { + description: "Lakehouse orders table", + columns: [{ path: ["order_id"], description: "Order ID" }], + }, + tasks: [ + { + type: "statement", + statement: + "create or replace table `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` as select 1 as order_id", + }, + ], + dependencyTargets: [], + }, + ], + }); + + const result = await new Runner(instance(mockedDbAdapter), graph).execute().result(); + expect(result.status).equals(dataform.RunResult.ExecutionStatus.SUCCESSFUL); + verify(mockedDbAdapter.setMetadata(anything(), anything())).once(); + }); +}); diff --git a/cli/api/dbadapters/bigquery.ts b/cli/api/dbadapters/bigquery.ts index c07f8d35c..91d6fdce6 100644 --- a/cli/api/dbadapters/bigquery.ts +++ b/cli/api/dbadapters/bigquery.ts @@ -22,6 +22,7 @@ import { parseBigqueryEvalError } from "df/cli/api/utils/error_parsing"; import { LimitedResultSet } from "df/cli/api/utils/results"; import { coerceAsError } from "df/common/errors/errors"; import { retry } from "df/common/promises"; +import { isLakehouseTarget } from "df/core/targets"; import { dataform } from "df/protos/ts"; const CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; @@ -376,8 +377,10 @@ export class BigQueryDbAdapter implements IDbAdapter { ? dataform.TableMetadata.Type.VIEW : dataform.TableMetadata.Type.UNKNOWN, target, - fields: metadata.schema.fields?.map((field) => convertField(field)), - lastUpdatedMillis: Long.fromString(metadata.lastModifiedTime), + fields: metadata.schema?.fields?.map((field) => convertField(field)), + lastUpdatedMillis: metadata.lastModifiedTime + ? Long.fromString(metadata.lastModifiedTime) + : Long.ZERO, description: metadata.description, labels: metadata.labels, bigquery: { @@ -387,6 +390,18 @@ export class BigQueryDbAdapter implements IDbAdapter { } public async deleteTable(target: dataform.ITarget): Promise { + if (isLakehouseTarget(target)) { + if (!target.database) { + throw new Error( + `Cannot delete Lakehouse table "${target.schema}.${target.name}" without a target database (project).`, + ); + } + await this.execute( + `drop table if exists \`${target.database}.${target.schema}.${target.name}\``, + ); + return; + } + await this.getClient(target.database) .dataset(target.schema) .table(target.name) @@ -401,8 +416,15 @@ export class BigQueryDbAdapter implements IDbAdapter { return data[0].map((dataset) => dataset.id); } - public async setMetadata(action: dataform.IExecutionAction): Promise { + public async setMetadata( + action: dataform.IExecutionAction, + options?: { bigquery?: IBigQueryExecutionOptions }, + ): Promise { const { target, actionDescriptor } = action; + if (isLakehouseTarget(target)) { + await this.setLakehouseMetadata(target, actionDescriptor, options); + return; + } const metadata = await this.getMetadata(target); const schemaWithDescription = addDescriptionToMetadata( @@ -417,6 +439,40 @@ export class BigQueryDbAdapter implements IDbAdapter { }); } + private async setLakehouseMetadata( + target: dataform.ITarget, + actionDescriptor?: dataform.IActionDescriptor | null, + options?: { bigquery?: IBigQueryExecutionOptions }, + ): Promise { + if (!target.database) { + throw new Error( + `Cannot set metadata on Lakehouse table "${target.schema}.${target.name}" without a target database (project).`, + ); + } + // BigQuery's tables.patch REST endpoint rejects 4-part Lakehouse Iceberg tables, so apply + // table and top-level column descriptions via SQL DDL instead. BigQuery does not allow + // mixing table SET OPTIONS and ALTER COLUMN in a single ALTER TABLE statement. + const qualifiedTable = `\`${target.database}.${target.schema}.${target.name}\``; + const statements: string[] = []; + if (actionDescriptor?.description != null && actionDescriptor.description !== "") { + statements.push( + `ALTER TABLE ${qualifiedTable} SET OPTIONS(description = ${escapeSqlStringLiteral(actionDescriptor.description)});`, + ); + } + const alterColumns = (actionDescriptor?.columns || []) + .filter((col) => col.path?.length === 1 && col.description != null && col.description !== "") + .map( + (col) => + `ALTER COLUMN IF EXISTS \`${col.path[0]}\` SET OPTIONS(description = ${escapeSqlStringLiteral(col.description)})`, + ); + if (alterColumns.length > 0) { + statements.push(`ALTER TABLE ${qualifiedTable} ${alterColumns.join(", ")};`); + } + if (statements.length > 0) { + await this.execute(statements.join("\n"), { bigquery: options?.bigquery }); + } + } + private async getMetadata(target: dataform.ITarget): Promise { try { const table = await this.getClient(target.database) @@ -425,7 +481,7 @@ export class BigQueryDbAdapter implements IDbAdapter { .getMetadata(); return table?.[0]; } catch (e) { - if (e?.errors?.[0]?.reason === "notFound") { + if (e?.code === 404 || e?.errors?.[0]?.reason === "notFound") { // if the table can't be found, just return null return null; } @@ -724,3 +780,11 @@ function createBigQueryError(jobMetadata: any): IBigQueryError { } return error; } + +function escapeSqlStringLiteral(value: string): string { + return `'${value + .replace(/\\/g, "\\\\") + .replace(/'/g, "\\'") + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r")}'`; +} diff --git a/cli/api/dbadapters/bigquery_test.ts b/cli/api/dbadapters/bigquery_test.ts index 13225171a..644f4b034 100644 --- a/cli/api/dbadapters/bigquery_test.ts +++ b/cli/api/dbadapters/bigquery_test.ts @@ -1,7 +1,8 @@ -import { Dataset, Table } from "@google-cloud/bigquery"; +import { BigQuery, Dataset, Table } from "@google-cloud/bigquery"; import { expect } from "chai"; import { EventEmitter } from "events"; import { AuthClient, gaxios, Impersonated } from "google-auth-library"; +import Long from "long"; import { anything, instance, mock, when } from "ts-mockito"; import { @@ -428,4 +429,197 @@ suite("BigQueryDbAdapter", () => { name: tableName, }); }); + + test("table() handles 4-part Lakehouse datasetId including when lastModifiedTime is absent", async () => { + const projectId = "my-gcp-project"; + const lakehouseDatasetId = "ecommerce_catalog.sales_dw"; + const tableName = "customer_orders"; + const mockBigQuery: any = { + dataset: (datasetId: string) => { + expect(datasetId).to.equal(lakehouseDatasetId); + return { + table: (id: string) => { + expect(id).to.equal(tableName); + return { + getMetadata: () => + Promise.resolve([ + { + kind: "bigquery#table", + id: `${projectId}:${lakehouseDatasetId}.${tableName}`, + tableReference: { + projectId, + datasetId: lakehouseDatasetId, + tableId: tableName, + }, + type: "TABLE", + location: "US", + biglakeConfiguration: { + storageUri: "gs://my-lakehouse-bucket/sales_dw/customer_orders/123/", + }, + schema: { + fields: [ + { name: "order_id", type: "INTEGER" }, + { name: "customer_id", type: "INTEGER" }, + ], + }, + // lastModifiedTime intentionally omitted (UIT / Lakehouse response) + }, + ]), + }; + }, + }; + }, + }; + const adapter = new BigQueryDbAdapter(dataform.BigQuery.create({ projectId, location: "US" }), { + clientProvider: () => mockBigQuery, + }); + const target = { database: projectId, schema: lakehouseDatasetId, name: tableName }; + const result = await adapter.table(target); + expect(result).to.not.equal(null); + expect(result.type).to.equal(dataform.TableMetadata.Type.TABLE); + expect(result.target).to.eql(target); + expect(result.fields.map((f) => f.name)).to.eql(["order_id", "customer_id"]); + expect(result.lastUpdatedMillis.equals(Long.ZERO)).to.equal(true); + }); + + test("table() returns null on 404 for nonexistent 4-part Lakehouse table", async () => { + const mockBigQuery: any = { + dataset: () => ({ + table: () => ({ + getMetadata: () => Promise.reject({ code: 404, message: "Not found" }), + }), + }), + }; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "my-gcp-project", location: "US" }), + { clientProvider: () => mockBigQuery }, + ); + const result = await adapter.table({ + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "missing_table", + }); + expect(result).to.equal(null); + }); + + test("setMetadata() applies table and top-level column descriptions via ALTER TABLE SQL for 4-part Lakehouse targets", async () => { + let datasetCalled = false; + const mockBigQuery = { + dataset: () => { + datasetCalled = true; + throw new Error("tables.patch must not be called for Lakehouse targets"); + }, + } as unknown as BigQuery; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "my-gcp-project", location: "US" }), + { concurrencyLimit: 1, clientProvider: () => mockBigQuery }, + ); + const executedCalls: Array<{ statement: string; location?: string }> = []; + adapter.execute = async (statement: string, options?: any) => { + executedCalls.push({ statement, location: options?.bigquery?.location }); + return { rows: [], metadata: {} }; + }; + await adapter.setMetadata( + dataform.ExecutionAction.create({ + target: { + database: "my-gcp-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + actionDescriptor: { + description: "Orders' main\ntable", + columns: [ + { path: ["order_id"], description: "Primary key" }, + { path: ["struct_col", "nested"], description: "Ignored nested field" }, + ], + }, + }), + { bigquery: { location: "us-central1" } }, + ); + expect(datasetCalled).to.equal(false); + expect(executedCalls).to.eql([ + { + statement: + "ALTER TABLE `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` SET OPTIONS(description = 'Orders\\' main\\ntable');\n" + + "ALTER TABLE `my-gcp-project.ecommerce_catalog.sales_dw.customer_orders` ALTER COLUMN IF EXISTS `order_id` SET OPTIONS(description = 'Primary key');", + location: "us-central1", + }, + ]); + }); + + test("deleteTable() executes 4-part DROP TABLE IF EXISTS for Lakehouse targets", async () => { + let restDeleteCalled = false; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "creds-project", location: "US" }), + { + clientProvider: () => + ({ + dataset: () => ({ + table: () => ({ + delete: () => { + restDeleteCalled = true; + return Promise.resolve(); + }, + }), + }), + }) as any, + }, + ); + const executedStatements: string[] = []; + adapter.execute = async (statement: string) => { + executedStatements.push(statement); + return { rows: [], metadata: {} }; + }; + await adapter.deleteTable({ + database: "explicit-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }); + let thrownError: Error | undefined; + try { + await adapter.deleteTable({ + schema: "ecommerce_catalog.sales_dw", + name: "missing_project_orders", + }); + } catch (e) { + thrownError = e as Error; + } + expect(thrownError?.message).to.include("without a target database"); + expect(restDeleteCalled).to.equal(false); + expect(executedStatements).to.eql([ + "drop table if exists `explicit-project.ecommerce_catalog.sales_dw.customer_orders`", + ]); + }); + + test("deleteTable() uses REST table.delete for standard 3-part targets", async () => { + let deletedWithIgnoreNotFound = false; + const adapter = new BigQueryDbAdapter( + dataform.BigQuery.create({ projectId: "p", location: "US" }), + { + clientProvider: () => + ({ + dataset: (schema: string) => { + expect(schema).to.equal("standard_dataset"); + return { + table: (name: string) => { + expect(name).to.equal("standard_table"); + return { + delete: (opts: { ignoreNotFound: boolean }) => { + deletedWithIgnoreNotFound = opts.ignoreNotFound; + return Promise.resolve(); + }, + }; + }, + }; + }, + }) as any, + }, + ); + await adapter.deleteTable({ + database: "p", + schema: "standard_dataset", + name: "standard_table", + }); + expect(deletedWithIgnoreNotFound).to.equal(true); + }); }); diff --git a/cli/api/dbadapters/execution_sql.ts b/cli/api/dbadapters/execution_sql.ts index 1be7c452b..ecf5c8b44 100644 --- a/cli/api/dbadapters/execution_sql.ts +++ b/cli/api/dbadapters/execution_sql.ts @@ -4,6 +4,7 @@ import { Task, Tasks } from "df/cli/api/dbadapters/tasks"; import { concatenateQueries } from "df/cli/api/utils/query_transforms"; import { ErrorWithCause } from "df/common/errors/errors"; import { CompilationSql } from "df/core/compilation_sql"; +import { isLakehouseTarget } from "df/core/targets"; import { tableTypeEnumToString } from "df/core/utils"; import { dataform } from "df/protos/ts"; @@ -138,7 +139,7 @@ from (${query}) as insertions`; this.preOps(table, runConfig, tableMetadata).forEach((statement) => tasks.add(statement)); const baseTableType = this.baseTableType(table.enumType); - if (tableMetadata && tableMetadata.type !== baseTableType) { + if (!isLakehouseTarget(table.target) && tableMetadata && tableMetadata.type !== baseTableType) { tasks.add( Task.statement(this.dropIfExists(table.target, this.oppositeTableType(baseTableType))), ); @@ -146,17 +147,19 @@ from (${query}) as insertions`; if (table.enumType === dataform.TableType.INCREMENTAL) { if (!this.shouldWriteIncrementally(table, runConfig, tableMetadata)) { - tasks.add(Task.statement(this.createOrReplace(table))); + const allowReplace = + !isLakehouseTarget(table.target) || Boolean(runConfig.fullRefresh && !table.protected); + tasks.add(Task.statement(this.createOrReplace(table, allowReplace))); } else { const onSchemaChange = table.onSchemaChange ?? dataform.OnSchemaChange.IGNORE; switch (onSchemaChange) { case dataform.OnSchemaChange.FAIL: case dataform.OnSchemaChange.EXTEND: case dataform.OnSchemaChange.SYNCHRONIZE: - this.buildIncrementalSchemaChangeTasks(tasks, table); + this.buildIncrementalSchemaChangeTasks(tasks, table, tableMetadata); // Fall through to run the static DML after the procedure alters the schema case dataform.OnSchemaChange.IGNORE: - const columns = tableMetadata?.fields.map((f) => f.name) || []; + const columns = tableMetadata?.fields?.map((f) => f.name) || []; tasks.add(Task.statement(this.getIncrementalDmlStatement(table, columns))); break; } @@ -231,7 +234,16 @@ from (${query}) as insertions`; return `and ${validPredicates.map((p) => `(${p})`).join(" and ")}`; } - private buildIncrementalSchemaChangeTasks(tasks: Tasks, table: dataform.ITable) { + private buildIncrementalSchemaChangeTasks( + tasks: Tasks, + table: dataform.ITable, + tableMetadata?: dataform.ITableMetadata, + ) { + if (isLakehouseTarget(table.target)) { + this.buildLakehouseIncrementalSchemaChangeTasks(tasks, table, tableMetadata); + return; + } + const uniqueId = this.uniqueIdGenerator(); const emptyTempTableTarget = { @@ -260,6 +272,70 @@ END;`; tasks.add(Task.statement(callProcedureSql)); } + private buildLakehouseIncrementalSchemaChangeTasks( + tasks: Tasks, + table: dataform.ITable, + tableMetadata?: dataform.ITableMetadata, + ) { + if (!tableMetadata?.fields?.length) { + throw new Error( + `Cannot apply onSchemaChange to Lakehouse table ${this.resolveTarget(table.target)} because warehouse metadata is missing schema fields.`, + ); + } + const uniqueId = this.uniqueIdGenerator(); + const tempTableName = `${table.target.name}_df_temp_${uniqueId}_empty`; + const emptyTempTableTarget = this.resolveScratchTarget(tempTableName); + const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); + // Note: `existingColumns` is a build-time snapshot from `tables.get` metadata (whereas the + // standard 3-part path queries INFORMATION_SCHEMA at execution time), so schema changes made + // by `incrementalPreOps` are not visible here. + const existingColumns = tableMetadata.fields.map((f) => f.name); + const scriptBody = this.incrementalSchemaChangeBody( + table, + this.resolveTarget(table.target), + emptyTempTableTarget, + existingColumns, + ); + const locationOption = this.project?.defaultLocation + ? ` OPTIONS(location=${this.CompilationSql.sqlString(this.project.defaultLocation)})` + : ""; + const createScratchSchemaSql = `CREATE SCHEMA IF NOT EXISTS \`${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}\`${locationOption};`; + const scriptSql = `${createScratchSchemaSql} +BEGIN +${scriptBody} +EXCEPTION WHEN ERROR THEN + DROP TABLE IF EXISTS ${emptyTempTableName}; + RAISE; +END;`; + tasks.add(Task.statement(scriptSql)); + } + + private resolveScratchTarget(tempTableName: string): dataform.ITarget { + const baseDefaultDatabase = this.project?.defaultDatabase; + if (!baseDefaultDatabase) { + throw new Error( + "When defining an incremental Lakehouse table with onSchemaChange, 'defaultProject' must be defined in workflow_settings.yaml.", + ); + } + const baseDefaultSchema = this.project?.defaultSchema; + if (!baseDefaultSchema) { + throw new Error( + "When defining an incremental Lakehouse table, 'defaultDataset' must be defined in workflow_settings.yaml.", + ); + } + const scratchDatabase = this.project?.databaseSuffix + ? `${baseDefaultDatabase}_${this.project.databaseSuffix}` + : baseDefaultDatabase; + const scratchSchema = this.project?.schemaSuffix + ? `${baseDefaultSchema}_${this.project.schemaSuffix}` + : baseDefaultSchema; + return { + database: scratchDatabase, + schema: scratchSchema, + name: tempTableName, + }; + } + private createProcedureName(target: dataform.ITarget, uniqueId: string): string { return this.resolveTarget({ ...target, @@ -293,10 +369,17 @@ DECLARE columns_removed ARRAY;`; return sql; } - private createEmptyTempTableSql(emptyTempTableName: string, query: string): string { + private createEmptyTempTableSql( + emptyTempTableName: string, + query: string, + setExpiration = false, + ): string { + const optionsClause = setExpiration + ? "\nOPTIONS(expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))" + : ""; return ` -- Create empty table to extract schema of new query. -CREATE OR REPLACE TABLE ${emptyTempTableName} AS ( +CREATE OR REPLACE TABLE ${emptyTempTableName}${optionsClause} AS ( SELECT * FROM (${query}) AS insertions LIMIT 0 );`; } @@ -304,18 +387,30 @@ CREATE OR REPLACE TABLE ${emptyTempTableName} AS ( private compareSchemasSql( target: dataform.ITarget, emptyTempTableTarget: dataform.ITarget, + existingColumns?: string[], ): string { - return ` --- Compare schemas -SET dataform_columns = ( + const setDataformColumnsSql = + existingColumns !== undefined + ? `SET dataform_columns = ARRAY[${existingColumns + .map((col) => this.CompilationSql.sqlString(col)) + .join(", ")}];` + : `SET dataform_columns = ( SELECT IFNULL(ARRAY_AGG(DISTINCT column_name), []) FROM \`${target.database}.${target.schema}.INFORMATION_SCHEMA.COLUMNS\` WHERE table_name = '${target.name}' -); +);`; + + const tempInfoSchemaPrefix = emptyTempTableTarget.database + ? `${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}` + : emptyTempTableTarget.schema; + + return ` +-- Compare schemas +${setDataformColumnsSql} SET temp_table_columns = ( SELECT IFNULL(ARRAY_AGG(STRUCT(column_name, data_type)), []) - FROM \`${emptyTempTableTarget.database}.${emptyTempTableTarget.schema}.INFORMATION_SCHEMA.COLUMNS\` + FROM \`${tempInfoSchemaPrefix}.INFORMATION_SCHEMA.COLUMNS\` WHERE table_name = '${emptyTempTableTarget.name}' ); @@ -417,14 +512,15 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; table: dataform.ITable, qualifiedTargetTableName: string, emptyTempTableTarget: dataform.ITarget, + existingColumns?: string[], ): string { const emptyTempTableName = this.resolveTarget(emptyTempTableTarget); const query = this.getIncrementalQuery(table); const onSchemaChange = table.onSchemaChange || dataform.OnSchemaChange.IGNORE; const statements: string[] = [ this.declareSchemaChangeVariablesSql(onSchemaChange), - this.createEmptyTempTableSql(emptyTempTableName, query), - this.compareSchemasSql(table.target, emptyTempTableTarget), + this.createEmptyTempTableSql(emptyTempTableName, query, isLakehouseTarget(table.target)), + this.compareSchemasSql(table.target, emptyTempTableTarget, existingColumns), this.applySchemaChangeStrategySql(table, qualifiedTargetTableName), this.cleanupSql(emptyTempTableName), ]; @@ -432,7 +528,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; return statements.join("\n\n"); } - private createOrReplace(table: dataform.ITable) { + private createOrReplace(table: dataform.ITable, allowReplace = true) { const options = []; if (table.bigquery && table.bigquery.partitionBy && table.bigquery.partitionExpirationDays) { options.push(`partition_expiration_days=${table.bigquery.partitionExpirationDays}`); @@ -446,7 +542,7 @@ DROP TABLE IF EXISTS ${emptyTempTableName}; } } - return `create or replace ${table.materialized ? "materialized " : ""}${this.tableTypeAsSql( + return `create ${allowReplace ? "or replace " : ""}${table.materialized ? "materialized " : ""}${this.tableTypeAsSql( this.baseTableType(table.enumType), )} ${this.resolveTarget(table.target)} ${ table.bigquery && table.bigquery.partitionBy diff --git a/cli/api/dbadapters/index.ts b/cli/api/dbadapters/index.ts index af2f13b85..aa2610be9 100644 --- a/cli/api/dbadapters/index.ts +++ b/cli/api/dbadapters/index.ts @@ -63,5 +63,16 @@ export interface IDbAdapter extends IDbClient { table(target: dataform.ITarget): Promise; deleteTable(target: dataform.ITarget): Promise; - setMetadata(action: dataform.IExecutionAction): Promise; + setMetadata( + action: dataform.IExecutionAction, + options?: { + bigquery?: { + labels?: { [label: string]: string }; + location?: string; + jobPrefix?: string; + dryRun?: boolean; + reservation?: string; + }; + }, + ): Promise; } diff --git a/cli/api/execution_sql_test.ts b/cli/api/execution_sql_test.ts index f4d4e9520..7bebc945f 100644 --- a/cli/api/execution_sql_test.ts +++ b/cli/api/execution_sql_test.ts @@ -278,3 +278,339 @@ suite("ExecutionSql for operations", () => { ).deep.equals([]); }); }); + +suite("ExecutionSql for 4-part Lakehouse targets", () => { + const executionSql = new ExecutionSql( + { + defaultDatabase: "project-id", + defaultSchema: "default_dataset", + }, + "3.1.5", + () => "test_uuid", + ); + + const lakehouseTarget: dataform.ITarget = { + database: "project-id", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }; + + const lakehouseMetadata: dataform.ITableMetadata = { + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + fields: [ + { name: "order_id", primitive: dataform.Field.Primitive.INTEGER }, + { name: "customer_id", primitive: dataform.Field.Primitive.INTEGER }, + ], + }; + + test("emits 4-part CREATE OR REPLACE TABLE and skips incompatible-type DROP VIEW IF EXISTS", () => { + const table: dataform.ITable = { + type: "table", + enumType: dataform.TableType.TABLE, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + bigquery: { + partitionBy: "BUCKET(order_id, 16)", + }, + }; + const incompatibleMetadata: dataform.ITableMetadata = { + type: dataform.TableMetadata.Type.UNKNOWN, + target: lakehouseTarget, + fields: [], + }; + + const tasks = executionSql + .publishTasks(table, { fullRefresh: false }, incompatibleMetadata) + .build(); + + expect(tasks.length).to.equal(1); + expect(tasks[0].statement).to.not.include("drop view"); + expect(tasks[0].statement).to.equal( + "create or replace table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "partition by BUCKET(order_id, 16) as SELECT 1 AS order_id, 2 AS customer_id", + ); + }); + + test("emits plain CREATE TABLE on first run / protected and CREATE OR REPLACE TABLE on fullRefresh of 4-part incremental table", () => { + const incTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 3 AS order_id, 4 AS customer_id", + }; + const firstRunSql = executionSql + .publishTasks(incTable, { fullRefresh: false }) + .build()[0].statement; + const protectedFullRefreshSql = executionSql + .publishTasks({ ...incTable, protected: true }, { fullRefresh: true }) + .build()[0].statement; + const fullRefreshSql = executionSql + .publishTasks(incTable, { fullRefresh: true }, lakehouseMetadata) + .build()[0].statement; + expect(firstRunSql).to.equal( + "create table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id", + ); + expect(protectedFullRefreshSql).to.equal( + "create table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id", + ); + expect(fullRefreshSql).to.equal( + "create or replace table `project-id.ecommerce_catalog.sales_dw.customer_orders` " + + "as SELECT 1 AS order_id, 2 AS customer_id", + ); + }); + + test("emits valid 4-part DML for incremental INSERT, MERGE, and INSERT_OVERWRITE", () => { + const baseIncTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 3 AS order_id, 4 AS customer_id", + }; + + const insertSql = executionSql + .publishTasks(baseIncTable, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + expect(insertSql).to.include( + "insert into `project-id.ecommerce_catalog.sales_dw.customer_orders`", + ); + expect(insertSql).to.include("(`order_id`,`customer_id`)"); + + const mergeSql = executionSql + .publishTasks( + { ...baseIncTable, uniqueKey: ["order_id"] }, + { fullRefresh: false }, + lakehouseMetadata, + ) + .build()[0].statement; + expect(mergeSql).to.include( + "merge `project-id.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + + const insertOverwriteSql = executionSql + .publishTasks( + { + ...baseIncTable, + incrementalStrategy: dataform.IncrementalStrategy.INSERT_OVERWRITE, + bigquery: { partitionBy: "DATE(ts)" }, + }, + { fullRefresh: false }, + lakehouseMetadata, + ) + .build()[0].statement; + expect(insertOverwriteSql).to.include( + "CREATE OR REPLACE TEMP TABLE `staging_table_temp_test_uuid`", + ); + expect(insertOverwriteSql).to.include( + "MERGE `project-id.ecommerce_catalog.sales_dw.customer_orders` DATAFORM_DEST", + ); + }); + + test("handles onSchemaChange = FAIL on 4-part target without procedure in catalog.namespace or 5-part INFORMATION_SCHEMA", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 2 AS customer_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.FAIL, + }; + + const sql = executionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.df_osc_"); + expect(sql).to.include( + "CREATE OR REPLACE TABLE `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include("SET dataform_columns = ARRAY['order_id', 'customer_id'];"); + expect(sql).to.include("FROM `project-id.default_dataset.INFORMATION_SCHEMA.COLUMNS`"); + expect(sql).to.include("Schema mismatch defined by on_schema_change = 'FAIL'."); + expect(sql).to.include( + "EXCEPTION WHEN ERROR THEN\n DROP TABLE IF EXISTS `project-id.default_dataset.customer_orders_df_temp_test_uuid_empty`;\n RAISE;\nEND;", + ); + expect(sql).to.include("CREATE SCHEMA IF NOT EXISTS `project-id.default_dataset`;\nBEGIN"); + expect(sql).to.include( + "OPTIONS(expiration_timestamp = TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY))", + ); + }); + + test("handles onSchemaChange = EXTEND on 4-part target and applies schemaSuffix to scratch dataset", () => { + const suffixedExecutionSql = new ExecutionSql( + { + defaultDatabase: "project-id", + defaultSchema: "default_dataset", + schemaSuffix: "staging", + }, + "3.1.5", + () => "test_uuid", + ); + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 2 AS customer_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.EXTEND, + }; + + const sql = suffixedExecutionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.include( + "CREATE OR REPLACE TABLE `project-id.default_dataset_staging.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include("FROM `project-id.default_dataset_staging.INFORMATION_SCHEMA.COLUMNS`"); + expect(sql).to.include( + '"ALTER TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` "', + ); + expect(sql).to.include("ADD COLUMN IF NOT EXISTS"); + }); + + test("handles onSchemaChange = SYNCHRONIZE on 4-part target with DECLARE statements first in BEGIN block", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + uniqueKey: ["order_id"], + query: "SELECT 1 AS order_id, 2 AS customer_id", + incrementalQuery: "SELECT 1 AS order_id, 'new' AS status", + onSchemaChange: dataform.OnSchemaChange.SYNCHRONIZE, + }; + + const sql = executionSql + .publishTasks(table, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.not.include("CREATE OR REPLACE PROCEDURE"); + expect(sql).to.not.include("ecommerce_catalog.sales_dw.INFORMATION_SCHEMA"); + expect(sql).to.include("DECLARE invalid_removed_columns ARRAY;"); + expect(sql).to.include( + "Cannot drop columns %T as they are part of the unique key for table `project-id.ecommerce_catalog.sales_dw.customer_orders`", + ); + expect(sql).to.include("DROP COLUMN IF EXISTS"); + expect(sql).to.include("ADD COLUMN IF NOT EXISTS"); + + const scriptBody = sql.split("BEGIN\n")[1].split("\nEXCEPTION WHEN ERROR THEN")[0]; + const statements = scriptBody + .split(";") + .map((s) => + s + .split("\n") + .filter((line) => !line.trim().startsWith("--")) + .join("\n") + .trim(), + ) + .filter((s) => s.length > 0); + + let seenNonDeclare = false; + for (const stmt of statements) { + if (stmt.toUpperCase().startsWith("DECLARE ")) { + expect(seenNonDeclare).to.equal(false); + } else { + seenNonDeclare = true; + } + } + }); + + test("creates operation tasks for 4-part Lakehouse operation target", () => { + const tasks = executionSql.createOperationTasks({ + target: lakehouseTarget, + hasOutput: true, + queries: [ + "CREATE OR REPLACE TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` (order_id INT64)", + ], + }); + expect(tasks).deep.equals([ + dataform.ExecutionTask.create({ + type: "statement", + statement: + "CREATE OR REPLACE TABLE `project-id.ecommerce_catalog.sales_dw.customer_orders` (order_id INT64)", + }), + ]); + }); + + test("throws when applying onSchemaChange to Lakehouse table with missing schema fields", () => { + const table: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: lakehouseTarget, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 1 AS order_id", + onSchemaChange: dataform.OnSchemaChange.FAIL, + }; + + expect(() => + executionSql.publishTasks( + table, + { fullRefresh: false }, + dataform.TableMetadata.create({ + type: dataform.TableMetadata.Type.TABLE, + target: lakehouseTarget, + }), + ), + ).to.throw(/missing schema fields/); + }); + + test("places scratch table in defaultDatabase when Lakehouse catalog lives in another project and throws if defaultProject is unset", () => { + const crossProjectExecutionSql = new ExecutionSql( + { + defaultDatabase: "default-project", + defaultLocation: "us-central1", + databaseSuffix: "dev", + defaultSchema: "default_dataset", + schemaSuffix: "staging", + }, + "3.1.5", + () => "test_uuid", + ); + const crossProjectTable: dataform.ITable = { + type: "incremental", + enumType: dataform.TableType.INCREMENTAL, + target: { + database: "catalog-project", + schema: "ecommerce_catalog.sales_dw", + name: "customer_orders", + }, + query: "SELECT 1 AS order_id", + incrementalQuery: "SELECT 1 AS order_id", + onSchemaChange: dataform.OnSchemaChange.EXTEND, + }; + + const sql = crossProjectExecutionSql + .publishTasks(crossProjectTable, { fullRefresh: false }, lakehouseMetadata) + .build()[0].statement; + + expect(sql).to.include( + "CREATE OR REPLACE TABLE `default-project_dev.default_dataset_staging.customer_orders_df_temp_test_uuid_empty`", + ); + expect(sql).to.include( + "FROM `default-project_dev.default_dataset_staging.INFORMATION_SCHEMA.COLUMNS`", + ); + expect(sql).to.include( + '"ALTER TABLE `catalog-project.ecommerce_catalog.sales_dw.customer_orders` "', + ); + expect(sql).to.include( + "CREATE SCHEMA IF NOT EXISTS `default-project_dev.default_dataset_staging` OPTIONS(location='us-central1');\nBEGIN", + ); + + const noDefaultDbExecutionSql = new ExecutionSql({ defaultSchema: "default_dataset" }, "3.1.5"); + expect(() => + noDefaultDbExecutionSql.publishTasks( + crossProjectTable, + { fullRefresh: false }, + lakehouseMetadata, + ), + ).to.throw(/'defaultProject' must be defined/); + }); +}); diff --git a/cli/api/lineage/payload_builder.ts b/cli/api/lineage/payload_builder.ts index 05c01d3cc..99936ce1e 100644 --- a/cli/api/lineage/payload_builder.ts +++ b/cli/api/lineage/payload_builder.ts @@ -3,6 +3,7 @@ import Long from "long"; import * as path from "path"; import { toLineageActionType } from "df/cli/api/lineage/action_type"; +import { isLakehouseTarget } from "df/core/targets"; import { version } from "df/core/version"; import { dataform } from "df/protos/ts"; @@ -81,13 +82,21 @@ export class LineagePayloadBuilder { } } - const emitDatasets = eventType === "COMPLETE" && hasLineageOutput(action, actionType); - + // Data Lineage expects 3-part `..` BigQuery dataset names; a 4-part + // Lakehouse target (`project.catalog.namespace.table`) would be misparsed as project + // `project.catalog`. Skip Lakehouse targets in `inputs`, and omit `inputs`/`outputs` altogether + // when the action's own target is a Lakehouse target. + const emitDatasets = + eventType === "COMPLETE" && + hasLineageOutput(action, actionType) && + !isLakehouseTarget(action.target); const inputs = emitDatasets - ? (action.dependencyTargets || []).map((dep) => ({ - namespace: "bigquery", - name: toFqn(dep, projectId), - })) + ? (action.dependencyTargets || []) + .filter((dep) => !isLakehouseTarget(dep)) + .map((dep) => ({ + namespace: "bigquery", + name: toFqn(dep, projectId), + })) : []; const outputs = emitDatasets diff --git a/cli/api/lineage/payload_builder_test.ts b/cli/api/lineage/payload_builder_test.ts index 728b655b1..d15206983 100644 --- a/cli/api/lineage/payload_builder_test.ts +++ b/cli/api/lineage/payload_builder_test.ts @@ -331,4 +331,33 @@ suite("LineagePayloadBuilder", () => { expect(opWithoutOutputComplete.inputs).to.deep.equal([]); expect(opWithoutOutputComplete.outputs).to.deep.equal([]); }); + + test("omits Lakehouse targets from inputs and omits inputs/outputs when action target is Lakehouse", () => { + const builder = new LineagePayloadBuilder("/tmp/proj"); + + const lakehouseAction = dataform.ExecutionAction.create({ + target: { database: "proj", schema: "my_catalog.my_ns", name: "lh_table" }, + type: "table", + dependencyTargets: [{ database: "proj", schema: "bq_dataset", name: "src_table" }], + }); + const lhComplete = builder.build(lakehouseAction, COMPLETE_RESULT, "proj", "us"); + expect(lhComplete.inputs).to.deep.equal([]); + expect(lhComplete.outputs).to.deep.equal([]); + + const bqDownstreamAction = dataform.ExecutionAction.create({ + target: { database: "proj", schema: "bq_dataset", name: "downstream_table" }, + type: "table", + dependencyTargets: [ + { database: "proj", schema: "my_catalog.my_ns", name: "lh_table" }, + { database: "proj", schema: "bq_dataset", name: "src_table" }, + ], + }); + const bqComplete = builder.build(bqDownstreamAction, COMPLETE_RESULT, "proj", "us"); + expect(bqComplete.inputs).to.deep.equal([ + { namespace: "bigquery", name: "proj.bq_dataset.src_table" }, + ]); + expect(bqComplete.outputs).to.deep.equal([ + { namespace: "bigquery", name: "proj.bq_dataset.downstream_table" }, + ]); + }); }); diff --git a/cli/commands/compile_options.ts b/cli/commands/compile_options.ts index 083efc8f4..76917ea19 100644 --- a/cli/commands/compile_options.ts +++ b/cli/commands/compile_options.ts @@ -31,11 +31,12 @@ export const outputActionsOption: INamedOption = { name: "output-actions", option: { // No wildcard support: prune()'s matchPatterns() does exact matching on the - // action name or its fully-qualified `database.schema.name`. + // action name or its fully-qualified `database.schema.name` (or 4-part + // `database.catalog.namespace.name`). describe: "Filters the printed output to these actions (comma-separated or repeatable). Each entry " + - "is an action name or a fully-qualified database.schema.name target. The whole project " + - "is still compiled.", + "is an action name or a fully-qualified database.schema.name (or 4-part " + + "database.catalog.namespace.name) target. The whole project is still compiled.", type: "array", coerce: splitCommas, }, diff --git a/cli/common_options.ts b/cli/common_options.ts index a2b62cc0e..9186dd7d3 100644 --- a/cli/common_options.ts +++ b/cli/common_options.ts @@ -45,10 +45,10 @@ export const actionsOption: INamedOption = { name: "actions", option: { // No wildcard support: prune()'s matchPatterns() does exact matching on the action name or - // its fully-qualified `database.schema.name`. + // its fully-qualified `database.schema.name` (or 4-part `database.catalog.namespace.name`). describe: "Actions to run (comma-separated or repeatable). Each entry is an action name or a " + - "fully-qualified database.schema.name target.", + "fully-qualified database.schema.name (or 4-part database.catalog.namespace.name) target.", type: "array", coerce: splitCommas, }, diff --git a/core/session.ts b/core/session.ts index 75c0474f5..4d39b3a7b 100644 --- a/core/session.ts +++ b/core/session.ts @@ -21,7 +21,7 @@ import { CompilationSql } from "df/core/compilation_sql"; import { validateSqlxActionBlocks } from "df/core/compilers"; import { Contextable, IActionContext, ITableContext, Resolvable } from "df/core/contextables"; import * as Path from "df/core/path"; -import { targetAsReadableString, targetStringifier } from "df/core/targets"; +import { isCatalogSchema, targetAsReadableString, targetStringifier } from "df/core/targets"; import * as utils from "df/core/utils"; import { ResolvableMap, toResolvable } from "df/core/utils"; import { version as dataformCoreVersion } from "df/core/version"; @@ -668,7 +668,7 @@ export class Session { public finalizeSchema(schema: string): string { // Normalized 4-part Lakehouse schemas ("${catalog}.${namespace}") do not take datasetSuffix. // Standard dataset names cannot contain '.' (enforced by ActionBuilder.validateTarget). - if (schema.includes(".")) { + if (isCatalogSchema(schema)) { return schema; } return `${schema}${this.getSchemaSuffixWithUnderscore()}`; diff --git a/core/targets.ts b/core/targets.ts index 9d1eef04f..f173263a8 100644 --- a/core/targets.ts +++ b/core/targets.ts @@ -19,3 +19,19 @@ export function targetAsReadableString(target: dataform.ITarget): string { } return nameParts.reverse().join("."); } + +/** + * Returns true when `schema` is a Lakehouse `catalog.namespace` pair rather than a plain BigQuery + * dataset. Standard BigQuery dataset names cannot contain `.`, so a dot unambiguously identifies a + * normalized 4-part Lakehouse target (`project.catalog.namespace.table`). + */ +export function isCatalogSchema(schema?: string | null): boolean { + return Boolean(schema && schema.includes(".")); +} + +/** + * Returns true when `target` refers to a 4-part Lakehouse Catalog object. + */ +export function isLakehouseTarget(target?: dataform.ITarget | null): boolean { + return isCatalogSchema(target?.schema); +} diff --git a/docs/lineage.md b/docs/lineage.md index 852fd112f..d47b86d24 100644 --- a/docs/lineage.md +++ b/docs/lineage.md @@ -79,6 +79,8 @@ Lineage events are emitted for every Dataform action that executes a BigQuery jo `inputs` and `outputs` are populated only on `COMPLETE` events (`START`, `FAIL`, and `ABORT` events carry empty `inputs` and `outputs` arrays) so failed or cancelled runs do not create false lineage edges. +For **4-part Lakehouse Catalog targets** (`project.catalog.namespace.table`), `Process` and `Run` events are still emitted, but Lakehouse targets are omitted from `inputs`, and both `inputs` and `outputs` are empty (`[]`) when the action's own target is a Lakehouse target (because the Data Lineage `bigquery` namespace expects 3-part `..
` identifiers). + --- ## Where Lineage Appears in Google Cloud diff --git a/tests/api/run.spec.ts b/tests/api/run.spec.ts index 45487ebaf..074a5c3e9 100644 --- a/tests/api/run.spec.ts +++ b/tests/api/run.spec.ts @@ -736,7 +736,7 @@ suite("@dataform/api/run", () => { executionOptions.push(options); return Promise.resolve({ rows: [], metadata: {} }); }); - when(mockedDbAdapter.setMetadata(anything())).thenCall( + when(mockedDbAdapter.setMetadata(anything(), anything())).thenCall( (action: dataform.IExecutionAction) => { setMetadataActionDescriptors.push(action.actionDescriptor!); return Promise.resolve(); @@ -786,7 +786,7 @@ suite("@dataform/api/run", () => { const result = await runner.execute().result(); expect(result.status).to.equal(dataform.RunResult.ExecutionStatus.SUCCESSFUL); - verify(mockedDbAdapter.setMetadata(anything())).never(); + verify(mockedDbAdapter.setMetadata(anything(), anything())).never(); }); test("resolveJobLabels falls back to bigqueryLabels when jobLabels is empty, and execution labels override them", () => { @@ -910,7 +910,9 @@ suite("@dataform/api/run", () => { rows: [], metadata: {}, }); - when(mockedDbAdapter.setMetadata(anything())).thenReject(new Error("Error during setMetadata")); + when(mockedDbAdapter.setMetadata(anything(), anything())).thenReject( + new Error("Error during setMetadata"), + ); const mockDbAdapterInstance = instance(mockedDbAdapter); diff --git a/tests/api/schemas.spec.ts b/tests/api/schemas.spec.ts index b23cd173a..df9447755 100644 --- a/tests/api/schemas.spec.ts +++ b/tests/api/schemas.spec.ts @@ -260,7 +260,7 @@ suite("schema creation", () => { errorMessage: "Run cancelled", }, ]); - verify(mockedDbAdapter.setMetadata(anything())).never(); + verify(mockedDbAdapter.setMetadata(anything(), anything())).never(); expect(executedStatements).deep.equals([wrapped("new_ds", "CREATE TABLE a")]); });