diff --git a/core/actions/incremental_table.ts b/core/actions/incremental_table.ts index d74687f5a..c68ee29ba 100644 --- a/core/actions/incremental_table.ts +++ b/core/actions/incremental_table.ts @@ -520,6 +520,14 @@ export class IncrementalTable extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + assertion.database(database); + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + this.rowConditionsAssertion.database(database); + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } @@ -544,6 +552,18 @@ export class IncrementalTable extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + if (this.inlineAssertionsFollowActionSchema()) { + assertion.schema(schema); + } + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + if (this.inlineAssertionsFollowActionSchema()) { + this.rowConditionsAssertion.schema(schema); + } + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } diff --git a/core/actions/index.ts b/core/actions/index.ts index 6e3201e2d..b23e15d90 100644 --- a/core/actions/index.ts +++ b/core/actions/index.ts @@ -41,6 +41,11 @@ export abstract class ActionBuilder { public session: Session; public includeAssertionsForDependency: Map = new Map(); protected isLakehouseTarget = false; + /** + * The dataset explicitly configured for this action's inline assertions (`assertions.dataset`), + * if any. Takes precedence over the project-level default assertion dataset. + */ + protected inlineAssertionsDataset?: string; constructor(session?: Session) { this.session = session; @@ -90,6 +95,14 @@ export abstract class ActionBuilder { /** Creates the final protobuf representation. */ public abstract compile(): T; + /** + * Whether inline assertions should be written to the same dataset as their parent action, i.e. + * neither an action-level nor a project-level assertion dataset has been configured. + */ + protected inlineAssertionsFollowActionSchema(): boolean { + return !this.inlineAssertionsDataset && !this.session.projectConfig.assertionSchema; + } + protected generateInlineAssertions( tableAssertionsConfig: dataform.ActionConfig.TableAssertionsConfig, proto: dataform.Table, @@ -117,12 +130,27 @@ export abstract class ActionBuilder { ]; } const jobLabels = proto.actionDescriptor?.jobLabels; + const project = proto.target.database || undefined; + // `schema` is the legacy name for `dataset`; it is accepted here for callers of the JS API's + // `.assertions()` method, which bypasses the legacy config conversion. + this.inlineAssertionsDataset = + (tableAssertionsConfig as any).schema || tableAssertionsConfig.dataset || undefined; + const dataset = + this.inlineAssertionsDataset || + this.session.projectConfig.assertionSchema || + proto.target.schema || + undefined; if (uniqueKeys) { uniqueKeys.forEach(({ uniqueKey }, index) => { const uniqueKeyAssertion = this.session .assert( `${assertionPrefix}${proto.target.schema}_${proto.target.name}_assertions_uniqueKey_${index}`, - dataform.ActionConfig.AssertionConfig.create({ filename: proto.fileName, jobLabels }), + dataform.ActionConfig.AssertionConfig.create({ + filename: proto.fileName, + project, + dataset, + jobLabels, + }), ) .query((ctx) => this.session.compilationSql().indexAssertion(ctx.ref(proto.target), uniqueKey), @@ -149,7 +177,12 @@ export abstract class ActionBuilder { inlineAssertions.rowConditionsAssertion = this.session .assert( `${assertionPrefix}${proto.target.schema}_${proto.target.name}_assertions_rowConditions`, - dataform.ActionConfig.AssertionConfig.create({ filename: proto.fileName, jobLabels }), + dataform.ActionConfig.AssertionConfig.create({ + filename: proto.fileName, + project, + dataset, + jobLabels, + }), ) .query((ctx) => this.session @@ -390,6 +423,7 @@ export interface ILegacyTableAssertions { uniqueKeys?: string[][]; nonNull?: string | string[]; rowConditions?: string[]; + schema?: string; } export class LegacyConfigConverter { @@ -436,6 +470,10 @@ export class LegacyConfigConverter { if (typeof legacyConfig.assertions.nonNull === "string") { legacyConfig.assertions.nonNull = [legacyConfig.assertions.nonNull]; } + if (legacyConfig.assertions.schema) { + legacyConfig.assertions.dataset = legacyConfig.assertions.schema; + delete legacyConfig.assertions.schema; + } } return legacyConfig; } diff --git a/core/actions/index_test.ts b/core/actions/index_test.ts index d9624e9c9..0c2ba4b27 100644 --- a/core/actions/index_test.ts +++ b/core/actions/index_test.ts @@ -60,13 +60,13 @@ export const exampleBuiltInAssertions = { }, }, target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_uniqueKey_0", }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_uniqueKey_0", }, dependencyTargets: [ @@ -94,13 +94,13 @@ export const exampleBuiltInAssertions = { }, }, target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_rowConditions", }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_rowConditions", }, dependencyTargets: [ @@ -145,13 +145,13 @@ export const exampleBuiltInAssertionsAsYaml = { }, }, target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_uniqueKey_0", }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_uniqueKey_0", }, dependencyTargets: [ @@ -181,13 +181,13 @@ export const exampleBuiltInAssertionsAsYaml = { }, }, target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_rowConditions", }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: "dataset_name_assertions_rowConditions", }, dependencyTargets: [ @@ -315,13 +315,13 @@ SELECT 1`, asPlainObject([ { target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: `builtin_dataset_name_assertions_uniqueKey_0`, }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: `builtin_dataset_name_assertions_uniqueKey_0`, }, dependencyTargets: [ @@ -342,13 +342,13 @@ SELECT 1`, }, { target: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: `builtin_dataset_name_assertions_rowConditions`, }, canonicalTarget: { - database: "defaultProject", - schema: "defaultDataset", + database: "project", + schema: "dataset", name: `builtin_dataset_name_assertions_rowConditions`, }, dependencyTargets: [ @@ -379,6 +379,350 @@ SELECT 1`, }), ); }); + + test(`${tableType} built-in assertions use parent database and defaultAssertionDataset when available`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultAssertionDataset: assertionDataset +defaultLocation: US +`, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.sqlx"), + ` +config { + type: "${tableType}", + database: "customProject", + schema: "customDataset", + name: "name", + assertions: { + uniqueKey: ["id"], + rowConditions: ["id > 0"], + } +} +SELECT 1 AS id`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "customProject", + schema: "assertionDataset", + name: "customDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "assertionDataset", + name: "customDataset_name_assertions_rowConditions", + }, + ]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => + asPlainObject(assertion.canonicalTarget), + ), + ).deep.equals([ + { + database: "customProject", + schema: "assertionDataset", + name: "customDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "assertionDataset", + name: "customDataset_name_assertions_rowConditions", + }, + ]); + }); + + test(`${tableType} built-in assertions fall back to defaultDataset when neither defaultAssertionDataset nor action schema is set`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultLocation: US +`, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.sqlx"), + ` +config { + type: "${tableType}", + database: "customProject", + name: "name", + assertions: { + uniqueKey: ["id"], + rowConditions: ["id > 0"], + } +} +SELECT 1 AS id`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "customProject", + schema: "defaultDataset", + name: "defaultDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "defaultDataset", + name: "defaultDataset_name_assertions_rowConditions", + }, + ]); + }); + + test(`${tableType} built-in assertions in JS API inherit database and schema from chained methods`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("name", { type: "${tableType}" }) + .assertions({ + uniqueKey: ["id"], + rowConditions: ["id > 0"], + }) + .database("customProject") + .schema("customDataset") + .query("SELECT 1 AS id"); +`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "customProject", + schema: "customDataset", + name: "defaultDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "customDataset", + name: "defaultDataset_name_assertions_rowConditions", + }, + ]); + }); + + ["schema", "dataset"].forEach((assertionDatasetKey) => { + test(`${tableType} built-in assertions use assertions.${assertionDatasetKey} over defaultAssertionDataset`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultAssertionDataset: assertionDataset +defaultLocation: US +`, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.sqlx"), + ` +config { + type: "${tableType}", + database: "customProject", + schema: "customDataset", + name: "name", + assertions: { + ${assertionDatasetKey}: "customAssertionDataset", + uniqueKey: ["id"], + rowConditions: ["id > 0"], + } +} +SELECT 1 AS id`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + const expectedTargets = [ + { + database: "customProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_rowConditions", + }, + ]; + expect( + result.compile.compiledGraph.assertions.map((assertion) => + asPlainObject(assertion.target), + ), + ).deep.equals(expectedTargets); + expect( + result.compile.compiledGraph.assertions.map((assertion) => + asPlainObject(assertion.canonicalTarget), + ), + ).deep.equals(expectedTargets); + // The parent action's own dataset is unaffected. + expect(asPlainObject(result.compile.compiledGraph.tables[0].target)).deep.equals({ + database: "customProject", + schema: "customDataset", + name: "name", + }); + }); + }); + + test(`${tableType} built-in assertions use assertions.schema when defaultAssertionDataset is not set`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.sqlx"), + ` +config { + type: "${tableType}", + schema: "customDataset", + name: "name", + assertions: { + schema: "customAssertionDataset", + uniqueKey: ["id"], + rowConditions: ["id > 0"], + } +} +SELECT 1 AS id`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "defaultProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_uniqueKey_0", + }, + { + database: "defaultProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_rowConditions", + }, + ]); + }); + + test(`${tableType} built-in assertions in JS API keep assertions.schema after chained methods`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/file.js"), + ` +publish("name", { type: "${tableType}" }) + .assertions({ + schema: "customAssertionDataset", + uniqueKey: ["id"], + rowConditions: ["id > 0"], + }) + .database("customProject") + .schema("customDataset") + .query("SELECT 1 AS id"); +`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "customProject", + schema: "customAssertionDataset", + name: "defaultDataset_name_assertions_uniqueKey_0", + }, + { + database: "customProject", + schema: "customAssertionDataset", + name: "defaultDataset_name_assertions_rowConditions", + }, + ]); + }); + + test(`${tableType} built-in assertions use assertions.dataset from actions.yaml`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + ` +defaultProject: defaultProject +defaultDataset: defaultDataset +defaultAssertionDataset: assertionDataset +defaultLocation: US +`, + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync(path.join(projectDir, "definitions/filename.sql"), "SELECT 1 AS id"); + fs.writeFileSync( + path.join(projectDir, "definitions/actions.yaml"), + ` +actions: +- ${tableType === "incremental" ? "incrementalTable" : tableType}: + name: name + dataset: customDataset + filename: filename.sql + assertions: + dataset: customAssertionDataset + uniqueKey: + - id + rowConditions: + - id > 0 +`, + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect( + result.compile.compiledGraph.assertions.map((assertion) => asPlainObject(assertion.target)), + ).deep.equals([ + { + database: "defaultProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_uniqueKey_0", + }, + { + database: "defaultProject", + schema: "customAssertionDataset", + name: "customDataset_name_assertions_rowConditions", + }, + ]); + }); }); ["table", "view", "incremental", "operations", "assertion"].forEach((tableType) => { diff --git a/core/actions/table.ts b/core/actions/table.ts index 6101ca45e..6b475f07a 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -489,6 +489,14 @@ export class Table extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + assertion.database(database); + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + this.rowConditionsAssertion.database(database); + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } @@ -513,6 +521,18 @@ export class Table extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + if (this.inlineAssertionsFollowActionSchema()) { + assertion.schema(schema); + } + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + if (this.inlineAssertionsFollowActionSchema()) { + this.rowConditionsAssertion.schema(schema); + } + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } diff --git a/core/actions/view.ts b/core/actions/view.ts index 4e501adeb..78fb436bd 100644 --- a/core/actions/view.ts +++ b/core/actions/view.ts @@ -455,6 +455,14 @@ export class View extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + assertion.database(database); + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + this.rowConditionsAssertion.database(database); + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } @@ -471,6 +479,18 @@ export class View extends ActionBuilder { this.proto.fileName, { validateTarget: true }, ); + this.uniqueKeyAssertions.forEach((assertion) => { + if (this.inlineAssertionsFollowActionSchema()) { + assertion.schema(schema); + } + assertion.setParentAction(dataform.Target.create(this.proto.target)); + }); + if (this.rowConditionsAssertion) { + if (this.inlineAssertionsFollowActionSchema()) { + this.rowConditionsAssertion.schema(schema); + } + this.rowConditionsAssertion.setParentAction(dataform.Target.create(this.proto.target)); + } return this; } diff --git a/core/main_workflow_settings_test.ts b/core/main_workflow_settings_test.ts index a50aff10e..9be96e048 100644 --- a/core/main_workflow_settings_test.ts +++ b/core/main_workflow_settings_test.ts @@ -339,8 +339,7 @@ vars: ); writeDefinitionFile( projectDir, - "file.sqlx", // TODO(https://github.com/dataform-co/dataform/issues/1295): add a test and fix - // functionality for assertions overriding database. + "file.sqlx", ` config { type: "table", @@ -375,7 +374,9 @@ select 1 AS \${dataform.projectConfig.vars.columnVar}`, assertions: [ { canonicalTarget: { + database: "databaseVal", name: "tableSchema_file_assertions_rowConditions", + schema: "tableSchema", }, dependencyTargets: [ { @@ -393,7 +394,9 @@ select 1 AS \${dataform.projectConfig.vars.columnVar}`, query: "\nSELECT\n 'columnValue IS NOT NULL' AS failing_row_condition,\n *\nFROM `databaseVal.tableSchema.file`\nWHERE NOT (columnValue IS NOT NULL)\n", target: { + database: "databaseVal", name: "tableSchema_file_assertions_rowConditions", + schema: "tableSchema", }, }, ], @@ -434,7 +437,9 @@ select 1 AS \${dataform.projectConfig.vars.columnVar}`, ], targets: [ { + database: "databaseVal", name: "tableSchema_file_assertions_rowConditions", + schema: "tableSchema", }, { database: "databaseVal", diff --git a/docs/reference/configs.md b/docs/reference/configs.md index 06500788b..3b2cb20dd 100644 --- a/docs/reference/configs.md +++ b/docs/reference/configs.md @@ -287,6 +287,7 @@ action types. | uniqueKeys | [ActionConfig.TableAssertionsConfig.UniqueKey](#dataform-ActionConfig-TableAssertionsConfig-UniqueKey) | repeated | | | nonNull | [string](#string) | repeated | Column(s) which may never be `NULL`. If set, the resulting assertion will fail if any row contains `NULL` values for these column(s). | | rowConditions | [string](#string) | repeated | General condition(s) which should hold true for all rows in the dataset. If set, the resulting assertion will fail if any row violates any of these condition(s). | +| dataset | [string](#string) | | Optional. The dataset (schema) in which to create the resulting assertion(s). Takes precedence over `defaultAssertionDataset` in `workflow_settings.yaml`. If neither is set, the assertion(s) are created in the same dataset as the action they are defined on. | diff --git a/protos/configs.proto b/protos/configs.proto index 9e5a61aaa..f50afbdd1 100644 --- a/protos/configs.proto +++ b/protos/configs.proto @@ -175,6 +175,12 @@ message ActionConfig { // If set, the resulting assertion will fail if any row violates any of // these condition(s). repeated string row_conditions = 4; + + // Optional. The dataset (schema) in which to create the resulting + // assertion(s). Takes precedence over `defaultAssertionDataset` in + // `workflow_settings.yaml`. If neither is set, the assertion(s) are created + // in the same dataset as the action they are defined on. + string dataset = 5; } message IcebergTableConfig {