From dec4d63ca40e418bc50ffc9d6410f36a12d3e6fc Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:43:54 +0200 Subject: [PATCH] feat(conformance): close 206 uncovered constraints; coverage 18.8% -> 40.7% The first wave shipped 269 cases and 16.8% mutation coverage. Case count is not coverage: the question that matters is whether deleting a constraint from the schema turns anything red, and for most of the schema the answer was no. exposePolicy and semanticModel -- both governance-critical -- had no cases at all. Seven new groups, each written against a measured list of constraints that nothing pinned: expose-policy 14 cases 13/13 targets closed semantic-model 22 cases 21/21 orchestration 47 cases 51/56 acquisition 71 cases 69/69 lineage-ml 27 cases 26/26 binding-governance 18 cases 17/17 expose-mcp 11 cases 9/9 206 of 211 targeted constraints closed. The five left open are airflowTask `allOf` branches that are genuinely unreachable -- allOf[0]'s `then` carries only a description, so no document can make it fail -- and they are reported as unreachable rather than papered over with a case that proves nothing. Coverage 18.8% -> 40.7% (192 -> 415 of 1020 falsifiable constraints). The constraints that carry real meaning -- enums, required members, closed objects, bounds -- went from 248 unpinned to 38. What remains is dominated by `type` deletions, which need a wrong-typed value for every property in the schema and are individually low-value. Also in this change: * mutation_coverage.py learns to tell "uncovered" from "redundant". A constraint sitting beside an `enum` that already restricts the instance to values satisfying it is unfalsifiable -- no case could ever catch its removal. fluidVersion is the clearest example: `pattern`, `type: string` and an enum of three version strings, so two of the three are dead weight. Counting those as gaps blames the corpus for something it cannot fix. 120 such constraints are now excluded from the denominator and reported as schema findings instead. * envelope.json gains the four missing required-member cases (kind, id, name, exposes) and the empty-name case. The mutation report found those; the original corpus tested only two of the six required members going missing. Every group was written by one agent and then adversarially reviewed by another that independently re-measured the coverage claim rather than trusting it, checked every invalid case raises exactly one error, and confirmed that reversing only the declared defect makes the document valid. All seven came back sound with the claim confirmed. Verified in a clean venv built from scratch (Python 3.14, jsonschema[format] only): 480/480 cases, meta-test 19/19, all four compat pairs clean, and the reference implementation agrees on 478 with 0 disagreements. Co-Authored-By: Claude Opus 5 --- conformance/mutation_coverage.py | 76 +- tests/0.7.5/acquisition.json | 4387 ++++++++++++++ tests/0.7.5/binding-governance.json | 1028 ++++ tests/0.7.5/envelope.json | 44 + tests/0.7.5/expose-mcp.json | 339 ++ tests/0.7.5/expose-policy.json | 499 ++ tests/0.7.5/lineage-ml.json | 1487 +++++ tests/0.7.5/orchestration.json | 8619 +++++++++++++++++++++++++++ tests/0.7.5/semantic-model.json | 1974 ++++++ tests/README.md | 25 + 10 files changed, 18473 insertions(+), 5 deletions(-) create mode 100644 tests/0.7.5/acquisition.json create mode 100644 tests/0.7.5/binding-governance.json create mode 100644 tests/0.7.5/expose-mcp.json create mode 100644 tests/0.7.5/expose-policy.json create mode 100644 tests/0.7.5/lineage-ml.json create mode 100644 tests/0.7.5/orchestration.json create mode 100644 tests/0.7.5/semantic-model.json diff --git a/conformance/mutation_coverage.py b/conformance/mutation_coverage.py index 4a03a69..e72eab4 100644 --- a/conformance/mutation_coverage.py +++ b/conformance/mutation_coverage.py @@ -136,6 +136,35 @@ def apply(schema: Dict[str, Any]) -> None: return apply +def is_unfalsifiable(node: Dict[str, Any], keyword: str) -> bool: + """True when deleting `keyword` cannot change which documents validate. + + The common case: a node carries both an ``enum`` and a narrower-looking + companion constraint (``pattern``, ``minLength``, ``type``, ``format``). + The enum already restricts the instance to a fixed set, so if every member + of that set satisfies the companion, the companion is dead weight -- and + NO test case could ever catch its removal. + + Reporting that as "uncovered" would be crying wolf: it blames the corpus + for a gap the corpus cannot close. It is a finding about the *schema* + instead, so it gets its own bucket. + """ + enum = node.get("enum") + if not isinstance(enum, list) or not enum or keyword == "enum": + return False + try: + from jsonschema import Draft202012Validator + except ImportError: # pragma: no cover + return False + with_kw = {k: v for k, v in node.items() if k != "enum"} + if keyword not in with_kw: + return False + without_kw = {k: v for k, v in with_kw.items() if k != keyword} + a = Draft202012Validator(with_kw) + b = Draft202012Validator(without_kw) + return all(a.is_valid(v) == b.is_valid(v) for v in enum) + + def corpus_is_red(cases: List[Dict[str, Any]], schema: Dict[str, Any]) -> bool: """True when at least one case fails against this (mutated) schema.""" validators = _build_validators(schema) @@ -157,6 +186,19 @@ def main(argv: Optional[Sequence[str]] = None) -> int: help="exit non-zero if coverage falls below this percentage", ) ap.add_argument("--limit", type=int, default=None, help="sample N mutations") + ap.add_argument( + "--group", + metavar="NAME", + help="measure only one corpus file's cases (its stem). Answers 'does " + "THIS group pin these constraints', which is what you want while " + "authoring one group -- and it isolates you from a sibling group that " + "is mid-edit.", + ) + ap.add_argument( + "--pointer-filter", + metavar="SUBSTR", + help="only report constraints whose pointer contains SUBSTR", + ) args = ap.parse_args(argv) report: Dict[str, Any] = {} @@ -164,7 +206,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: for version in args.versions or discover_versions(): try: - cases = load_corpus(version, include_optional=True) + cases = load_corpus(version, include_optional=True, group=args.group) except CorpusError as exc: print(f"mutation-coverage: {exc}", file=sys.stderr) return 2 @@ -172,6 +214,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: mutations: List[Mutation] = [] _walk(schema, "", mutations) + if args.pointer_filter: + mutations = [m for m in mutations if args.pointer_filter in m[0]] if args.limit: mutations = mutations[: args.limit] @@ -179,15 +223,17 @@ def main(argv: Optional[Sequence[str]] = None) -> int: # mutation trivially "fails" and the whole report is meaningless. if corpus_is_red(cases, copy.deepcopy(schema)): print( - f"mutation-coverage: the {version} corpus is not green against the " - "unmutated schema; fix that first -- otherwise every constraint " - "reads as covered.", + f"mutation-coverage: the {version} corpus" + + (f" (group {args.group})" if args.group else "") + + " is not green against the unmutated schema; fix that first -- " + "otherwise every constraint reads as covered.", file=sys.stderr, ) return 2 covered: List[str] = [] uncovered: List[Tuple[str, str]] = [] + redundant: List[Tuple[str, str]] = [] for pointer, label, apply in mutations: mutated = copy.deepcopy(schema) try: @@ -198,6 +244,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int: continue if corpus_is_red(cases, mutated): covered.append(f"{pointer}: {label}") + continue + # Before blaming the corpus, ask whether ANY case could have caught + # this. A constraint the schema makes unfalsifiable is not a + # coverage gap. + keyword = label.replace("delete ", "").split()[0] + node_ptr = pointer.rsplit("/", 1)[0] + try: + node = _resolve(schema, node_ptr) + except (KeyError, IndexError, TypeError): + node = None + if isinstance(node, dict) and is_unfalsifiable(node, keyword): + redundant.append((pointer, label)) else: uncovered.append((pointer, label)) @@ -209,13 +267,21 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "mutations": total, "covered": len(covered), "uncovered": [{"pointer": p, "mutation": m} for p, m in uncovered], + "redundant": [{"pointer": p, "mutation": m} for p, m in redundant], "coveragePct": round(pct, 1), } if args.format == "text": - print(f"\n{version}: {len(cases)} cases vs {total} schema constraints") + print(f"\n{version}: {len(cases)} cases vs {total} falsifiable constraints") print(f" covered {len(covered):4} ({pct:.1f}%)") print(f" UNCOVERED {len(uncovered):4}") + if redundant: + print( + f" redundant {len(redundant):4} (schema findings, not corpus " + "gaps: an enum already makes these unfalsifiable)" + ) + for pointer, label in redundant[:10]: + print(f" {pointer} -- {label}") for pointer, label in uncovered[:40]: print(f" {pointer} -- {label}") if len(uncovered) > 40: diff --git a/tests/0.7.5/acquisition.json b/tests/0.7.5/acquisition.json new file mode 100644 index 0000000..2a93e76 --- /dev/null +++ b/tests/0.7.5/acquisition.json @@ -0,0 +1,4387 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "acquisition", + "description": "The acquisition build pattern: source-aligned ingestion from an external system (source, sink, delivery/DLQ semantics, schema evolution policy, pre-land quality gates and anomaly signals, cost budgets, catalog auto-registration, single-flight concurrency, lineage emission, the kafka-connect and debezium engine blocks, and the shared deployment block for engine-managed provisioning).", + "cases": [ + { + "description": "streaming CDC acquisition reaching every sub-block: source, sink, delivery, schema evolution, quality gates and anomalies, cost budget, catalog, concurrency lock, lineage, kafka-connect (schema registry + streaming sink + managed deployment), and debezium", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "cdc", + "connection": { + "host": "db.internal", + "port": 5432, + "database": "crm", + "secretRef": "vault://secret/data/crm-db", + "schema": "public", + "schemas": [ + "public", + "audit" + ] + }, + "cursor_field": "updated_at", + "watermark": { + "strategy": "lsn", + "allowedLateness": "PT5M" + }, + "streams": [ + "customers" + ], + "reader": { + "format": "avro" + } + }, + "sink": { + "format": "iceberg", + "catalog": "rest", + "partitionBy": [ + "ingestion_date" + ] + }, + "delivery": { + "guarantee": "exactly_once", + "dlq": { + "enabled": true, + "sink": { + "format": "parquet", + "location": "s3://acme-dlq/customers" + }, + "maxRecordsBeforeAbort": 5000, + "alertOn": [ + "schema_violation", + "destination_write_failed" + ] + } + }, + "schemaEvolution": { + "policy": "evolve_safe", + "onAddedColumn": "include", + "onRemovedColumn": "warn", + "onTypeChange": "cast", + "sourceFingerprint": "required" + }, + "preLand": [ + "dlp_scan", + "quality_gate" + ], + "quality": { + "gates": [ + { + "rule": "not_null", + "columns": [ + "customer_id" + ], + "severity": "error" + }, + { + "rule": "regex", + "column": "email", + "pattern": "^.+@.+$", + "severity": "warn" + } + ], + "onError": "route_to_dlq", + "anomalies": [ + { + "signal": "record_count_drop", + "algorithm": "ewma", + "sensitivity": 0.5, + "severity": "warn" + } + ] + }, + "cost": { + "budget": { + "monthly": { + "rows": 1000000, + "bytes": "50GB", + "computeMinutes": 600 + }, + "onExceed": "warn" + }, + "chargeback": { + "team": "customer-analytics" + } + }, + "catalog": { + "register": [ + "datahub", + "glue" + ], + "documentation": "auto" + }, + "concurrency": { + "lock": { + "scope": "product", + "timeout": "PT30M", + "onContended": "queue" + } + }, + "lineage": { + "emit": true + }, + "kafka-connect": { + "connector_class": "io.debezium.connector.postgresql.PostgresConnector", + "connector_name": "customers-source", + "tasks_max": 2, + "schema_registry": { + "url": "http://schema-registry:8081", + "auth": { + "secretRef": "vault://secret/data/sr-auth" + } + }, + "streamingSink": { + "commitIntervalMs": 60000, + "routeField": "table", + "dynamicEnabled": true + }, + "deployment": { + "mode": "managed", + "managed": { + "target": "kubernetes", + "profile": "medium", + "secrets": [ + { + "name": "sr-auth", + "ref": "vault://secret/data/sr-auth" + } + ], + "network": { + "egressAllowList": [ + "schema-registry.internal" + ] + } + } + } + }, + "debezium": { + "connector_class": "io.debezium.connector.postgresql.PostgresConnector", + "snapshot_mode": "initial" + } + } + } + }, + "valid": true + }, + { + "description": "acquisition build carrying only the required source kind and mode", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + } + } + } + }, + "valid": true + }, + { + "description": "preLand hook not in the recognized chain", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "preLand": [ + "dlp_scan", + "hand_wave_at_it" + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/preLand/1", + "keyword": "enum" + } + ] + }, + { + "description": "kafka-connect tasks_max at zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "tasks_max": 0 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/tasks_max", + "keyword": "minimum" + } + ] + }, + { + "description": "debezium snapshot_mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "debezium": { + "snapshot_mode": "full_dump" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/debezium/snapshot_mode", + "keyword": "enum" + } + ] + }, + { + "description": "acquisitionPattern.lineage carrying an unrecognized member alongside emit", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "lineage": { + "emit": true, + "backend": "openlineage" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/lineage", + "keyword": "additionalProperties", + "detail": "backend" + } + ] + }, + { + "description": "kafka-connect schema_registry carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "schema_registry": { + "url": "http://sr:8081", + "compatibility": "BACKWARD" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/schema_registry", + "keyword": "additionalProperties", + "detail": "compatibility" + } + ] + }, + { + "description": "kafka-connect schema_registry.auth carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "schema_registry": { + "auth": { + "secretRef": "vault://secret/data/sr", + "mechanism": "SASL" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/schema_registry/auth", + "keyword": "additionalProperties", + "detail": "mechanism" + } + ] + }, + { + "description": "kafka-connect streamingSink carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "streamingSink": { + "commitIntervalMs": 30000, + "flushOnBatch": true + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/streamingSink", + "keyword": "additionalProperties", + "detail": "flushOnBatch" + } + ] + }, + { + "description": "kafka-connect streamingSink commitIntervalMs at zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "kafka-connect": { + "streamingSink": { + "commitIntervalMs": 0 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/kafka-connect/streamingSink/commitIntervalMs", + "keyword": "minimum" + } + ] + }, + { + "description": "source omits the required kind", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "mode": "full_refresh" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "required", + "detail": "kind" + } + ] + }, + { + "description": "source omits the required mode", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "required", + "detail": "mode" + } + ] + }, + { + "description": "source carrying an unrecognized member alongside kind and mode", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "driverVersion": "42.7.3" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source", + "keyword": "additionalProperties", + "detail": "driverVersion" + } + ] + }, + { + "description": "source connection secretRef with no scheme prefix", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "connection": { + "secretRef": "crm-db-password" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/connection/secretRef", + "keyword": "pattern" + } + ] + }, + { + "description": "source connection schemas repeating the same schema twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "connection": { + "schemas": [ + "public", + "public" + ] + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/connection/schemas", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "source mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "batch_dump" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/mode", + "keyword": "enum" + } + ] + }, + { + "description": "source watermark strategy outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "watermark": { + "strategy": "sequence_number" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/watermark/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "source reader format outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "reader": { + "format": "xml" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/reader/format", + "keyword": "enum" + } + ] + }, + { + "description": "source watermark carrying an unrecognized member alongside strategy", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh", + "watermark": { + "strategy": "lsn", + "checkpointInterval": "PT1M" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/source/watermark", + "keyword": "additionalProperties", + "detail": "checkpointInterval" + } + ] + }, + { + "description": "sink carrying an unrecognized member alongside format", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "format": "iceberg", + "compression": "zstd" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink", + "keyword": "additionalProperties", + "detail": "compression" + } + ] + }, + { + "description": "sink format outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "format": "avro_table" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/format", + "keyword": "enum" + } + ] + }, + { + "description": "sink catalog outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "sink": { + "catalog": "polaris" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/sink/catalog", + "keyword": "enum" + } + ] + }, + { + "description": "delivery guarantee outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "guarantee": "best_effort" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/guarantee", + "keyword": "enum" + } + ] + }, + { + "description": "delivery dlq sink format outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "sink": { + "format": "avro" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/sink/format", + "keyword": "enum" + } + ] + }, + { + "description": "delivery dlq maxRecordsBeforeAbort below zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "maxRecordsBeforeAbort": -1 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/maxRecordsBeforeAbort", + "keyword": "minimum" + } + ] + }, + { + "description": "delivery dlq alertOn signal outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "alertOn": [ + "disk_full" + ] + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/alertOn/0", + "keyword": "enum" + } + ] + }, + { + "description": "delivery carrying an unrecognized member alongside guarantee", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "guarantee": "at_least_once", + "retryPolicy": "exponential" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery", + "keyword": "additionalProperties", + "detail": "retryPolicy" + } + ] + }, + { + "description": "delivery dlq carrying an unrecognized member alongside enabled", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "enabled": true, + "retentionDays": 30 + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq", + "keyword": "additionalProperties", + "detail": "retentionDays" + } + ] + }, + { + "description": "delivery dlq sink carrying an unrecognized member alongside format", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "delivery": { + "dlq": { + "sink": { + "format": "json", + "compression": "gzip" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/delivery/dlq/sink", + "keyword": "additionalProperties", + "detail": "compression" + } + ] + }, + { + "description": "schemaEvolution policy outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "policy": "auto" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/policy", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onAddedColumn outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onAddedColumn": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onAddedColumn", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onRemovedColumn outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onRemovedColumn": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onRemovedColumn", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution onTypeChange outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "onTypeChange": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/onTypeChange", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution sourceFingerprint outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "sourceFingerprint": "best_effort" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution/sourceFingerprint", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution carrying an unrecognized member alongside policy", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "schemaEvolution": { + "policy": "strict", + "onSchemaMismatch": "fail" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/schemaEvolution", + "keyword": "additionalProperties", + "detail": "onSchemaMismatch" + } + ] + }, + { + "description": "quality gate omits the required rule", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "severity": "error" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0", + "keyword": "required", + "detail": "rule" + } + ] + }, + { + "description": "quality gate omits the required severity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "not_null" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "quality gate rule outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "checksum", + "severity": "error" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/rule", + "keyword": "enum" + } + ] + }, + { + "description": "quality gate algorithm outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "row_count_anomaly", + "severity": "warn", + "algorithm": "zscore" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/algorithm", + "keyword": "enum" + } + ] + }, + { + "description": "quality gate severity outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "not_null", + "severity": "critical" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "quality onError outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "onError": "ignore" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/onError", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly omits the required signal", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "severity": "warn" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "required", + "detail": "signal" + } + ] + }, + { + "description": "quality anomaly omits the required severity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "cursor_stalled" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "required", + "detail": "severity" + } + ] + }, + { + "description": "quality anomaly signal outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "disk_pressure", + "severity": "warn" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/signal", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly algorithm outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "warn", + "algorithm": "zscore" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/algorithm", + "keyword": "enum" + } + ] + }, + { + "description": "quality anomaly severity outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "critical" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "quality carrying an unrecognized member alongside onError", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "onError": "route_to_dlq", + "notifyChannel": "#data-alerts" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality", + "keyword": "additionalProperties", + "detail": "notifyChannel" + } + ] + }, + { + "description": "quality gate pattern given as a number instead of a regex string", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "gates": [ + { + "rule": "regex", + "severity": "warn", + "pattern": 42 + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/gates/0/pattern", + "keyword": "type" + } + ] + }, + { + "description": "quality anomaly carrying an unrecognized member alongside signal and severity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "quality": { + "anomalies": [ + { + "signal": "record_count_drop", + "severity": "warn", + "runbookUrl": "https://x" + } + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/quality/anomalies/0", + "keyword": "additionalProperties", + "detail": "runbookUrl" + } + ] + }, + { + "description": "cost budget monthly rows below zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "rows": -1 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly/rows", + "keyword": "minimum" + } + ] + }, + { + "description": "cost budget monthly computeMinutes below zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "computeMinutes": -1 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly/computeMinutes", + "keyword": "minimum" + } + ] + }, + { + "description": "cost budget onExceed outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "pause" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/onExceed", + "keyword": "enum" + } + ] + }, + { + "description": "cost carrying an unrecognized member alongside budget", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "warn" + }, + "currency": "USD" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost", + "keyword": "additionalProperties", + "detail": "currency" + } + ] + }, + { + "description": "cost budget carrying an unrecognized member alongside monthly", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "onExceed": "warn", + "annualCap": "600GB" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget", + "keyword": "additionalProperties", + "detail": "annualCap" + } + ] + }, + { + "description": "cost budget monthly carrying an unrecognized member alongside rows", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "cost": { + "budget": { + "monthly": { + "rows": 1000, + "gbScanned": 5 + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/cost/budget/monthly", + "keyword": "additionalProperties", + "detail": "gbScanned" + } + ] + }, + { + "description": "catalog register target outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "register": [ + "alation" + ] + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog/register/0", + "keyword": "enum" + } + ] + }, + { + "description": "catalog documentation outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "documentation": "generated" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog/documentation", + "keyword": "enum" + } + ] + }, + { + "description": "catalog carrying an unrecognized member alongside documentation", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "catalog": { + "documentation": "manual", + "owner": "data-platform" + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/catalog", + "keyword": "additionalProperties", + "detail": "owner" + } + ] + }, + { + "description": "concurrency lock scope outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "organization" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock/scope", + "keyword": "enum" + } + ] + }, + { + "description": "concurrency lock onContended outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "onContended": "retry_forever" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock/onContended", + "keyword": "enum" + } + ] + }, + { + "description": "concurrency carrying an unrecognized member alongside lock", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "build" + }, + "maxParallelRuns": 1 + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency", + "keyword": "additionalProperties", + "detail": "maxParallelRuns" + } + ] + }, + { + "description": "concurrency lock carrying an unrecognized member alongside scope", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "concurrency": { + "lock": { + "scope": "product", + "leaseId": "abc123" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/concurrency/lock", + "keyword": "additionalProperties", + "detail": "leaseId" + } + ] + }, + { + "description": "airbyte deployment omits the required mode", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": {} + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment", + "keyword": "required", + "detail": "mode" + } + ] + }, + { + "description": "airbyte deployment mode outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "self-hosted" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/mode", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment managed block omits the required target", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": {} + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed", + "keyword": "required", + "detail": "target" + } + ] + }, + { + "description": "airbyte deployment managed target outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "nomad" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/target", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment managed profile outside the recognized set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "profile": "xlarge" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/profile", + "keyword": "enum" + } + ] + }, + { + "description": "airbyte deployment carrying an unrecognized member alongside mode", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "embedded", + "region": "us-east-1" + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment", + "keyword": "additionalProperties", + "detail": "region" + } + ] + }, + { + "description": "airbyte deployment managed carrying an unrecognized member alongside target", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "namespace": "data-platform" + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed", + "keyword": "additionalProperties", + "detail": "namespace" + } + ] + }, + { + "description": "airbyte deployment managed secret entry carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "secrets": [ + { + "name": "sr-auth", + "ref": "vault://secret/data/sr", + "rotate": true + } + ] + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/secrets/0", + "keyword": "additionalProperties", + "detail": "rotate" + } + ] + }, + { + "description": "airbyte deployment managed network carrying an unrecognized member", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "acquisition", + "engine": "kafka-connect", + "properties": { + "source": { + "kind": "postgres", + "mode": "full_refresh" + }, + "airbyte": { + "deployment": { + "mode": "managed", + "managed": { + "target": "docker", + "network": { + "egressAllowList": [ + "api.stripe.com" + ], + "vpcId": "vpc-1" + } + } + } + } + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/properties/airbyte/deployment/managed/network", + "keyword": "additionalProperties", + "detail": "vpcId" + } + ] + } + ] +} diff --git a/tests/0.7.5/binding-governance.json b/tests/0.7.5/binding-governance.json new file mode 100644 index 0000000..a5ae078 --- /dev/null +++ b/tests/0.7.5/binding-governance.json @@ -0,0 +1,1028 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "binding-governance", + "description": "Per-resource bindingGovernance mounted at exposes[].binding.governance: the closed governance and lakeFormation property sets, the grants[] items object (required principal/permissions, the ARN pattern on principal, the LF permission enum, minItems/uniqueItems on permissions, uniqueItems on permissionsWithGrantOption/columns/excludedColumns, and the closed per-grant property set), and the rowFilter object (required name/rowExpression, uniqueItems on columnNames/excludedColumnNames, and its closed property set).", + "cases": [ + { + "description": "bindingGovernance.lakeFormation with a column allow-list grant, a column deny-list grant, LF tags, and a row- and column-filtered rowFilter", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "governance carries a future cloud-specific block not yet defined in the schema", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + }, + "snowflakeMaskingPolicy": { + "enabled": true + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance", + "keyword": "additionalProperties", + "detail": "snowflakeMaskingPolicy" + } + ] + }, + { + "description": "lakeFormation carries a retention setting outside its documented property set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "registerLocation": true, + "retentionPolicy": "90d", + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + }, + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "date_of_birth"] + } + ], + "tags": { + "pii-classification": "restricted" + }, + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation", + "keyword": "additionalProperties", + "detail": "retentionPolicy" + } + ] + }, + { + "description": "a grant omits the required principal", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "required", + "detail": "principal" + } + ] + }, + { + "description": "a grant omits the required permissions", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "required", + "detail": "permissions" + } + ] + }, + { + "description": "a grant carries a grant-option-expiry property outside its closed set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"], + "grantOptionExpiry": "2027-01-01" + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0", + "keyword": "additionalProperties", + "detail": "grantOptionExpiry" + } + ] + }, + { + "description": "a grant's principal is a bare role name instead of an IAM ARN", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "analytics-reader-role", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/principal", + "keyword": "pattern" + } + ] + }, + { + "description": "a grant's permissions array is empty", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": [], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions", + "keyword": "minItems" + } + ] + }, + { + "description": "a grant repeats the same permission twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "SELECT"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's permissions list includes a value outside the LF permission enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "MODIFY"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissions/1", + "keyword": "enum" + } + ] + }, + { + "description": "a grant's permissionsWithGrantOption repeats the same permission twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT", "SELECT"], + "columns": ["customer_id", "email", "region"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/permissionsWithGrantOption", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's columns list repeats the same column twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/analytics-reader", + "permissions": ["SELECT", "DESCRIBE"], + "permissionsWithGrantOption": ["SELECT"], + "columns": ["customer_id", "customer_id"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/columns", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "a grant's excludedColumns list repeats the same column twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "grants": [ + { + "principal": "arn:aws:iam::123456789012:role/compliance-auditor", + "permissions": ["SELECT"], + "excludedColumns": ["ssn", "ssn"] + } + ] + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/grants/0/excludedColumns", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "the rowFilter omits the required name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "the rowFilter omits the required rowExpression", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "columnNames": ["customer_id", "region"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "required", + "detail": "rowExpression" + } + ] + }, + { + "description": "the rowFilter carries a priority property outside its closed set", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "region"], + "priority": 1 + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter", + "keyword": "additionalProperties", + "detail": "priority" + } + ] + }, + { + "description": "the rowFilter's columnNames repeats the same column twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "columnNames": ["customer_id", "customer_id"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter/columnNames", + "keyword": "uniqueItems" + } + ] + }, + { + "description": "the rowFilter's excludedColumnNames repeats the same column twice", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "aws", + "format": "iceberg", + "location": { + "bucket": "acme-lake", + "path": "warehouse/customers/profiles" + }, + "governance": { + "lakeFormation": { + "rowFilter": { + "name": "region_filter", + "rowExpression": "region = 'us-east-1'", + "excludedColumnNames": ["ssn", "ssn"] + } + } + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/binding/governance/lakeFormation/rowFilter/excludedColumnNames", + "keyword": "uniqueItems" + } + ] + } + ] +} diff --git a/tests/0.7.5/envelope.json b/tests/0.7.5/envelope.json index 6888aaf..1168956 100644 --- a/tests/0.7.5/envelope.json +++ b/tests/0.7.5/envelope.json @@ -589,6 +589,50 @@ "detail": "exposes" } ] + }, + { + "description": "name is the empty string", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/name", + "keyword": "minLength" + } + ] } ] } diff --git a/tests/0.7.5/expose-mcp.json b/tests/0.7.5/expose-mcp.json new file mode 100644 index 0000000..dd625a0 --- /dev/null +++ b/tests/0.7.5/expose-mcp.json @@ -0,0 +1,339 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "expose-mcp", + "description": "The exposeMcp gateway block (exposes[].mcp: sampling.maxRows, classification.dataClass) and the completeness/latency members of defaultSLIs (exposes[].observability.defaultSLIs): each object's closed property set and the dataClass/severity enums.", + "cases": [ + { + "description": "expose combines an MCP gateway block with completeness and latency SLI severities", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500 }, + "classification": { "dataClass": "confidential" } + }, + "observability": { + "defaultSLIs": { + "enabled": true, + "completeness": { "severity": "info" }, + "latency": { "severity": "critical" } + } + } + } + ] + }, + "valid": true + }, + { + "description": "MCP classification carries the public dataClass value", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "public" } + } + } + ] + }, + "valid": true + }, + { + "description": "defaultSLIs completeness carries an unrecognized property alongside severity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "completeness": { "severity": "info", "sampleRate": 0.5 } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/completeness", "keyword": "additionalProperties", "detail": "sampleRate" } + ] + }, + { + "description": "defaultSLIs completeness severity outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "completeness": { "severity": "catastrophic" } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/completeness/severity", "keyword": "enum" } + ] + }, + { + "description": "defaultSLIs latency carries an unrecognized property alongside severity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "latency": { "severity": "critical", "sampleRate": 0.5 } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/latency", "keyword": "additionalProperties", "detail": "sampleRate" } + ] + }, + { + "description": "defaultSLIs latency severity outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "observability": { + "defaultSLIs": { + "latency": { "severity": "catastrophic" } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/observability/defaultSLIs/latency/severity", "keyword": "enum" } + ] + }, + { + "description": "mcp block carries an unrecognized property alongside sampling and classification", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500 }, + "classification": { "dataClass": "confidential" }, + "auditLog": true + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp", "keyword": "additionalProperties", "detail": "auditLog" } + ] + }, + { + "description": "mcp sampling carries an unrecognized property alongside maxRows", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 500, "maxRowsPerSecond": 50 } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/sampling", "keyword": "additionalProperties", "detail": "maxRowsPerSecond" } + ] + }, + { + "description": "mcp sampling maxRows at zero", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "sampling": { "maxRows": 0 } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/sampling/maxRows", "keyword": "minimum" } + ] + }, + { + "description": "mcp classification carries an unrecognized property alongside dataClass", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "confidential", "reviewedBy": "security-team" } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/classification", "keyword": "additionalProperties", "detail": "reviewedBy" } + ] + }, + { + "description": "mcp classification dataClass outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "mcp": { + "classification": { "dataClass": "top-secret" } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/mcp/classification/dataClass", "keyword": "enum" } + ] + } + ] +} diff --git a/tests/0.7.5/expose-policy.json b/tests/0.7.5/expose-policy.json new file mode 100644 index 0000000..a7352a8 --- /dev/null +++ b/tests/0.7.5/expose-policy.json @@ -0,0 +1,499 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "expose-policy", + "description": "The exposePolicy block reached via exposes[].policy: its own closed property set, the authn enum, the authz sub-object (closed property set, columnRestrictions items' closed property set and access enum), the privacy sub-object (closed property set, masking items' required column/strategy, closed property set and strategy enum, rowLevelPolicy's required expression and closed property set), and the classification enum.", + "cases": [ + { + "description": "policy block populated with authn, authz, privacy, and classification", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "oidc", + "authz": { + "readers": ["group:analytics"], + "writers": ["group:platform"], + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "deny", + "tags": ["pii-guard"], + "labels": { "reviewed-by": "security-team" } + } + ], + "tags": ["policy-authz"], + "labels": { "owner-team": "security" } + }, + "privacy": { + "masking": [ + { + "column": "ssn", + "strategy": "tokenize", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ], + "rowLevelPolicy": { + "expression": "region = current_region()", + "tags": ["rls-rule"], + "labels": { "scope": "region" } + }, + "tags": ["privacy-policy"], + "labels": { "data-class": "sensitive" } + }, + "classification": "Confidential", + "tags": ["policy-top"], + "labels": { "policy-owner": "governance-team" } + } + } + ] + }, + "valid": true + }, + { + "description": "policy carries an unrecognized top-level property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "oidc", + "classification": "Confidential", + "encryption": { "atRest": "kms" } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy", "keyword": "additionalProperties", "detail": "encryption" }] + }, + { + "description": "authn value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authn": "saml" + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authn", "keyword": "enum" }] + }, + { + "description": "authz carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "readers": ["group:analytics"], + "writers": ["group:platform"], + "approvers": ["group:governance"] + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authz", "keyword": "additionalProperties", "detail": "approvers" }] + }, + { + "description": "columnRestrictions item carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "deny", + "reason": "PII exposure" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/authz/columnRestrictions/0", "keyword": "additionalProperties", "detail": "reason" } + ] + }, + { + "description": "columnRestrictions access value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "authz": { + "columnRestrictions": [ + { + "principal": "group:analytics", + "columns": ["ssn"], + "access": "redact" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/authz/columnRestrictions/0/access", "keyword": "enum" }] + }, + { + "description": "privacy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [{ "column": "ssn", "strategy": "tokenize" }], + "retentionDays": 30 + } + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/privacy", "keyword": "additionalProperties", "detail": "retentionDays" }] + }, + { + "description": "masking rule omits column", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "strategy": "tokenize", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "required", "detail": "column" } + ] + }, + { + "description": "masking rule omits strategy", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "column": "ssn", + "params": { "format": "fpe" }, + "tags": ["masking-rule"], + "labels": { "applies-to": "ssn" } + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "required", "detail": "strategy" } + ] + }, + { + "description": "masking rule carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [ + { + "column": "ssn", + "strategy": "tokenize", + "algorithm": "aes256" + } + ] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0", "keyword": "additionalProperties", "detail": "algorithm" } + ] + }, + { + "description": "masking strategy value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "masking": [{ "column": "ssn", "strategy": "obfuscate" }] + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/masking/0/strategy", "keyword": "enum" } + ] + }, + { + "description": "rowLevelPolicy omits expression", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "rowLevelPolicy": { + "tags": ["rls-rule"], + "labels": { "scope": "region" } + } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/rowLevelPolicy", "keyword": "required", "detail": "expression" } + ] + }, + { + "description": "rowLevelPolicy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "privacy": { + "rowLevelPolicy": { + "expression": "region = current_region()", + "priority": 1 + } + } + } + } + ] + }, + "valid": false, + "errors": [ + { "pointer": "/exposes/0/policy/privacy/rowLevelPolicy", "keyword": "additionalProperties", "detail": "priority" } + ] + }, + { + "description": "classification value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { "owner": { "team": "customer-analytics" } }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { "project": "acme-prod", "dataset": "customers", "table": "profiles" } + }, + "contract": { "schema": [{ "name": "customer_id", "type": "string" }] }, + "policy": { + "classification": "Secret" + } + } + ] + }, + "valid": false, + "errors": [{ "pointer": "/exposes/0/policy/classification", "keyword": "enum" }] + } + ] +} diff --git a/tests/0.7.5/lineage-ml.json b/tests/0.7.5/lineage-ml.json new file mode 100644 index 0000000..87a322d --- /dev/null +++ b/tests/0.7.5/lineage-ml.json @@ -0,0 +1,1487 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "lineage-ml", + "description": "The optional lineage block (granularity, upstream dependency entries with their fieldMappings/transformation nesting and relationship enum, and downstream consumer entries with the impact enum), the schemaEvolution block (strategy/compatibility enums and the nested changePolicy object), and the machineLearning block (framework enum and model entries with their required fields, type enum, and nested validation object) -- including every closed object's additionalProperties:false boundary.", + "cases": [ + { + "description": "lineage, schemaEvolution and machineLearning populated to their nested edges", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "field_level", + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "relationship": "aggregation", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "calculation", + "expression": "MAX(event_ts)" + } + } + ] + } + ], + "downstream": [ + { + "consumer": "churn_model", + "impact": "high" + } + ] + }, + "schemaEvolution": { + "strategy": "semantic_versioning", + "compatibility": "backward_compatible", + "changePolicy": { + "changeWindowDays": 14, + "approvalRequired": true, + "approvers": [ + "data-governance" + ] + } + }, + "machineLearning": { + "enabled": true, + "framework": "mlflow", + "models": [ + { + "name": "churn_predictor", + "version": "1.2.0", + "type": "classification", + "algorithm": "xgboost", + "features": [ + "tenure_days", + "monthly_spend" + ], + "target": "churned", + "validation": { + "metrics": [ + "accuracy", + "auc_roc" + ], + "thresholds": { + "accuracy": 0.85 + } + } + } + ] + } + }, + "valid": true + }, + { + "description": "lineage carries an unrecognized property alongside granularity", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "table_level", + "retentionOverride": "90d" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage", + "keyword": "additionalProperties", + "detail": "retentionOverride" + } + ] + }, + { + "description": "lineage granularity outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "granularity": "row_level" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/granularity", + "keyword": "enum" + } + ] + }, + { + "description": "upstream entry omits productId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "exposeId": "events_table" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "required", + "detail": "productId" + } + ] + }, + { + "description": "upstream entry omits exposeId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "upstream entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "sourceSystem": "kafka" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0", + "keyword": "additionalProperties", + "detail": "sourceSystem" + } + ] + }, + { + "description": "fieldMappings entry omits sourceField", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "targetField": "profile_updated_at" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "required", + "detail": "sourceField" + } + ] + }, + { + "description": "fieldMappings entry omits targetField", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "required", + "detail": "targetField" + } + ] + }, + { + "description": "fieldMappings entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "notes": "legacy mapping, do not touch" + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0", + "keyword": "additionalProperties", + "detail": "notes" + } + ] + }, + { + "description": "fieldMappings transformation carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "calculation", + "engine": "spark" + } + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0/transformation", + "keyword": "additionalProperties", + "detail": "engine" + } + ] + }, + { + "description": "fieldMappings transformation type outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "fieldMappings": [ + { + "sourceField": "event_ts", + "targetField": "profile_updated_at", + "transformation": { + "type": "join" + } + } + ] + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/fieldMappings/0/transformation/type", + "keyword": "enum" + } + ] + }, + { + "description": "upstream relationship outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "upstream": [ + { + "productId": "raw_events", + "exposeId": "events_table", + "relationship": "streaming_sync" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/upstream/0/relationship", + "keyword": "enum" + } + ] + }, + { + "description": "downstream entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "downstream": [ + { + "consumer": "churn_model", + "notifyChannel": "slack" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/downstream/0", + "keyword": "additionalProperties", + "detail": "notifyChannel" + } + ] + }, + { + "description": "downstream impact outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "lineage": { + "downstream": [ + { + "consumer": "churn_model", + "impact": "catastrophic" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/lineage/downstream/0/impact", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "strategy": "semantic_versioning", + "migrationGuideUrl": "https://docs.acme.internal/schema-migrations" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution", + "keyword": "additionalProperties", + "detail": "migrationGuideUrl" + } + ] + }, + { + "description": "schemaEvolution strategy outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "strategy": "calendar_based" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "schemaEvolution compatibility outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "compatibility": "none" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/compatibility", + "keyword": "enum" + } + ] + }, + { + "description": "changePolicy carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "changePolicy": { + "changeWindowDays": 14, + "escalationContact": "data-governance@acme.com" + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/changePolicy", + "keyword": "additionalProperties", + "detail": "escalationContact" + } + ] + }, + { + "description": "changePolicy changeWindowDays given as a negative number", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "schemaEvolution": { + "changePolicy": { + "changeWindowDays": -5 + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/schemaEvolution/changePolicy/changeWindowDays", + "keyword": "minimum" + } + ] + }, + { + "description": "machineLearning carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "enabled": true, + "servingEndpoint": "https://models.acme.internal/churn" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning", + "keyword": "additionalProperties", + "detail": "servingEndpoint" + } + ] + }, + { + "description": "machineLearning framework outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "framework": "tensorflow_serving" + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/framework", + "keyword": "enum" + } + ] + }, + { + "description": "model entry omits name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "version": "1.0.0", + "type": "classification" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "model entry omits version", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "type": "classification" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "version" + } + ] + }, + { + "description": "model entry omits type", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "model entry carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "classification", + "trainingDataset": "customers_v3" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0", + "keyword": "additionalProperties", + "detail": "trainingDataset" + } + ] + }, + { + "description": "model type outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "forecasting" + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "model validation carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "machineLearning": { + "models": [ + { + "name": "churn_predictor", + "version": "1.0.0", + "type": "classification", + "validation": { + "metrics": [ + "accuracy" + ], + "reviewedBy": "data-science-lead" + } + } + ] + } + }, + "valid": false, + "errors": [ + { + "pointer": "/machineLearning/models/0/validation", + "keyword": "additionalProperties", + "detail": "reviewedBy" + } + ] + } + ] +} diff --git a/tests/0.7.5/orchestration.json b/tests/0.7.5/orchestration.json new file mode 100644 index 0000000..a82f917 --- /dev/null +++ b/tests/0.7.5/orchestration.json @@ -0,0 +1,8619 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "orchestration", + "description": "The scheduling and workflow-engine surface: the top-level orchestration block and build.execution's inline orchestration twin (engine/mode), build.execution's trigger (type, dataset references, datasetsOperator), and the Airflow-specific shapes reached through orchestration.airflow -- airflowOrchestration's dagId/dagConfig/taskDefaults/errorHandling/tasks/taskGroups, airflowTask's identifier and type-taxonomy fields plus its five conditional allOf branches (provider_action, fluid_execute, bash, python), and airflowSensor. A conformant implementation reproduces every verdict below.", + "cases": [ + { + "description": "orchestration block reaches every mounted definition -- root orchestration, build.execution's trigger and inline orchestration, airflowOrchestration's dagConfig/taskDefaults/errorHandling, every airflowTask type variant (provider_action, fluid_execute, bash, python, custom), a sensor, and a task group -- with legal values throughout", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "a custom-typed task carries a legacy 'operator' alias and a params object that satisfies none of provider_action/fluid_execute/bash/python's type-specific requirements -- valid because its type/operator match none of those allOf branches", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + }, + { + "taskId": "generic_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "deliberately lacks contract_path, bash_command/command, and python_callable/callable" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "a bash task additionally carries the deprecated operator alias alongside type=bash, and its params satisfy the bash param shape but neither the fluid_execute nor python shapes", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + }, + { + "taskId": "legacy_bash_alias_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": true + }, + { + "description": "build.execution carries a top-level member the schema does not define", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + }, + "scheduleHint": "run me first" + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution", + "keyword": "additionalProperties", + "detail": "scheduleHint" + } + ] + }, + { + "description": "build.execution.orchestration is present but omits engine", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration", + "keyword": "required", + "detail": "engine" + } + ] + }, + { + "description": "build.execution.orchestration.engine names an engine outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "step_functions", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration/engine", + "keyword": "enum" + } + ] + }, + { + "description": "build.execution.orchestration.mode names a mode outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "auto" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/orchestration/mode", + "keyword": "enum" + } + ] + }, + { + "description": "build.execution.trigger.type names a trigger type outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "cron_only", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/type", + "keyword": "enum" + } + ] + }, + { + "description": "a trigger dataset given as a data-product reference omits productId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + { + "exposeId": "raw_customers_table" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets/0", + "keyword": "oneOf" + } + ] + }, + { + "description": "a trigger dataset given as a data-product reference omits exposeId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + { + "productId": "raw_customers" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasets/0", + "keyword": "oneOf" + } + ] + }, + { + "description": "trigger.datasetsOperator names a value outside its any/all enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "majority" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/build/execution/trigger/datasetsOperator", + "keyword": "enum" + } + ] + }, + { + "description": "the top-level orchestration block omits engine", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration", + "keyword": "required", + "detail": "engine" + } + ] + }, + { + "description": "the top-level orchestration.engine names an engine outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "temporal", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/engine", + "keyword": "enum" + } + ] + }, + { + "description": "the top-level orchestration.mode names a mode outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "auto", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/mode", + "keyword": "enum" + } + ] + }, + { + "description": "orchestration.airflow is present but omits dagId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow", + "keyword": "required", + "detail": "dagId" + } + ] + }, + { + "description": "orchestration.airflow.dagId contains characters the identifier pattern forbids", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "Customer Profiles DAG!", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/dagId", + "keyword": "pattern" + } + ] + }, + { + "description": "dagConfig.maxActiveRuns is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 0 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/dagConfig/maxActiveRuns", + "keyword": "minimum" + } + ] + }, + { + "description": "taskDefaults.retries is set below its minimum of 0", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": -1, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/retries", + "keyword": "minimum" + } + ] + }, + { + "description": "taskDefaults.triggerRule names a rule outside Airflow's trigger-rule enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "always", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/triggerRule", + "keyword": "enum" + } + ] + }, + { + "description": "taskDefaults.errorHandling.strategy names a strategy outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "immediate", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/strategy", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's action names a value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "page", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/action", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's severity names a value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "urgent", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/severity", + "keyword": "enum" + } + ] + }, + { + "description": "an onError entry's when names a value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "immediately" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskDefaults/errorHandling/onError/0/when", + "keyword": "enum" + } + ] + }, + { + "description": "orchestration.airflow.tasks is present but empty", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks", + "keyword": "minItems" + } + ] + }, + { + "description": "a taskGroups entry omits groupId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "description": "staging tasks, no id" + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/taskGroups/0", + "keyword": "required", + "detail": "groupId" + } + ] + }, + { + "description": "a task omits taskId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "taskId" + } + ] + }, + { + "description": "a task's taskId contains characters the identifier pattern forbids", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "Ensure Bucket!", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/taskId", + "keyword": "pattern" + } + ] + }, + { + "description": "a task's type names a value outside the task-type taxonomy", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "spark_job", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task names a provider outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "oracle", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/provider", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task's action omits the required service.operation dot", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "ensurebucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/action", + "keyword": "pattern" + } + ] + }, + { + "description": "a task's produces entry omits exposeId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/produces/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "a task's produces entry names an updateStrategy outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "overwrite" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/produces/0/updateStrategy", + "keyword": "enum" + } + ] + }, + { + "description": "a task's dataProductDependencies entry omits productId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/dataProductDependencies/0", + "keyword": "required", + "detail": "productId" + } + ] + }, + { + "description": "a task's dataProductDependencies entry omits exposeId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/dataProductDependencies/0", + "keyword": "required", + "detail": "exposeId" + } + ] + }, + { + "description": "a task's xcom push entry names a serializer outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "yaml" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0/xcom/push/0/serializer", + "keyword": "enum" + } + ] + }, + { + "description": "a provider_action task omits provider while action and params remain", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "provider" + } + ] + }, + { + "description": "a provider_action task omits action while provider and params remain", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "action" + } + ] + }, + { + "description": "a provider_action task omits params while provider and action remain", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/0", + "keyword": "required", + "detail": "params" + } + ] + }, + { + "description": "a fluid_execute task's params omit contract_path", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": {} + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/1/params", + "keyword": "required", + "detail": "contract_path" + } + ] + }, + { + "description": "a bash task's params supply neither bash_command nor command", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": {} + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/2/params", + "keyword": "anyOf" + } + ] + }, + { + "description": "a python task's params supply neither python_callable nor callable", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": {} + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/tasks/3/params", + "keyword": "anyOf" + } + ] + }, + { + "description": "a sensor omits taskId", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0", + "keyword": "required", + "detail": "taskId" + } + ] + }, + { + "description": "a sensor omits sensorType", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0", + "keyword": "required", + "detail": "sensorType" + } + ] + }, + { + "description": "a sensor's sensorType names a value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "KafkaSensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/sensorType", + "keyword": "enum" + } + ] + }, + { + "description": "a sensor's pokInterval is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 0, + "timeout": 1800, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/pokInterval", + "keyword": "minimum" + } + ] + }, + { + "description": "a sensor's timeout is set below its minimum of 1", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 0, + "mode": "reschedule" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/timeout", + "keyword": "minimum" + } + ] + }, + { + "description": "a sensor's mode names a value outside the poke/reschedule enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + } + } + ], + "build": { + "pattern": "hybrid-reference", + "engine": "dbt", + "properties": { + "model": "stg_customers" + }, + "execution": { + "trigger": { + "type": "dataset", + "datasets": [ + "dataset://gcp/acme-prod/customers/raw_customers", + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0" + } + ], + "datasetsOperator": "any" + }, + "orchestration": { + "engine": "airflow", + "mode": "generated" + } + } + }, + "orchestration": { + "engine": "airflow", + "mode": "generated", + "generateOnChange": true, + "airflow": { + "dagId": "customer_profiles_dag", + "dagConfig": { + "description": "Builds customer_profiles daily.", + "schedule": "@daily", + "catchup": false, + "maxActiveRuns": 1 + }, + "taskDefaults": { + "retries": 2, + "triggerRule": "all_success", + "errorHandling": { + "strategy": "exponential_backoff", + "onError": [ + { + "action": "notify", + "channel": "#data-platform", + "severity": "high", + "when": "maxAttemptsExceeded" + } + ] + } + }, + "tasks": [ + { + "taskId": "ensure_bucket", + "type": "provider_action", + "operator": "CustomOperator", + "provider": "aws", + "action": "s3.ensure_bucket", + "params": { + "bucket": "acme-raw" + }, + "produces": [ + { + "exposeId": "profiles_table", + "updateStrategy": "replace" + } + ], + "dataProductDependencies": [ + { + "productId": "raw_customers", + "exposeId": "raw_customers_table", + "versionConstraint": "^1.0.0", + "awaitFreshness": "PT6H" + } + ], + "xcom": { + "push": [ + { + "key": "bucket_arn", + "serializer": "json" + } + ] + } + }, + { + "taskId": "validate_contract", + "type": "fluid_execute", + "operator": "CustomOperator", + "params": { + "contract_path": "contracts/customer_profiles.fluid.yaml" + } + }, + { + "taskId": "run_bash_step", + "type": "bash", + "operator": "BashOperator", + "params": { + "bash_command": "echo hi" + } + }, + { + "taskId": "run_python_step", + "type": "python", + "operator": "CustomOperator", + "params": { + "python_callable": "transform.run" + } + }, + { + "taskId": "custom_marker_step", + "type": "custom", + "operator": "CustomOperator", + "params": { + "note": "no special validation applies to a custom-typed task" + } + } + ], + "sensors": [ + { + "taskId": "wait_for_raw_file", + "sensorType": "S3KeySensor", + "pokInterval": 30, + "timeout": 1800, + "mode": "async" + } + ], + "taskGroups": [ + { + "groupId": "staging", + "description": "Staging tasks for customer_profiles.", + "tasks": [ + { + "taskId": "stage_customers", + "type": "custom" + } + ] + } + ] + } + } + }, + "valid": false, + "errors": [ + { + "pointer": "/orchestration/airflow/sensors/0/mode", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/0.7.5/semantic-model.json b/tests/0.7.5/semantic-model.json new file mode 100644 index 0000000..4ed9686 --- /dev/null +++ b/tests/0.7.5/semantic-model.json @@ -0,0 +1,1974 @@ +{ + "corpusVersion": "0.7.5", + "schema": "https://open-data-protocol.github.io/fluid/schema/fluid-schema-0.7.5.json", + "group": "semantic-model", + "description": "The expose-level semantic model block (expose.semantics): its own closed object type, and the entities/measures/dimensions/metrics arrays -- their required members, closed property sets, and the type/agg enums, including the nested nonAdditiveDimension and typeParams sub-objects.", + "cases": [ + { + "description": "semantic model on a table expose carries entities, measures (with a non-additive dimension), dimensions (with typeParams), and metrics", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": true + }, + { + "description": "semantic model block carries an unrecognized top-level property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ], + "grain": "one row per customer" + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics", + "keyword": "additionalProperties", + "detail": "grain" + } + ] + }, + { + "description": "entity omits its name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "entity omits its type", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "entity carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id", + "sensitivity": "high" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0", + "keyword": "additionalProperties", + "detail": "sensitivity" + } + ] + }, + { + "description": "entity type value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "compound", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/entities/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "measure omits its name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "measure omits its aggregation function", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "required", + "detail": "agg" + } + ] + }, + { + "description": "measure carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + }, + "unit": "USD" + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0", + "keyword": "additionalProperties", + "detail": "unit" + } + ] + }, + { + "description": "measure aggregation function outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "stddev", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/agg", + "keyword": "enum" + } + ] + }, + { + "description": "measure's non-additive dimension carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max", + "granularity": "day" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/nonAdditiveDimension", + "keyword": "additionalProperties", + "detail": "granularity" + } + ] + }, + { + "description": "non-additive dimension windowChoice value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "avg" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/measures/0/nonAdditiveDimension/windowChoice", + "keyword": "enum" + } + ] + }, + { + "description": "dimension omits its name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "dimension omits its type", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "dimension carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + }, + "sortOrder": "asc" + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0", + "keyword": "additionalProperties", + "detail": "sortOrder" + } + ] + }, + { + "description": "dimension type value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "geospatial", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/type", + "keyword": "enum" + } + ] + }, + { + "description": "dimension typeParams carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day", + "format": "ISO8601" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/typeParams", + "keyword": "additionalProperties", + "detail": "format" + } + ] + }, + { + "description": "dimension typeParams timeGranularity value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "fortnight" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/dimensions/0/typeParams/timeGranularity", + "keyword": "enum" + } + ] + }, + { + "description": "metric omits its name", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "required", + "detail": "name" + } + ] + }, + { + "description": "metric omits its type", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "required", + "detail": "type" + } + ] + }, + { + "description": "metric carries an unrecognized property", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "simple", + "measure": "lifetime_value", + "filter": "status = 'active'", + "unit": "count" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0", + "keyword": "additionalProperties", + "detail": "unit" + } + ] + }, + { + "description": "metric type value outside the enum", + "contract": { + "fluidVersion": "0.7.5", + "kind": "DataProduct", + "id": "customer_profiles", + "name": "Customer Profiles", + "metadata": { + "owner": { + "team": "customer-analytics" + } + }, + "exposes": [ + { + "exposeId": "profiles_table", + "kind": "table", + "binding": { + "platform": "gcp", + "format": "bigquery_table", + "location": { + "project": "acme-prod", + "dataset": "customers", + "table": "profiles" + } + }, + "contract": { + "schema": [ + { + "name": "customer_id", + "type": "string" + } + ] + }, + "semantics": { + "name": "customer_profile_model", + "description": "Canonical semantic model for customer lifetime-value analytics.", + "defaultAggTimeDimension": "signup_date", + "entities": [ + { + "name": "customer", + "type": "primary", + "expr": "customer_id" + } + ], + "measures": [ + { + "name": "lifetime_value", + "agg": "sum", + "expr": "ltv_amount", + "nonAdditiveDimension": { + "name": "balance_date", + "windowChoice": "max" + } + } + ], + "dimensions": [ + { + "name": "signup_date", + "type": "time", + "expr": "created_at", + "typeParams": { + "timeGranularity": "day" + } + }, + { + "name": "region", + "type": "categorical" + } + ], + "metrics": [ + { + "name": "active_customers", + "type": "computed", + "measure": "lifetime_value", + "filter": "status = 'active'" + } + ] + } + } + ] + }, + "valid": false, + "errors": [ + { + "pointer": "/exposes/0/semantics/metrics/0/type", + "keyword": "enum" + } + ] + } + ] +} diff --git a/tests/README.md b/tests/README.md index 9ab40fa..9ad9861 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,6 +148,31 @@ JSON pointers. It becomes a silent-acceptance bug the moment a schema uses `maxContains` — Draft 7 ignores unknown keywords, so the implementation would accept documents the published standard rejects, with no error anywhere. +## Measuring what the corpus is worth + +Counting cases measures effort. `conformance/mutation_coverage.py` measures +coverage: it deletes one schema constraint at a time and re-runs the corpus. + +```bash +python3 conformance/mutation_coverage.py # whole corpus +python3 conformance/mutation_coverage.py --group agentpolicy # one group +python3 conformance/mutation_coverage.py --min-coverage 40 # gate mode +``` + +A constraint is **covered** when deleting it turns some case red, **uncovered** +when the corpus does not notice, and **redundant** when no case could ever +notice — because a sibling `enum` already restricts the instance to a set that +satisfies the constraint anyway. `fluidVersion` is the clearest example: it +carries `pattern`, `type: string` and an `enum` of three version strings, so +the pattern and the type are dead weight. Redundant constraints are excluded +from the denominator; counting them would blame the corpus for a gap it cannot +close, and they are findings about the *schema* instead. + +Today: **40.7%** (415 of 1020 falsifiable constraints), with 120 redundant. +Most of what remains is `type` constraints, which need a wrong-typed value for +every property in the schema. Of the constraints that carry real meaning — +enums, required members, closed objects, bounds — 38 remain unpinned. + ## Adding a version 1. Publish `schema/fluid-schema-.json` (authored in forge-cli, synced