From 52f22aef6742808d31e724a6f7519c54f638f945 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:57:10 +0200 Subject: [PATCH] docs: correct a false compatibility claim, add the governance a standard needs Two gaps between what this repository says about itself and what is true. ## The 0.7.2 release note was wrong docs/releases/0.7.2.md claimed "100% backward compatible with 0.7.1" and "No breaking changes". The compatibility gate added in #44 disproved it: $defs/notification gained additionalProperties: false, so a contract carrying any extra member on a build.execution.notifications[] entry validated under 0.7.1 and is rejected by 0.7.2. The page now names the change, shows the contract that breaks, and links to the waiver holding the evidence. Finding a false claim and leaving it standing because it is old would make the gate decorative. Also: docs/schema/versions.md said "the latest version is 0.7.4" and did not list 0.7.5 at all, while 0.7.5 is published in schema/, announced in the README and targeted by the whole conformance corpus. The public Schema Versions page was a release behind. Both linked 0.7.5 assets were confirmed present before linking them. ## The repo had no governance files at all 0 of 5, while forge-cli has 5/5 and forge-cli-sdk and FLUX have 4/5. A repository describing itself as "the open, declarative standard for Data Products" offered no route to contribute and no way to report a vulnerability privately. Added: CONTRIBUTING.md leads with the rule that surprises everyone: schema/ is vendored from forge-cli and a PR editing it will be reverted by the next sync. Says what IS editable here, and sets the corpus bar as falsifiability rather than case count. GOVERNANCE.md steward-maintainer, stated plainly rather than dressed up as multi-vendor governance that does not exist. Names the conflict of interest -- the maintainers also ship a commercial control plane on FLUID -- and lists the structural mitigations: MIT, no CLA, DCO, compatibility enforced by CI, ambiguities written down. SECURITY.md private vulnerability reporting, with a threat model appropriate to a spec repo: a schema that fails open, a case that passes for the wrong reason, a gate that cannot fail. CODE_OF_CONDUCT.md Contributor Covenant 2.1, with a reporting route. NOTICE is deliberately NOT added. It has to name a legal entity, and the estate currently carries five different copyright strings -- "Agentics Transformation Limited" (forge-cli), "Agentics Transformation Pty Ltd" (forge-cli-sdk), "Ltd" (flux-engine), "The FLUX Project authors" (flux-spec) and "fluid-steward" (this repo's LICENSE). Picking one here would bake in a guess. That decision is the owner's. ## Removed fluid-schema-0.3.1 -- a 12 KB JSON schema at the repo root with no .json extension, an $id pointing at open-data-protocol.org (not a domain we control) labelled "v2.0", never present in schema/, never listed in versions.md, never rendered to specs/, and referenced by nothing. Recoverable with `git show 550dea8:fluid-schema-0.3.1`. ## Verified Link-check's base-prefix rule run locally (passes). VuePress build green -- and because that build exits 0 even when a page fails to render, both edited pages were checked in the rendered HTML: the correction and its example appear, the heading no longer claims full compatibility, the only surviving instance of the old phrase is inside the quoted correction, and versions.html shows 0.7.5 as latest with both linked assets present in dist/. Corpus 480/480, meta-test 19/19, compat gate clean. Co-Authored-By: Claude Opus 5 --- CODE_OF_CONDUCT.md | 54 +++++ CONTRIBUTING.md | 114 ++++++++++ GOVERNANCE.md | 78 +++++++ SECURITY.md | 75 +++++++ docs/releases/0.7.2.md | 32 ++- docs/schema/versions.md | 5 +- fluid-schema-0.3.1 | 452 ---------------------------------------- 7 files changed, 354 insertions(+), 456 deletions(-) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 GOVERNANCE.md create mode 100644 SECURITY.md delete mode 100644 fluid-schema-0.3.1 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3c05e10 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,54 @@ +# Code of Conduct + +## The short version + +Be straight with people and assume they are doing the same. Argue about the +work, not the person doing it. Technical disagreement is the point of an open +specification and is always welcome; contempt is not. + +## Expected behaviour + +- Engage with what someone actually wrote, not the weakest version of it. +- Say what you verified and how. In a project whose whole premise is that claims + should be checkable, "I ran X and it printed Y" carries more weight than + confidence. +- Be willing to be wrong in public. Correcting a claim is normal maintenance + here — this repository ships a release note correcting one of its own. +- Give people room to be new. The schema-authorship rule in + [CONTRIBUTING.md](CONTRIBUTING.md) catches almost everyone the first time. + +## Unacceptable behaviour + +- Harassment, insults, or personal attacks, public or private. +- Discriminatory language or conduct, including about protected characteristics. +- Publishing others' private information without permission. +- Sustained disruption of discussion, or deliberately arguing in bad faith. +- Sexualised language, imagery, or attention in any project space. + +## Scope + +This applies in every project space — issues, pull requests, discussions, +commits, and any channel where you are representing the project. + +## Reporting + +Report conduct concerns to **change@agenticstransformation.com**. + +Reports are handled confidentially. You will get an acknowledgement within 48 +hours and an indication of what we intend to do. If your report concerns a +maintainer, say so in the message — it will be handled by someone else. + +We would rather hear about something small and early than a pattern that has +been building for months. + +## Enforcement + +Maintainers will act in proportion to what happened: a private word, a public +correction, editing or removing contributions, or a temporary or permanent ban +from project spaces. We will say what we decided and why, in as much detail as +respects the reporter's privacy. + +## Attribution + +Adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d5015ae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,114 @@ +# Contributing to FLUID + +Thanks for helping build FLUID. Read the first section before opening a pull +request — it is the rule that most often surprises people. + +## The schemas are not authored here + +`schema/fluid-schema-*.json` is **vendored**. The FLUID JSON Schemas are +authored in [`forge-cli`](https://github.com/Agenticstiger/forge-cli), the +reference implementation, and land here through +[`.github/workflows/schema-sync.yml`](.github/workflows/schema-sync.yml), which +fails the build if the two ever drift. This repository is the public +distribution point. + +**So a pull request that edits a file under `schema/` will be rejected**, not +because the change is wrong but because it will be silently reverted by the next +sync. Propose schema changes upstream in forge-cli. The sync currently covers +0.7.2 and later; earlier versions diverge historically and are excluded on +purpose. + +What you *can* change here: + +| Area | Notes | +|---|---| +| `tests/**` | The conformance corpus. This is where "FLUID-conformant" is defined — see [tests/README.md](tests/README.md). | +| `conformance/**` | The runner, the reference check, the coverage tool. | +| `scripts/check-compat.py`, `scripts/compat-waivers.txt` | The backward-compatibility gate. | +| `docs/**` | The documentation site. | +| `examples/**` | Example contracts. | + +## Setup + +```bash +pip install "jsonschema[format]>=4.22" + +python3 conformance/run.py # the corpus must be green +python3 tests/meta_test.py # the gates must be able to fail +python3 scripts/check-compat.py # no release may break its promise +``` + +The `[format]` extra is not optional. Without it `jsonschema` registers no +`date-time` checker, and the `tests/optional/` cases pass vacuously in one +environment and fail in another. + +Optionally, to check the corpus against the reference implementation: + +```bash +pip install data-product-forge +python3 conformance/check_reference.py +``` + +## Adding conformance cases + +The corpus is the most valuable thing you can contribute, because it is what +makes conformance a fact rather than a claim. [tests/README.md](tests/README.md) +has the file format and the rules in full; the short version: + +- Assert on **JSON Schema keyword + RFC 6901 pointer**, never message text. + Wording is a rendering choice; keywords and pointers are interoperable facts. +- An invalid case must declare **why** it is invalid. The loader refuses one + that does not — a case asserting only "rejected" passes for any reason at all. +- Keep cases **minimal**: an invalid document differs from a valid one in + exactly the one way under test. +- The bar is **falsifiability**, not case count. Run + `python3 conformance/mutation_coverage.py --group `; if deleting the + constraint you meant to pin leaves the corpus green, your case is not doing + the work you think it is. + +## Changing the schema's meaning + +Anything that changes which documents validate is normative. Open an issue +before writing code, and expect to be asked for the corpus cases that pin the +new behaviour. Editorial changes — typos, prose, descriptions that do not affect +validation — can go straight to a pull request. + +## The compatibility promise + +Every FLUID release has claimed that valid contracts keep validating. That +promise is now enforced by `scripts/check-compat.py` on every pull request. + +If it reports a narrowing change, one of two things is true: the change is a +mistake, or it is deliberate. A deliberate one goes in +[`scripts/compat-waivers.txt`](scripts/compat-waivers.txt) **with the evidence +that settled it**. The waiver file is a decision record, not a mute button — +read the existing entries for the standard being applied. + +## Pull requests + +- One concern per pull request. +- Say what you verified and how. "Tests pass" is less useful than the command + you ran and what it printed. +- New behaviour needs a case that would fail without it. +- Be honest about what you did not do. A known gap that is written down is worth + more than a claim nobody checked. + +## Reporting problems + +- **Bugs and spec questions** — open an issue. +- **Security vulnerabilities** — do not open an issue. See + [SECURITY.md](SECURITY.md). +- **Conduct** — see [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). + +## Licence and provenance + +Contributions are accepted under this repository's [MIT licence](LICENSE). By +opening a pull request you certify that you wrote the contribution or otherwise +have the right to submit it under that licence — the +[Developer Certificate of Origin](https://developercertificate.org/). Sign your +commits with `git commit -s` if you would like that certification recorded +explicitly. + +There is no contributor licence agreement. A CLA on a specification would signal +reserved relicensing rights, which is not the intent — see +[GOVERNANCE.md](GOVERNANCE.md). diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 0000000..bc99863 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,78 @@ +# Governance + +## Current model: steward-maintainer + +FLUID is pre-1.0. The specification is stewarded by its originating maintainers +in the [open-data-protocol](https://github.com/open-data-protocol) organization. +Decisions on spec changes are made by the maintainers, in the open, through +issues and pull requests on this repository. + +This is an honest description of a small project, not a claim of formal +multi-vendor governance. There is no steering committee and no vote, because +there is not yet a constituency to hold one. + +## Where things are decided + +FLUID has two homes, on purpose, and the split matters for anyone proposing a +change: + +| Artefact | Authored in | Why | +|---|---|---| +| The JSON Schemas | [`forge-cli`](https://github.com/Agenticstiger/forge-cli) | It is the only place with an exercised test suite over them, and it is the engine that enforces them. | +| The conformance corpus | **here** | Conformance must be checkable by someone who has neither the engine nor an account with us. | + +That second row is the load-bearing one. A standard whose only definition is +"whatever the reference implementation accepts" is a vendor format with a +specification-shaped document attached. The corpus in [`tests/`](tests/) is what +makes "FLUID-conformant" mean something independent, and forge-cli is +implementation #1 under test — never the referee. + +## Vendor neutrality + +FLUID already lives in a vendor-neutral organization. The maintainers are +employed by a company that also ships a commercial control plane built on FLUID, +and that is a real conflict of interest, so the mitigations are structural +rather than promissory: + +- The schemas are **MIT** and the corpus is data. Anyone can implement FLUID and + demonstrate conformance without permission, tooling, or a relationship with us. +- **No CLA.** Contributions are accepted under the repository licence via the + [DCO](https://developercertificate.org/). There is no mechanism by which + contributed work could be relicensed out from under contributors. +- Compatibility is **enforced by CI**, not asserted in prose, so a change that + would strand existing contracts is visible to everyone at review time rather + than discovered later by users. +- Anything the specification does not decide is written down as an open question + rather than settled silently by implementation behaviour. See "Known + ambiguities" in [tests/README.md](tests/README.md). + +As FLUID gains external implementers, the intent is to move toward a formal +governance model with representation from parties who do not share an employer. +Concrete triggers: a second independent implementation demonstrating +conformance, or sustained contribution from outside the current maintainers. + +## Versioning and compatibility + +Pre-1.0, minor versions may add. They may not narrow: a contract valid under one +version must stay valid under its successor. `scripts/check-compat.py` enforces +this on every pull request, and deliberate exceptions are recorded in +[`scripts/compat-waivers.txt`](scripts/compat-waivers.txt) with the evidence that +justified them. + +One historical break is on record (0.7.1 → 0.7.2, `notification` became a closed +object). It was found by the gate after the fact, and the release note that +claimed otherwise has been corrected rather than quietly left standing. + +## Trademarks + +"FLUID" as a name for this specification is reserved by the maintainers. The +format is free to implement, describe, and build products on; the mark exists so +that "FLUID-conformant" keeps meaning what the corpus says it means. If you ship +an implementation, say so plainly — you do not need permission to state that +your tool implements FLUID. + +## Contact + +Open an issue for anything about the specification. See +[SECURITY.md](SECURITY.md) for vulnerabilities and +[CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for conduct concerns. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..db65854 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,75 @@ +# Security Policy + +## What "security" means for a specification repository + +This repository publishes JSON Schemas, a conformance corpus, and the small +Python tools that check them. It is not a service and holds no user data, so the +realistic threat model is narrower than for an application — but not empty: + +- A schema that **fails open** — accepting a contract it should reject — can let + an unsafe data product through a downstream gate that trusted it. +- A conformance case that **passes for the wrong reason** launders a false + assurance into every implementation that trusts the corpus. +- The **compatibility gate** or the **conformance runner** silently not doing + what it claims. A gate that cannot fail is worse than no gate, because it is + believed. This is why [`tests/meta_test.py`](tests/meta_test.py) exists and + runs first in CI. +- Anything that would cause a consumer to execute code, exfiltrate data, or + escalate privilege while processing a published artefact. + +If you have found something in those categories, we want to hear about it, even +if you are unsure it qualifies. + +## Supported versions + +| Schema version | Supported | +|---|---| +| 0.7.5 | ✅ | +| 0.7.3 – 0.7.4 | ✅ | +| ≤ 0.7.2 | ❌ | + +Fixes land on the current version. Because the compatibility promise forbids +narrowing, a fix that would reject a previously-valid contract is a version +increment, not a patch to a published schema — a published schema is an +immutable artefact that people validate against by URL. + +## Reporting a vulnerability + +**Please do not report security vulnerabilities through public GitHub issues.** + +Use GitHub's +[Private Vulnerability Reporting](https://github.com/open-data-protocol/fluid/security/advisories/new). +It keeps the report private until a fix ships, gives us somewhere to draft the +advisory, and lets you follow the fix without leaving GitHub. + +If you cannot use that form, email **change@agenticstransformation.com**. + +You should get a response within **48 hours**. If you do not, please follow up — +assume the message was lost rather than ignored. + +Please include as much as you can: + +- What the issue is, and which artefact it affects (schema version, corpus file, + or tool). +- A contract, corpus case, or command that demonstrates it. +- What you expected to happen and what happened instead. +- Any impact you can see on downstream consumers. + +## What happens next + +1. We acknowledge within 48 hours. +2. We reproduce it, and tell you if we cannot. +3. We agree a fix and a disclosure timeline with you. We will not sit on a + confirmed issue quietly; if we disagree on severity we will say so rather + than let it go silent. +4. We credit you in the advisory unless you would rather we did not. + +## Scope + +**In scope:** the schemas under `schema/`, the corpus under `tests/`, and the +tools under `conformance/` and `scripts/`. + +**Out of scope here, but still worth reporting to their owners:** the reference +implementation ([forge-cli](https://github.com/Agenticstiger/forge-cli)) and any +product built on FLUID. If you are unsure which, report it here and we will route +it. diff --git a/docs/releases/0.7.2.md b/docs/releases/0.7.2.md index bc2111e..5a8ac36 100644 --- a/docs/releases/0.7.2.md +++ b/docs/releases/0.7.2.md @@ -54,11 +54,39 @@ binding: --- -## 🔄 100% backward compatible with 0.7.1 +## 🔄 Backward compatibility with 0.7.1 — one narrowing change -- ✅ No breaking changes - ✅ `semantics` and `icebergConfig` are both opt-in - ✅ All 0.7.1 features (agentPolicy, sovereignty, provider-first orchestration, root accessPolicy) fully preserved +- ⚠️ **One breaking change**, corrected here after the fact: `notification` became a + closed object. + +This page previously claimed "100% backward compatible" and "no breaking changes". +That was not true, and the conformance gate added in +[#44](https://github.com/open-data-protocol/fluid/pull/44) proved it. `$defs/notification` +gained `"additionalProperties": false` in 0.7.2, so a contract carrying **any** extra +member on a `build.execution.notifications[]` entry validated under 0.7.1 and is +rejected by 0.7.2: + +```yaml +build: + execution: + notifications: + - type: slack + target: "#data-alerts" + condition: failure + retryOnFailure: true # accepted by 0.7.1, REJECTED by 0.7.2 +``` + +Verified with that document plus a negative control — the same contract without +`retryOnFailure` validates under both — so the rejection is attributable to this +change and nothing else. The finding is recorded in +[`scripts/compat-waivers.txt`](https://github.com/open-data-protocol/fluid/blob/main/scripts/compat-waivers.txt), +and `scripts/check-compat.py` now runs on every pull request so a narrowing change +cannot ship unannounced again. + +If you hit this, drop the unrecognised member from your `notifications` entries; every +field the schema does declare is unchanged. **Migration:** diff --git a/docs/schema/versions.md b/docs/schema/versions.md index 3d50541..feeb138 100644 --- a/docs/schema/versions.md +++ b/docs/schema/versions.md @@ -1,10 +1,11 @@ # Schema Versions -Every published version of the FLUID JSON Schema. Point your validator at a version's JSON Schema; read its generated HTML reference for the field-by-field detail. The latest version is **0.7.4**. +Every published version of the FLUID JSON Schema. Point your validator at a version's JSON Schema; read its generated HTML reference for the field-by-field detail. The latest version is **0.7.5**. | Version | JSON Schema | HTML Reference | |---|---|---| -| **0.7.4** **(latest)** | [`fluid-schema-0.7.4.json`](/fluid/schema/fluid-schema-0.7.4.json) | [`0.7.4/fluid-spec.html`](/fluid/specs/0.7.4/fluid-spec.html) | +| **0.7.5** **(latest)** | [`fluid-schema-0.7.5.json`](/fluid/schema/fluid-schema-0.7.5.json) | [`0.7.5/fluid-spec.html`](/fluid/specs/0.7.5/fluid-spec.html) | +| 0.7.4 | [`fluid-schema-0.7.4.json`](/fluid/schema/fluid-schema-0.7.4.json) | [`0.7.4/fluid-spec.html`](/fluid/specs/0.7.4/fluid-spec.html) | | 0.7.3 | [`fluid-schema-0.7.3.json`](/fluid/schema/fluid-schema-0.7.3.json) | [`0.7.3/fluid-spec.html`](/fluid/specs/0.7.3/fluid-spec.html) | | 0.7.2 | [`fluid-schema-0.7.2.json`](/fluid/schema/fluid-schema-0.7.2.json) | [`0.7.2/fluid-spec.html`](/fluid/specs/0.7.2/fluid-spec.html) | | 0.7.1 | [`fluid-schema-0.7.1.json`](/fluid/schema/fluid-schema-0.7.1.json) | [`0.7.1/fluid-spec.html`](/fluid/specs/0.7.1/fluid-spec.html) | diff --git a/fluid-schema-0.3.1 b/fluid-schema-0.3.1 deleted file mode 100644 index 4e4e6fc..0000000 --- a/fluid-schema-0.3.1 +++ /dev/null @@ -1,452 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://open-data-protocol.org/fluid/fluid.schema.v2.0.json", - "title": "FLUID (Federated Layered Unified Interchange Definition) Specification", - "description": "A comprehensive, production-ready schema for FLUID data product contracts, merging rich operational details with a streamlined, developer-friendly authoring experience.", - "type": "object", - "additionalProperties": false, - "required": [ - "fluidVersion", - "kind", - "id", - "name", - "description", - "domain", - "metadata", - "exposes" - ], - "properties": { - "fluidVersion": { - "type": "string", - "description": "Version of the FLUID spec this contract adheres to.", - "pattern": "^2\\.\\d+(\\.\\d+)?$", - "examples": ["2.0.0"] - }, - "kind": { - "type": "string", - "description": "The type of data product definition.", - "enum": [ - "DataProduct", - "VirtualDataProduct", - "EgressFlow", - "IngestFlow", - "MLModelProduct", - "FeatureStoreProduct" - ] - }, - "id": { - "type": "string", - "description": "Globally-unique, versioned data product identifier.", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable product name." - }, - "description": { - "type": "string", - "description": "A brief, business-focused description of the product's purpose." - }, - "domain": { - "type": "string", - "description": "The owning business domain (e.g., 'Marketing', 'Finance')." - }, - "metadata": { - "$ref": "#/$defs/metadata" - }, - "consumes": { - "description": "An optional list of input data sources required to build the product.", - "type": "array", - "items": { - "$ref": "#/$defs/consume" - } - }, - "build": { - "$ref": "#/$defs/build" - }, - "exposes": { - "description": "The public output interfaces (ports) of the data product.", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/expose" - } - }, - "slo": { - "$ref": "#/$defs/sla" - }, - "accessPolicy": { - "$ref": "#/$defs/accessPolicy" - }, - "operations": { - "$ref": "#/$defs/operations" - }, - "security": { - "$ref": "#/$defs/security" - }, - "governance": { - "$ref": "#/$defs/governance" - } - }, - "$defs": { - "metadata": { - "type": "object", - "properties": { - "layer": { - "type": "string", - "enum": ["Bronze", "Silver", "Gold", "Platinum"], - "description": "The architectural layer of the data product." - }, - "owner": { - "description": "The team or individual responsible for the data product.", - "oneOf": [ - { - "type": "string", - "description": "A contact email address for the owner." - }, - { - "type": "object", - "properties": { - "team": { "type": "string" }, - "email": { "type": "string", "format": "email" }, - "slack": { "type": "string" } - }, - "required": ["team"] - } - ] - }, - "status": { - "type": "string", - "enum": ["Development", "Published", "Deprecated"], - "default": "Development" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "A list of arbitrary tags for categorization." - } - }, - "required": ["layer", "owner"] - }, - "consume": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "A local alias for the consumed data source." - }, - "ref": { - "type": "string", - "description": "A reference to another data product (e.g., URN)." - }, - "description": { - "type": "string" - } - }, - "required": ["id", "ref"] - }, - "build": { - "type": "object", - "description": "Describes the logical transformation process and its operational details.", - "properties": { - "engine": { - "type": "string", - "description": "The transformation engine used (e.g., 'dbt', 'spark')." - }, - "model": { - "type": "string", - "description": "A reference to the specific model or script (e.g., dbt model path)." - }, - "config": { - "type": "object", - "description": "Engine-specific configuration key-value pairs.", - "additionalProperties": true - }, - "trigger": { - "$ref": "#/$defs/trigger" - }, - "runtime": { - "$ref": "#/$defs/runtime" - }, - "retries": { - "$ref": "#/$defs/retryPolicy" - }, - "notifications": { - "type": "array", - "items": { - "$ref": "#/$defs/notification" - } - } - }, - "required": ["engine", "model"] - }, - "expose": { - "type": "object", - "description": "A single output port of the data product.", - "properties": { - "id": { - "type": "string", - "description": "The unique identifier for this output port." - }, - "type": { - "type": "string", - "description": "The physical type of the output (e.g., 'snowflake_table', 'bigquery_view')." - }, - "description": { - "type": "string" - }, - "location": { - "$ref": "#/$defs/location" - }, - "tags": { - "type": "object", - "properties": { - "archetype": { - "type": "string", - "enum": ["hub", "satellite", "link", "view", "table"], - "description": "The modeling archetype of the port." - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/relationship" - } - } - } - }, - "schema": { - "type": "array", - "items": { - "$ref": "#/$defs/column" - } - }, - "quality": { - "type": "array", - "items": { - "$ref": "#/$defs/qualityRule" - } - }, - "privacy": { - "type": "array", - "items": { - "$ref": "#/$defs/privacyRule" - } - }, - "semantics": { - "$ref": "#/$defs/semantics" - } - }, - "required": ["id", "type", "schema", "location"] - }, - "column": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "description": "The physical data type." - }, - "description": { - "type": "string" - }, - "nullable": { - "type": "boolean", - "default": true - }, - "semantic": { - "type": "string", - "description": "Reference to an ontology term or glossary ID." - }, - "tags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Semantic tags like 'primary_key', 'foreign_key', 'pii'." - } - }, - "required": ["name", "type"] - }, - "relationship": { - "type": "object", - "properties": { - "to": { - "type": "string", - "description": "The 'id' of the target port for the relationship." - }, - "cardinality": { - "type": "string", - "enum": ["one-to-one", "one-to-many", "many-to-one", "many-to-many"] - }, - "description": { - "type": "string" - } - }, - "required": ["to", "cardinality"] - }, - "qualityRule": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "rule": { "type": "string" }, - "onFailure": { - "type": "object", - "properties": { - "action": { "type": "string", "enum": ["fail_pipeline", "alert"] } - }, - "required": ["action"] - } - }, - "required": ["name", "rule", "onFailure"] - }, - "privacyRule": { - "type": "object", - "properties": { - "classification": { "type": "string", "enum": ["PII", "SPI", "Confidential"] }, - "columns": { "type": "array", "items": { "type": "string" } }, - "treatment": { - "type": "object", - "properties": { - "type": { "type": "string", "enum": ["hashing", "masking", "encryption"] } - }, - "required": ["type"] - } - }, - "required": ["columns", "treatment"] - }, - "semantics": { - "type": "object", - "properties": { - "ontology": { "type": "string", "format": "uri" }, - "classifications": { "type": "object" } - } - }, - "location": { - "type": "object", - "properties": { - "format": { "type": "string", "enum": ["parquet", "delta", "iceberg", "json", "csv"] }, - "properties": { - "type": "object", - "description": "Technology-specific properties (e.g., project, dataset, table)." - } - }, - "required": ["properties"] - }, - "trigger": { - "type": "object", - "description": "Defines how the build is initiated.", - "oneOf": [ - { - "properties": { - "type": { "const": "schedule" }, - "cron": { "type": "string" } - }, - "required": ["type", "cron"] - }, - { - "properties": { - "type": { "const": "event" }, - "eventType": { "type": "string" } - }, - "required": ["type", "eventType"] - }, - { - "properties": { "type": { "const": "manual" } }, - "required": ["type"] - } - ] - }, - "runtime": { - "type": "object", - "properties": { - "platform": { "type": "string" }, - "resources": { "type": "object" } - }, - "required": ["platform"] - }, - "retryPolicy": { - "type": "object", - "properties": { - "count": { "type": "integer", "minimum": 0, "default": 0 }, - "delaySeconds": { "type": "integer", "minimum": 0, "default": 0 }, - "backoff": { "type": "string", "enum": ["none", "exponential"], "default": "none" } - } - }, - "notification": { - "type": "object", - "properties": { - "channel": { "type": "string", "enum": ["email", "slack", "pagerduty"] }, - "target": { "type": "string" } - }, - "required": ["channel", "target"] - }, - "accessPolicy": { - "type": "object", - "properties": { - "grants": { - "type": "array", - "items": { - "$ref": "#/$defs/accessGrant" - } - } - } - }, - "accessGrant": { - "type": "object", - "properties": { - "principal": { "type": "string" }, - "permissions": { "type": "array", "items": { "type": "string" } } - }, - "required": ["principal", "permissions"] - }, - "operations": { - "type": "object", - "properties": { - "sla": { "$ref": "#/$defs/sla" }, - "lifecycle": { "$ref": "#/$defs/lifecycle" } - } - }, - "sla": { - "type": "object", - "properties": { - "latencyMs": { "type": "integer" }, - "freshnessMinutes": { "type": "integer" }, - "availabilityPct": { "type": "number" } - } - }, - "lifecycle": { - "type": "object", - "properties": { - "retentionPeriodDays": { "type": "integer" }, - "deletionPolicy": { "type": "string", "enum": ["hard-delete", "soft-delete", "anonymize"] } - } - }, - "security": { - "type": "object", - "properties": { - "encryptionAtRest": { "type": "string" }, - "encryptionInTransit": { "type": "string" } - } - }, - "governance": { - "type": "object", - "properties": { - "rules": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "requirement": { "type": "string" } - }, - "required": ["name", "requirement"] - } - } - } - } - } -}