feat(conformance): make FLUID's claims executable

.github/workflows/conformance.yml

@@ -0,0 +1,122 @@
+# The gate that makes FLUID a standard rather than a schema with a docs site.
+#
+# Three jobs, deliberately separate so a red build says WHICH promise broke:
+#
+#   meta         proves the other two gates can fail. Runs first: a green
+#                conformance run means nothing if the runner cannot go red.
+#   conformance  runs the vector corpus against the published schemas, and
+#                again against the reference implementation (forge-cli) so
+#                the corpus and the engine cannot drift apart unnoticed.
+#   compat       enforces the backward-compatibility promise the README and
+#                every release note make.
+
+name: conformance
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "schema/**"
+      - "tests/**"
+      - "conformance/**"
+      - "scripts/check-compat.py"
+      - "scripts/compat-waivers.txt"
+      - ".github/workflows/conformance.yml"
+  pull_request:
+    paths:
+      - "schema/**"
+      - "tests/**"
+      - "conformance/**"
+      - "scripts/check-compat.py"
+      - "scripts/compat-waivers.txt"
+      - ".github/workflows/conformance.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  meta:
+    name: gates can fail
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # The [format] extra is REQUIRED, not optional polish: without it,
+      # jsonschema silently registers no checker for date-time (it needs
+      # rfc3339-validator), so tests/optional/ format assertions pass
+      # vacuously in one environment and fail in another. Pinning the extra
+      # is what makes a local run and a CI run mean the same thing.
+      - run: pip install "jsonschema[format]>=4.22"
+      - name: prove the conformance runner and compat gate can go red
+        run: python3 tests/meta_test.py
+
+  conformance:
+    name: corpus
+    runs-on: ubuntu-latest
+    needs: meta
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # The [format] extra is REQUIRED, not optional polish: without it,
+      # jsonschema silently registers no checker for date-time (it needs
+      # rfc3339-validator), so tests/optional/ format assertions pass
+      # vacuously in one environment and fail in another. Pinning the extra
+      # is what makes a local run and a CI run mean the same thing.
+      - run: pip install "jsonschema[format]>=4.22"
+
+      - name: run the corpus against the published schemas
+        run: python3 conformance/run.py --junit conformance-report.xml
+
+      # The corpus defines conformance; forge-cli is implementation #1 under
+      # test, never the referee. Installing latest-stable here is deliberate:
+      # it is how the Command Center installs it, so a divergence between the
+      # standard and the engine surfaces on this repo's PRs rather than in a
+      # customer's container.
+      - name: install the reference implementation
+        run: pip install data-product-forge
+        continue-on-error: true
+        id: install_forge
+
+      - name: check the reference implementation against the corpus
+        if: steps.install_forge.outcome == 'success'
+        run: python3 conformance/check_reference.py
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conformance-report
+          path: conformance-report.xml
+
+  compat:
+    name: backward compatibility
+    runs-on: ubuntu-latest
+    needs: meta
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # The [format] extra is REQUIRED, not optional polish: without it,
+      # jsonschema silently registers no checker for date-time (it needs
+      # rfc3339-validator), so tests/optional/ format assertions pass
+      # vacuously in one environment and fail in another. Pinning the extra
+      # is what makes a local run and a CI run mean the same thing.
+      - run: pip install "jsonschema[format]>=4.22"
+
+      # Pre-0.7.1 schemas predate the compatibility promise and already
+      # diverge from forge-cli at the byte level (see schema-sync.yml), so the
+      # gate runs over the range where the promise was actually made.
+      - name: every release keeps its backward-compatibility promise
+        run: |
+          set -euo pipefail
+          for pair in "0.7.1 0.7.2" "0.7.2 0.7.3" "0.7.3 0.7.4" "0.7.4 0.7.5"; do
+            set -- $pair
+            echo "::group::$1 -> $2"
+            python3 scripts/check-compat.py --from "$1" --to "$2"
+            echo "::endgroup::"
+          done

.gitignore

@@ -32,3 +32,11 @@ npm-debug.log*
 # Environment
 .env
 .env.local
+
+# Python — this repo gained a conformance runner and gate scripts; until then
+# it carried no importable Python, so it had no Python ignores.
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+.pytest_cache/

conformance/check_reference.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Run the corpus against the reference implementation, not just the schema.
+
+``conformance/run.py`` checks the corpus against the published JSON Schema
+using a generic validator. That proves the corpus is right about the schema.
+It does not prove the *reference implementation* agrees -- and forge-cli does
+more than schema validation, so the two can drift.
+
+This is the check that matters most, because the FLUID contract already has
+more than one first-party reader:
+
+  * ``fluid_build.schema_manager.FluidSchemaManager`` (forge-cli)
+  * ``flux_engine.seam`` (the FLUX engine, which resolves and compiles a
+    referenced FLUID contract with its own type parser)
+  * the Command Center
+
+Nothing has ever proved those agree about what a FLUID contract means. A
+shared corpus is how they are held to one answer. This script covers the
+first; the others plug in the same way.
+
+    pip install data-product-forge
+    python3 conformance/check_reference.py
+    python3 conformance/check_reference.py --junit reference.xml
+
+Exit 0 when the implementation's verdict matches the corpus on every case,
+1 on any disagreement, 2 if the implementation is not installed.
+
+A disagreement is not automatically an implementation bug. It can equally mean
+the corpus is wrong, or that the implementation deliberately applies semantic
+rules the schema cannot express. The first two must be fixed; the third belongs
+in ``tests/optional/`` where it is asserted rather than tolerated.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from run import (  # noqa: E402
+    CorpusError,
+    discover_versions,
+    load_corpus,
+    write_junit,
+)
+
+
+def load_reference():
+    try:
+        from fluid_build.schema_manager import FluidSchemaManager
+    except ImportError:
+        print(
+            "check_reference: the reference implementation is not installed.\n"
+            "  pip install data-product-forge\n"
+            "This check is skipped rather than failed when forge-cli is absent, "
+            "so the corpus stays runnable on its own.",
+            file=sys.stderr,
+        )
+        return None
+    return FluidSchemaManager()
+
+
+def verdict(manager, contract: Dict[str, Any], version: str) -> Dict[str, Any]:
+    """Normalise the implementation's answer to valid/invalid plus a reason."""
+    try:
+        result = manager.validate_contract(contract, schema_version=version)
+    except Exception as exc:  # an exception is a verdict of 'invalid', loudly
+        return {"valid": False, "reason": f"{type(exc).__name__}: {exc}", "raised": True}
+
+    errors = list(getattr(result, "errors", []) or [])
+    is_valid = getattr(result, "is_valid", None)
+    if is_valid is None:
+        is_valid = not errors
+    return {
+        "valid": bool(is_valid),
+        "reason": "; ".join(str(e) for e in errors[:3]),
+        "raised": False,
+    }
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--version", action="append", dest="versions", metavar="X.Y.Z")
+    ap.add_argument("--junit", type=Path, metavar="PATH")
+    args = ap.parse_args(argv)
+
+    manager = load_reference()
+    if manager is None:
+        return 2
+
+    versions = args.versions or discover_versions()
+    results: List[Dict[str, Any]] = []
+    disagreements: List[str] = []
+    unimplemented: List[str] = []
+
+    for version in versions:
+        try:
+            # optional/ asserts semantics beyond the schema, which is exactly
+            # what an implementation MAY implement -- include it here.
+            cases = load_corpus(version, include_optional=True)
+        except CorpusError as exc:
+            print(f"check_reference: {exc}", file=sys.stderr)
+            return 2
+
+        for case in cases:
+            test_id = f"{version}/{case['id']}"
+            got = verdict(manager, case["contract"], version)
+            agrees = got["valid"] == case["valid"]
+            optional = bool(case.get("optional"))
+            detail = ""
+            if not agrees and optional:
+                # The optional tier asserts behaviour the specification permits
+                # but does not require. An implementation that does not do it is
+                # still conformant, so this is reported, never failed.
+                unimplemented.append(test_id)
+                results.append(
+                    {
+                        "id": test_id,
+                        "group": f"{version}.{case['group']}",
+                        "source": case["source"],
+                        "status": "skip",
+                        "detail": "optional behaviour not implemented",
+                    }
+                )
+                continue
+            if not agrees:
+                expected = "VALID" if case["valid"] else "INVALID"
+                actual = "VALID" if got["valid"] else "INVALID"
+                detail = (
+                    f"corpus says {expected}, {version} reference implementation "
+                    f"says {actual}"
+                )
+                if got["reason"]:
+                    detail += f"\n  implementation said: {got['reason'][:300]}"
+                disagreements.append(test_id)
+            results.append(
+                {
+                    "id": test_id,
+                    "group": f"{version}.{case['group']}",
+                    "source": case["source"],
+                    "status": "pass" if agrees else "fail",
+                    "detail": detail,
+                }
+            )
+
+    for r in results:
+        if r["status"] == "fail":
+            print(f"\nDISAGREEMENT  {r['id']}\n  ({r['source']})\n  {r['detail']}")
+
+    for test_id in unimplemented:
+        print(f"\noptional, not implemented  {test_id}")
+
+    print(
+        f"\nreference implementation vs corpus: {len(results)} cases, "
+        f"{len(results) - len(disagreements) - len(unimplemented)} agree, "
+        f"{len(disagreements)} disagree, "
+        f"{len(unimplemented)} optional behaviour(s) not implemented"
+    )
+
+    if args.junit:
+        write_junit(args.junit, results)
+        print(f"  junit: {args.junit}")
+
+    return 1 if disagreements else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

conformance/known-failures.txt

@@ -0,0 +1,23 @@
+# Conformance cases that are expected to FAIL.
+#
+# One test id per line, '#' for comments, kept sorted. Get ids from:
+#     python3 conformance/run.py --list
+#
+# Two rules make this file a record rather than a mute button:
+#
+#   1. Listed cases are still EXECUTED. A case here that starts passing is an
+#      error, not a quiet success -- the fix and the removal land together, so
+#      the file never drifts from reality.
+#   2. An entry naming a test that no longer exists is also an error. A
+#      renamed case cannot leave a stale exemption behind.
+#
+# Convention from protocolbuffers/protobuf's conformance failure_list
+# (BSD-3-Clause), with the still-execute refinement from connectrpc/conformance
+# (Apache-2.0).
+#
+# What belongs here: a real, understood shortcoming that cannot be fixed in the
+# same change that discovers it. What does NOT belong here: a case that is
+# merely inconvenient, or one whose expectation nobody has checked. If a case
+# is wrong, fix the case.
+#
+# Currently empty: every published case passes against every published schema.