build.yml

# Build and verify the ontology corpus — NO DEPLOY, BY DESIGN.
#
# This repository publishes a corpus and the pipeline that compiles it. This
# workflow builds and gates that corpus; it deploys nothing. The deploy path is
# deliberately absent: no GitHub Pages step, no publishing action, and no
# reference to any secret anywhere in this file. `permissions: contents: read`
# is the whole permission set — declared at workflow and job level both — so
# this workflow cannot write to the repository even if a step tried.
#
# The repository root does carry a CNAME (narrativegoldmine.com) for the
# published site, but no step below reads, writes or acts on it; the Pages push
# lives in the private publishing CI (docs/ci-cd/build-and-gates.md §1).
#
# That private CI runs this same seven-stage pipeline and then the steps that
# cannot be reproduced from a clone: SPA/WASM build, browser smoke test and the
# gh-pages deploy. Its nine gates are listed in docs/ci-cd/build-and-gates.md §2;
# three of them re-appear here (gate 2 pipeline unit tests, gate 4 build
# validation, gate 5 standalone validation), and the other six cover the explorer
# and the built site. The secret scan and the class-count contract below are
# specific to this extracted corpus and have no counterpart in that list. What
# you see here is the half that can be reproduced by anyone with a clone, no
# credentials required.
#
# The corpus itself is mostly AI-generated synthetic content produced under human
# direction, by design. It is an ontology testbed, not an authoritative
# encyclopaedia; its provenance (did:nostr, generatedAtTime, URNs) attests
# traceable generation under human direction, not human authorship. The pipeline
# emits that framing rather than leaving it to prose — see `corpus.nature:
# "synthetic"` in dist-ci/data/graph/stats.json.
#
# What this workflow proves, on every push and pull request:
#   1. the corpus contains no credential-shaped strings (load-bearing: the
#      public corpus feeds the export — projected Markdown shares its privacy filter)
#   2. the NGG1 binary writer still matches its 183-byte golden fixture
#      (pipeline/tests/test_emit_graph_tiers.py, pipeline/tests/fixtures/ngg1-3n2e.bin)
#   3. the 7-stage pipeline runs end to end from ontology/pages/
#   4. the build produced exactly 8138 OWL classes — a hard contract gate, not
#      a printed statistic
#   5. pipeline.validate reports zero errors over the corpus
#   6. every input file is accounted for — parsed, rejected with a reason, or
#      excluded — and the strict build refuses to publish a corpus containing a
#      malformed page (pipeline/census.py)
#   7. the built class set matches the committed sorted class-IRI set, so an
#      equal-count identity substitution fails even though the count agrees;
#      the same gate checks the consumer schema shape and scans every public
#      artefact for private identifiers (pipeline/release_gate.py)
#   8. the generation manifest re-verifies: every artefact's SHA-256 matches
#      what the build recorded (pipeline/manifest.py)
#
# Reproduce locally:
#   pip install "rdflib>=7.0.0" pytest
#   python -m pytest pipeline/tests -q
#   python -m pipeline.build ontology/pages dist-ci --strict
#   python -m pipeline.release_gate dist-ci --pages ontology/pages \
#       --expected-classes 8138
#   python -m pipeline.manifest dist-ci

name: Build and verify

on:
  push:
  pull_request:
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: build-${{ github.ref }}
  cancel-in-progress: true

env:
  # The corpus contract. Both dist-ci/data/graph/stats.json (`classes`) and
  # dist-ci/data/ontology.json (`class[]` length) must report this many OWL
  # classes or the build fails. Change it only alongside the corpus.
  #
  # This stays a hand-typed, independently-authored number (ADR-2003): an
  # auto-derived expectation moves with any parse regression and can therefore
  # never detect one. GATE 5 adds *membership* on top of this *count*, because a
  # count cannot see an equal-count identity substitution — delete one class,
  # add another, and this gate still passes. Both must move in the same commit
  # as any corpus size change; the gate fails if they disagree.
  EXPECTED_CLASSES: '8138'

jobs:
  build:
    name: Pipeline build and gates
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - name: Checkout
        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262

      # ------------------------------------------------------------------
      # GATE 1 — secret scan over the published corpus (cheapest first)
      # ------------------------------------------------------------------
      # ontology/pages/*.md feeds public exports. A leaked key in a page is a leaked
      # key on the internet, so this gate blocks the build rather than warning.
      #
      # Every pattern is anchored to a token-shaped suffix on purpose. A bare
      # `sk-` substring match hits 1107 of the 7874 corpus files (risk-, task-,
      # disk-, desk-assistant …), which would make the gate useless noise; the
      # anchored form `sk-[A-Za-z0-9]{20,}` matches 0. Likewise the bare prefix
      # `[Bb]earer ` used below appears in 64 pages as HTTP prose, while
      # `[Bb]earer <20+ token chars>` matches 0. Verified against the corpus at
      # 7874 pages: 0 hits total.
      - name: Secret scan (corpus must contain no credential-shaped strings)
        run: |
          set -uo pipefail
          patterns='(sk-[A-Za-z0-9]{20,}|pplx-[A-Za-z0-9]{20,}|ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{20,}|gho_[A-Za-z0-9]{36}|AKIA[0-9A-Z]{16}|-----BEGIN[A-Z ]*PRIVATE KEY-----|[Bb]earer [A-Za-z0-9._~+/-]{20,})'
          # Actions runs `run:` steps under `bash -e`, and `set -uo pipefail`
          # does not clear it. A plain `hits=$(grep …)` therefore aborts the step
          # the instant grep exits 1 — i.e. on a *clean* corpus, the pass case.
          # `|| rc=$?` puts the assignment in a tested context and keeps the code.
          rc=0
          hits=$(grep -rInE "$patterns" ontology/) || rc=$?
          if [ "$rc" -eq 0 ]; then
            echo "Secret scan FAILED — credential-shaped string(s) in the published corpus:"
            echo "$hits" | cut -c1-200
            exit 1
          fi
          # grep exits 1 for "no match" and 2+ for a real error. Only 1 is a pass;
          # a broken scan must never be mistaken for a clean corpus.
          if [ "$rc" -ne 1 ]; then
            echo "Secret scan ERRORED: grep exited $rc"
            exit 1
          fi
          echo "Secret scan passed: no credential-shaped strings under ontology/"

      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
        with:
          python-version: '3.12'

      - name: Install Python dependencies
        run: pip install "rdflib>=7.0.0" pytest

      # ------------------------------------------------------------------
      # GATE 2 — pipeline unit tests
      # ------------------------------------------------------------------
      # Includes the byte-exact 183-byte NGG1 golden (explorer/FORMAT-NGG1.md §7), parsed
      # by a struct reader written independently of the writer, plus the
      # overview.json consumer contract (40 nodes: 6 domains then 34
      # categories, order frozen) and layout determinism.
      - name: Pipeline unit tests (pytest)
        run: python -m pytest pipeline/tests -q

      # ------------------------------------------------------------------
      # BUILD — the 7 stages: parse, validate, Turtle, WebVOWL JSON,
      # page API, search index, NGG1 graph tiers
      # ------------------------------------------------------------------
      # Both strict and non-strict canonical builds now refuse malformed or
      # ambiguously public input and validation errors before promotion.
      # Strict remains explicit for release-contract and explorer checks.
      # Local authoring diagnostics use census/validate directly; public output
      # receives aggregate diagnostics and projected Markdown only.
      # Successful staging promotion replaces obsolete generated pages.
      - name: Run the JSON-LD pipeline (strict)
        run: |
          set -euo pipefail
          python -m pipeline.build ontology/pages dist-ci --strict
          echo "--- outputs ---"
          ls -la dist-ci/data/
          ls -la dist-ci/data/graph/
          echo "--- input census ---"
          python - <<'PY'
          import json
          from pathlib import Path
          c = json.loads(Path("dist-ci/api/census.json").read_text())
          print(f"  input files : {c['input_files']}")
          print(f"  parsed      : {c['parsed']} ({c['public_pages']} public)")
          print(f"  rejected    : {c['rejected']}")
          print(f"  excluded    : {c['excluded']}")
          print(f"  balanced    : {c['balanced']}")
          PY

      # ------------------------------------------------------------------
      # GATE 3 — corpus contract: the build must produce 8138 OWL classes
      # ------------------------------------------------------------------
      # Read from the artefact the pipeline actually wrote, not from stdout.
      # A silent parse regression (a dropped JSON-LD fence, a changed public
      # filter) shows up here as a number, not as a warning nobody reads.
      - name: Assert corpus contract (8138 classes)
        run: |
          set -euo pipefail
          python - <<'PY'
          import json, os, sys
          from pathlib import Path

          expected = int(os.environ["EXPECTED_CLASSES"])
          stats = json.loads(Path("dist-ci/data/graph/stats.json").read_text())
          vowl = json.loads(Path("dist-ci/data/ontology.json").read_text())

          checks = {
              "stats.json classes": stats["classes"],
              "ontology.json class[] length": len(vowl["class"]),
          }
          failed = [f"{k}: expected {expected}, got {v}" for k, v in checks.items() if v != expected]
          for k, v in checks.items():
              print(f"  {k}: {v}")
          print(f"  stats.json nodes: {stats['nodes']}, pages: {stats['pages']}, "
                f"domains: {stats['domains']}, categories: {stats['categories']}")
          print(f"  edges declared: {stats['edges']['declared']}, "
                f"resolvable: {stats['edges']['resolvable']}")
          if failed:
              print("Corpus contract FAILED:")
              for line in failed:
                  print(f"  {line}")
              sys.exit(1)
          print(f"Corpus contract OK: {expected} OWL classes")
          PY

      # ------------------------------------------------------------------
      # GATE 4 — validation must report zero errors
      # ------------------------------------------------------------------
      # pipeline.validate exits 1 on any error. Warnings do not fail the build,
      # and the corpus currently carries none: the report reads 0 errors, 0
      # warnings, 1401 info, all of the info entries MULTI_PARENT. That code was
      # a warning until it was reclassified — multiple inheritance is deliberate
      # here, so 957 of the old 961 warnings were the design, not a defect list
      # (docs/ci-cd/build-and-gates.md §3). The bridging it counts is published
      # in stats.json (`bridging`), bridges.json and overview.json.
      - name: Validate corpus (0 errors required)
        run: python -m pipeline.validate ontology/pages

      # ------------------------------------------------------------------
      # GATE 5 — identity, schema and publication-visibility contracts
      # ------------------------------------------------------------------
      # The count above is one number, and three failures leave it unchanged:
      # an equal-count identity substitution, an altered publication visibility,
      # and a shape drift that still parses as JSON but no longer parses in the
      # explorer. This gate compares the built class set against the committed
      # sorted class-IRI set (pipeline/contracts/class-identity.txt), asserts
      # the consumer-shaped class/property arrays are present, and re-derives
      # the private identifiers from source to prove none of them occurs in a
      # published artefact. Private identifiers are never written into dist-ci.
      - name: Assert release contracts (identity set, schema, visibility)
        run: |
          set -euo pipefail
          python -m pipeline.release_gate dist-ci \
            --pages ontology/pages \
            --expected-classes "$EXPECTED_CLASSES"

      # ------------------------------------------------------------------
      # GATE 6 — the generation manifest re-verifies
      # ------------------------------------------------------------------
      # Every artefact's SHA-256, the source revision, the counts and a
      # generation id are recorded with the export. Re-hashing the tree here
      # proves the manifest describes the artefacts actually uploaded.
      - name: Verify generation manifest
        run: python -m pipeline.manifest dist-ci

      # ------------------------------------------------------------------
      # ARTEFACT — build output, downloadable from the run page. Nothing is
      # published anywhere; the artefact expires with the run's retention.
      # ------------------------------------------------------------------
      # `if: always()` means this also runs when gate 1 or 2 failed, in which
      # case dist-ci was never created. `if-no-files-found: warn` keeps that case
      # to one honest failure — the gate that actually broke — instead of adding
      # a second, misleading upload failure on top of it.
      # Archive first: title mirrors intentionally preserve ':' and case-sensitive
      # names, which upload-artifact cannot transport as loose files.
      # Download dist-ci then run: tar -xzf dist-ci.tar.gz
      - name: Package build output without filename normalisation
        if: always()
        run: |
          set -euo pipefail
          if [ -d dist-ci ]; then
            tar -czf dist-ci.tar.gz dist-ci
          fi

      - name: Upload build output
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
        with:
          name: dist-ci
          path: dist-ci.tar.gz
          retention-days: 14
          if-no-files-found: warn