diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f1e88c4..06e56e6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,8 +361,20 @@ jobs: test/kyaml_test.exs \ test/kyaml_conformance_test.exs + - name: Checkout the governance-pinned baseline helper + uses: actions/checkout@v7.0.1 + with: + repository: metadatastician/standards + ref: a797d308c0517d4e2d21cc7f1cb08d41f82b1a12 + path: .standards + sparse-checkout: scripts/apply-baseline.sh + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Review-gap prevention regressions (1009) - run: mix test --no-start test/ci_changed_test.exs test/kyaml_spec_mutation_test.exs test/structured_format_test.exs + env: + HYPATIA_BASELINE_HELPER: .standards/scripts/apply-baseline.sh + run: mix test --no-start --include baseline_helper test/ci_changed_test.exs test/ci_changed_receipt_test.exs test/kyaml_spec_mutation_test.exs test/structured_format_test.exs # ADR-001 retirement guard: Hypatia rules are pure Elixir # (lib/rules/*.ex). This fails if an active .lgt/.logtalk source diff --git a/.gitignore b/.gitignore index c2a331e5..27f49774 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ dist/ /.kyq # yaml-test-suite checkout read by test/kyaml_yaml_test_suite_test.exs /.yts +# Local checkout of the governance-pinned baseline implementation. +/.standards/ diff --git a/.hypatia-baseline.json b/.hypatia-baseline.json index 9d8fc9d1..62b02f74 100644 --- a/.hypatia-baseline.json +++ b/.hypatia-baseline.json @@ -1,5 +1,55 @@ [ +{ + "severity": "medium", + "rule_module": "structured_format", + "type": "SF001", + "file": "lib/rules/cicd_rules.ex", + "severity_override": "advisory", + "expires_at": "2027-02-04", + "tracking_issue": "hyperpolymath/hypatia#971", + "note": "Standing SF001 heuristic inventory at 0083453e. Preserve visible advisory status during canonical workflow-reader migration and candidate assessment under #971; no wildcard or new-file exemption." + }, + { + "severity": "medium", + "rule_module": "structured_format", + "type": "SF001", + "file": "lib/rules/workflow_hardening.ex", + "severity_override": "advisory", + "expires_at": "2027-02-04", + "tracking_issue": "hyperpolymath/hypatia#971", + "note": "Standing SF001 heuristic inventory at 0083453e. Preserve visible advisory status during canonical workflow-reader migration and candidate assessment under #971; no wildcard or new-file exemption." + }, + { + "severity": "medium", + "rule_module": "structured_format", + "type": "SF001", + "file": "lib/rules/supply_chain.ex", + "severity_override": "advisory", + "expires_at": "2027-02-04", + "tracking_issue": "hyperpolymath/hypatia#971", + "note": "Standing SF001 heuristic inventory at 0083453e. Preserve visible advisory status during canonical workflow-reader migration and candidate assessment under #971; no wildcard or new-file exemption." + }, + { + "severity": "medium", + "rule_module": "structured_format", + "type": "SF001", + "file": "lib/rules/workflow_audit.ex", + "severity_override": "advisory", + "expires_at": "2027-02-04", + "tracking_issue": "hyperpolymath/hypatia#971", + "note": "Standing SF001 heuristic inventory at 0083453e. Preserve visible advisory status during canonical workflow-reader migration and candidate assessment under #971; no wildcard or new-file exemption." + }, { + "severity": "medium", + "rule_module": "structured_format", + "type": "SF001", + "file": "lib/rules/rules.ex", + "severity_override": "advisory", + "expires_at": "2027-02-04", + "tracking_issue": "hyperpolymath/hypatia#971", + "note": "Standing SF001 heuristic inventory at 0083453e. Preserve visible advisory status during canonical workflow-reader migration and candidate assessment under #971; no wildcard or new-file exemption." + }, +{ "severity": "low", "rule_module": "code_safety", "type": "expect_in_hot_path", diff --git a/docs/REVIEW-GAP-LEDGER-1009.adoc b/docs/REVIEW-GAP-LEDGER-1009.adoc index 0cdef2fc..c899ed01 100644 --- a/docs/REVIEW-GAP-LEDGER-1009.adoc +++ b/docs/REVIEW-GAP-LEDGER-1009.adoc @@ -47,7 +47,7 @@ The local branch described in the issue was not available in this checkout; its reported test counts are historical evidence, not a reproduced receipt. |7 |Copilot emitted a COMMENTED review containing insufficient quota, #39 |Review absent |Display head-bound reviewer coverage separately from verdicts -|Companion metadatastician/standards#1417 adds an advisory coverage snapshot after the +|Companion metadatastician/standards#1417, corrected by #1418, adds an advisory coverage snapshot after the existing grids. Quota COMMENTED reviews are NO REVIEW, stale/pending/dismissed reviews do not cover a new head, and unavailable evidence is indeterminate. No fourth lane or automatic approval is introduced. Consumers need to adopt @@ -89,22 +89,75 @@ Orchestrate 42201913539 failed. This is not a claim all main workflows pass. == Local use and limits +The newly exposed standing SF001 inventory contains 15 advisory candidates +in five existing rule files. The baseline retains them as visible advisory +debt, with five exact-path entries tied to #971 and expiry 2027-02-04. +The commit and pull-request body declare this specific ledger growth using +the exemption ratchet's required `Ratchet-exception` record. +It does not use a wildcard or exempt new files. The canonical baseline gate +grades every finding at info or higher regardless of the scanner's advisory +flag, so this explicit, temporary classification is necessary to preserve +SF001's declared advisory meaning. It is not a claim that each regex is a +confirmed vulnerability or that the reader migration is complete. + Run `just ci-changed BASE` (default `origin/main`) before pushing; use `just ci-changed-plan BASE` to inspect selection. Both committed differences and uncommitted/untracked files are included. A shallow checkout must contain the base commit; a missing base is an error, never an empty successful diff. -New/changed job definitions are selected by parsed semantics, including alias -changes. Moving comments/line numbers does not select an otherwise unchanged -job. Global environment/event changes conservatively select all remaining -jobs. Deleted jobs have nothing left to run. +For workflow-only changes, job definitions are selected by parsed semantics, +including aliases; moving comments/line numbers does not select an otherwise +unchanged job. Any non-workflow input change conservatively selects all current +workflows, since a script or fixture can affect an unchanged job. Global +environment/event changes also select all remaining jobs. The current needs +graph is validated even when deletion would otherwise select no surviving job; +missing dependencies and cycles are errors. + +The scan retains directory-scoped absence findings when a changed file is +below that directory. It applies the canonical governance baseline helper +before grading, including expiry checks, acknowledgements and overrides. Set +`HYPATIA_BASELINE_HELPER` to `scripts/apply-baseline.sh` from the same standards +revision as `governance.yml` (`a797d308c0517d4e2d21cc7f1cb08d41f82b1a12` here), +or provision it under `.standards/scripts/`. A missing helper with an active +baseline is an error. CI fetches that exact helper and explicitly includes its +integration test; other test runs exclude the provisioned-helper tag. -The scan uses existing offline code-safety and workflow rules, with existing -suppressions, and grades changed files at medium or higher; SF001 is advisory. -`act` needs a supported container engine, runner images and any legitimately -required event/context/secrets. Never provide production credentials merely -to simulate a PR. Hosted GitHub CI remains authoritative for platform-specific -actions and conditional jobs. Inspect act logs for skipped conditions; a -selection plan and mocked executor test are not hosted parity receipts. +Existing suppressions remain active; changed files are graded at medium or +higher and SF001 remains advisory. Each job gets a supported workflow event; +`mix hypatia.ci_changed --event EVENT` can select another declared event. +Execution requests act JSON/debug logs and requires non-dry-run `jobResult` +success for the selected job. Skips, failures and absent receipts cannot count +as completion. Matrix members observed as skipped or failed invalidate the +receipt. New/changed checkout configuration, reusable jobs and write-capable +jobs require hosted validation and cause the local task to refuse a receipt. +The same blockers apply across the transitive `needs` graph before act can +run any prerequisite. Composite-action findings retain their actual root +or nested action path; workflow paths are qualified by their producer. +For other jobs, act's unchanged checkout simulation is a declared local limit; +the task does not bind a writable host workspace into the container. +Changed prerequisites select downstream consumers, and changed local reusable +contracts select their callers. Removed or repositioned checkouts also require +hosted validation. Missing or unresolved permissions, conditional steps, tolerated +failures, custom containers and secret-dependent execution cannot earn a local +receipt. The runner uses empty input/environment/secret files, an explicitly empty +GitHub token, an isolated configuration home and a cleared parent environment; +project `.actrc` files require hosted validation. Docker daemon/context and client +configuration remain available to act for container transport, without mounting +the daemon socket into job containers or loading workflow secrets from them. +The trusted act process can read registry credentials from `DOCKER_CONFIG` +and endpoint credentials from `DOCKER_CERT_PATH`; preserving the chosen daemon +connection is a separate trust boundary from credentials inside job containers. +Selected jobs without checkout, jobs using GitHub expression contexts (including +repository variables and event/input data), and matrix jobs also require hosted +validation. A nonempty subset of successful matrix rows cannot certify the +unobserved members. Static, explicitly read-only jobs with an unchanged checkout +remain eligible for the bounded local receipt. +Eligibility also requires `contents: read`, a supported Ubuntu container runner, +and no opaque/composite actions or direct `GITHUB_EVENT_PATH` consumption. +These restrictions deliberately +limit local parity rather than treating simulated success as hosted evidence. +Hosted CI remains authoritative for platform-specific actions, context, +unobserved matrix expansion and permissions. A plan or mocked executor test +is not a hosted execution receipt. The recipe is not silently installed as a Git hook in this repository or the estate. Fleet rollout, canonical workflow-reader migration (#971) and upstream diff --git a/lib/hypatia/ci_changed.ex b/lib/hypatia/ci_changed.ex index 8a1fa3c8..782db332 100644 --- a/lib/hypatia/ci_changed.ex +++ b/lib/hypatia/ci_changed.ex @@ -1,25 +1,37 @@ # SPDX-License-Identifier: MPL-2.0 defmodule Hypatia.CiChanged do @moduledoc """ - Local execution of new/changed workflow jobs, using the canonical KYAML - reader. Global workflow changes conservatively select every remaining job. - No regex or line-based reconstruction of workflow structure is used. + Parser-based local CI selection and observed execution receipts. Source/input + changes conservatively select every workflow. Skipped jobs, missing receipts, + changed checkout configuration and write-capable jobs cannot certify execution. """ alias Hypatia.Kyaml + @doc "Select changed job semantics and reject dangling or cyclic needs." def changed_jobs(before, after_text) do old = workflow!(before) new = workflow!(after_text) - old_jobs = Map.get(old, "jobs", %{}) - new_jobs = Map.get(new, "jobs", %{}) - global_change = Map.delete(old, "jobs") !== Map.delete(new, "jobs") + validate_graph!(Map.fetch!(new, "jobs")) + globals_changed = Map.delete(old, "jobs") !== Map.delete(new, "jobs") - for {id, job} <- new_jobs, - global_change or Map.get(old_jobs, id) !== job, - do: id + selected = + for {id, job} <- new["jobs"], globals_changed or get_in(old, ["jobs", id]) !== job, do: id + + downstream(new["jobs"], MapSet.new(selected)) |> MapSet.to_list() |> Enum.sort() + end + + defp downstream(jobs, selected) do + expanded = + Enum.reduce(jobs, selected, fn {id, job}, affected -> + if Enum.any?(List.wrap(Map.get(job, "needs", [])), &MapSet.member?(selected, &1)), + do: MapSet.put(affected, id), + else: affected + end) + + if expanded == selected, do: selected, else: downstream(jobs, expanded) end - defp workflow!(nil), do: %{} + defp workflow!(nil), do: %{"jobs" => %{}} defp workflow!(text) do case Kyaml.parse(text) do @@ -52,23 +64,240 @@ defmodule Hypatia.CiChanged do end end - def plan(repo, base) do + defp validate_graph!(jobs) do + graph = + Map.new(jobs, fn {id, job} -> + unless is_map(job), do: raise(ArgumentError, "job #{id} must be a mapping") + steps = Map.get(job, "steps", []) + + unless is_list(steps) and Enum.all?(steps, &is_map/1), + do: raise(ArgumentError, "job #{id} steps must be a sequence of mappings") + + needs = List.wrap(Map.get(job, "needs", [])) + + unless Enum.all?(needs, &(is_binary(&1) and Map.has_key?(jobs, &1))), + do: raise(ArgumentError, "job #{id} needs a missing or invalid job") + + {id, MapSet.new(needs)} + end) + + acyclic!(graph) + end + + defp acyclic!(graph) when map_size(graph) == 0, do: :ok + + defp acyclic!(graph) do + ready = for {id, deps} <- graph, MapSet.size(deps) == 0, do: id + if ready == [], do: raise(ArgumentError, "workflow contains cyclic needs") + + graph + |> Map.drop(ready) + |> Map.new(fn {id, deps} -> {id, MapSet.difference(deps, MapSet.new(ready))} end) + |> acyclic!() + end + + @doc "Include committed, working and untracked inputs, with a supported event per workflow." + def plan(repo, base, requested_event \\ nil) do git!(repo, ["rev-parse", "--verify", "#{base}^{commit}"]) changed = git!(repo, ["diff", "--name-only", "-z", base, "--"]) untracked = git!(repo, ["ls-files", "--others", "--exclude-standard", "-z"]) files = String.split(changed <> untracked, <<0>>, trim: true) |> Enum.uniq() |> Enum.sort() + inputs_changed = Enum.any?(files, &(not workflow_file?(&1))) + + workflows = + Path.wildcard(Path.join(repo, ".github/workflows/*")) + |> Enum.map(&Path.relative_to(&1, repo)) + |> Enum.filter(&workflow_file?/1) + |> Map.new(fn file -> {file, File.read!(Path.join(repo, file))} end) + + callers = affected_callers(workflows, MapSet.new(Enum.filter(files, &workflow_file?/1))) jobs = - for file <- files, - Path.dirname(file) == ".github/workflows", - Path.extname(file) in [".yml", ".yaml"], - File.regular?(Path.join(repo, file)), - id <- changed_jobs(previous(repo, base, file), File.read!(Path.join(repo, file))), - do: %{workflow: file, job: id} + for {file, current} <- workflows, + old <- [previous(repo, base, file)], + selected = changed_jobs(old, current), + new = workflow!(current), + id <- + if(inputs_changed or MapSet.member?(callers, file), + do: Map.keys(new["jobs"]), + else: selected + ), + do: job_plan(file, id, workflow!(old), new, requested_event) %{base: base, files: files, jobs: Enum.sort_by(jobs, &{&1.workflow, &1.job})} end + defp affected_callers(workflows, changed, callers \\ MapSet.new()) do + affected_workflows = MapSet.union(changed, callers) + + expanded = + Enum.reduce(workflows, callers, fn {file, text}, affected -> + calls_changed = + workflow!(text)["jobs"] + |> Enum.any?(fn {_id, job} -> + case job["uses"] do + "./.github/workflows/" <> _ = reference -> + MapSet.member?(affected_workflows, String.trim_leading(reference, "./")) + + _ -> + false + end + end) + + if calls_changed, do: MapSet.put(affected, file), else: affected + end) + + if expanded == callers, do: callers, else: affected_callers(workflows, changed, expanded) + end + + defp workflow_file?(file), + do: Path.dirname(file) == ".github/workflows" and Path.extname(file) in [".yml", ".yaml"] + + defp event!(workflow, requested) do + events = + case Map.get(workflow, "on") do + text when is_binary(text) -> [text] + events when is_list(events) -> events + events when is_map(events) -> Map.keys(events) + _ -> [] + end + + if requested && requested not in events, + do: raise(ArgumentError, "requested event is not supported by workflow") + + event = + requested || + Enum.find( + ["pull_request", "push", "workflow_dispatch", "schedule", "workflow_call"], + &(&1 in events) + ) || List.first(Enum.sort(events)) + + event || raise ArgumentError, "workflow has no supported event" + end + + defp checkout_steps(job) do + case Map.get(job || %{}, "steps", []) do + steps when is_list(steps) -> + Enum.filter(Enum.with_index(steps), fn {step, _index} -> + is_map(step) and is_binary(step["uses"]) and + String.starts_with?(String.downcase(step["uses"]), "actions/checkout@") + end) + + _ -> + [:invalid_base_steps] + end + end + + defp hosted_context?(value) when is_binary(value), + do: Regex.match?(~r/\$\{\{|\bGITHUB_EVENT_PATH\b/, value) + + defp hosted_context?(value) when is_map(value), + do: Enum.any?(value, fn {key, val} -> hosted_context?(key) or hosted_context?(val) end) + + defp hosted_context?(value) when is_list(value), do: Enum.any?(value, &hosted_context?/1) + defp hosted_context?(_), do: false + + defp job_plan(file, id, old, new, event) do + blocker = + [id | prerequisites(id, new["jobs"])] + |> Enum.find_value(fn candidate -> + case direct_blocker(candidate, old, new) do + nil -> nil + reason when candidate == id -> reason + reason -> "prerequisite #{candidate}: #{reason}" + end + end) + + %{workflow: file, job: id, event: event!(new, event), local_blocker: blocker} + end + + defp prerequisites(id, jobs), + do: walk_needs(List.wrap(Map.get(jobs[id], "needs", [])), jobs, MapSet.new()) + + defp walk_needs([], _jobs, seen), do: seen |> MapSet.to_list() |> Enum.sort() + + defp walk_needs([id | rest], jobs, seen) do + if MapSet.member?(seen, id), + do: walk_needs(rest, jobs, seen), + else: + walk_needs( + List.wrap(Map.get(jobs[id], "needs", [])) ++ rest, + jobs, + MapSet.put(seen, id) + ) + end + + defp direct_blocker(id, old, new) do + job = new["jobs"][id] + old_job = get_in(old, ["jobs", id]) + + checkout_context_changed = + Map.delete(old, "jobs") !== Map.delete(new, "jobs") or + Map.drop(old_job || %{}, ["steps"]) !== Map.drop(job, ["steps"]) + + checkout_changed = + checkout_steps(old_job) !== checkout_steps(job) or + (checkout_steps(job) != [] and checkout_context_changed) + + permissions = Map.get(job, "permissions", Map.get(new, "permissions")) + + writes = + permissions == "write-all" or (is_map(permissions) and "write" in Map.values(permissions)) + + cond do + job["uses"] -> + "reusable workflow needs a hosted receipt" + + is_nil(permissions) -> + "inherited permissions need a hosted receipt" + + writes -> + "write-capable job needs a hosted receipt" + + not (permissions == "read-all" or + (is_map(permissions) and + Enum.all?(Map.values(permissions), &(&1 in ["read", "none"])))) -> + "unresolved permissions need a hosted receipt" + + is_map(permissions) and permissions["contents"] != "read" -> + "checkout needs contents: read or a hosted receipt" + + Enum.any?(Map.get(job, "steps", []), fn step -> + is_binary(step["uses"]) and + not String.starts_with?(String.downcase(step["uses"]), "actions/checkout@") + end) -> + "opaque or composite actions need a hosted receipt" + + checkout_changed -> + "changed checkout inputs need a hosted receipt; act skips checkout" + + Map.has_key?(job, "if") || Enum.any?(Map.get(job, "steps", []), &Map.has_key?(&1, "if")) -> + "conditional execution needs a hosted receipt" + + job["continue-on-error"] || + Enum.any?(Map.get(job, "steps", []), &Map.get(&1, "continue-on-error", false)) -> + "tolerated failures need a hosted receipt" + + job["strategy"] -> + "matrix execution needs a hosted receipt for every member" + + hosted_context?([Map.delete(new, "jobs"), job]) -> + "credential, repository-variable or hosted-expression-dependent execution needs a hosted receipt" + + job["container"] || job["services"] -> + "custom containers need a hosted receipt" + + checkout_steps(job) == [] -> + "checkout-less jobs need a hosted receipt; act copies local source" + + job["runs-on"] not in ["ubuntu-latest", "ubuntu-24.04", "ubuntu-22.04", "ubuntu-20.04"] -> + "unsupported or host-native runners need a hosted receipt" + + true -> + nil + end + end + defp previous(repo, base, path) do case System.cmd("git", ["show", "#{base}:#{path}"], cd: repo, stderr_to_stdout: true) do {text, 0} -> text @@ -83,9 +312,9 @@ defmodule Hypatia.CiChanged do end end - @doc "Offline rules, scoped to the changed files, including existing suppressions." + @doc "Offline rules on changed files, including ancestor-directory findings." def scan(repo, files) do - changed = MapSet.new(files, &Path.expand(&1, repo)) + changed = Enum.map(files, &Path.expand(&1, repo)) Hypatia.CLI.collect_findings(repo, [ :code_safety, @@ -93,36 +322,199 @@ defmodule Hypatia.CiChanged do :workflow_hardening, :research_extensions ]) - |> Enum.filter(fn finding -> + |> Enum.map(fn finding -> file = Map.get(finding, :file, ".") - MapSet.member?(changed, Path.expand(file, repo)) or - MapSet.member?(changed, Path.expand(Path.join(".github/workflows", file), repo)) + Map.put(finding, :file, file |> Path.expand(repo) |> Path.relative_to(repo)) + end) + |> Enum.filter(fn finding -> + path = Path.expand(finding.file, repo) + Enum.any?(changed, &(&1 == path or String.starts_with?(&1, path <> "/"))) end) end - def execute_jobs(repo, jobs, runner \\ &System.cmd/3) do - Enum.map(jobs, fn %{workflow: file, job: id} -> - # An argv list, never shell interpolation. act runs the actual workflow, - # including setup actions and `needs`, rather than copied run commands. - {output, status} = - runner.( - "act", - [ - "pull_request", - "-W", - file, - "-j", - id, - "--bind", - "--container-options", - "--cap-drop=ALL" - ], - cd: repo, - stderr_to_stdout: true + @doc "Apply the canonical governance helper before grading, preserving its schema and expiry checks." + def apply_baseline(repo, findings, helper \\ nil) do + baseline = Path.join(repo, ".hypatia-baseline.json") + + if File.regular?(baseline) do + helper = + helper || System.get_env("HYPATIA_BASELINE_HELPER") || + Path.join(repo, ".standards/scripts/apply-baseline.sh") + + unless File.regular?(helper), + do: + raise( + ArgumentError, + "set HYPATIA_BASELINE_HELPER to the governance-pinned standards/scripts/apply-baseline.sh" + ) + + directory = + Path.join( + System.tmp_dir!(), + "hypatia-local-baseline-#{System.pid()}-#{System.os_time(:nanosecond)}" ) - %{workflow: file, job: id, exit: status, output: output} - end) + File.mkdir_p!(directory) + + try do + input = Path.join(directory, "findings.json") + File.write!(input, Jason.encode!(findings)) + + case System.cmd("bash", [Path.expand(helper), input, baseline, "advisory"], + env: [{"GITHUB_STEP_SUMMARY", ""}] + ) do + {out, 0} -> + decoded = + case Jason.decode!(out) do + %{"findings_kept" => findings} when is_list(findings) -> findings + _ -> raise ArgumentError, "baseline helper did not return findings" + end + + names = + Map.new( + [:severity, :advisory, :rule_module, :type, :file, :line, :reason, :action], + &{Atom.to_string(&1), &1} + ) + + Enum.map(decoded, fn finding -> + Map.new(finding, fn {key, val} -> {Map.get(names, key, key), val} end) + end) + + {_out, status} -> + raise ArgumentError, "canonical baseline helper failed (#{status})" + end + after + File.rm_rf!(directory) + end + else + findings + end + end + + @doc "Run eligible jobs with JSON receipts; skipped, dry-run and absent results are not completion." + def execute_jobs(repo, jobs, runner \\ &System.cmd/3) do + scratch = + Path.join(System.tmp_dir!(), "hypatia-act-#{System.pid()}-#{System.os_time(:nanosecond)}") + + File.mkdir_p!(scratch) + empty = Path.join(scratch, "empty") + File.write!(empty, "", [:exclusive]) + + env = + System.get_env() + |> Map.new(fn {key, _} -> {key, nil} end) + |> Map.merge( + Map.take(System.get_env(), [ + "PATH", + "Path", + "SystemRoot", + "SYSTEMROOT", + "PATHEXT", + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH" + ]) + ) + |> Map.put( + "DOCKER_CONFIG", + System.get_env("DOCKER_CONFIG") || Path.join(System.user_home() || scratch, ".docker") + ) + |> Map.merge(%{ + "HOME" => scratch, + "USERPROFILE" => scratch, + "XDG_CONFIG_HOME" => scratch, + "TMPDIR" => scratch, + "TEMP" => scratch, + "TMP" => scratch + }) + |> Map.to_list() + + try do + Enum.map(jobs, fn %{workflow: file, job: id, event: event, local_blocker: blocker} -> + blocker = + blocker || + if(File.exists?(Path.join(repo, ".actrc")), + do: "project act configuration needs a hosted receipt" + ) + + if blocker do + %{ + workflow: file, + job: id, + exit: nil, + outcome: :requires_hosted_receipt, + output: blocker + } + else + {output, status} = + runner.( + "act", + [ + event, + "-W", + file, + "-j", + id, + "--json", + "--verbose", + "--secret-file", + empty, + "--env-file", + empty, + "--var-file", + empty, + "--input-file", + empty, + "--secret", + "GITHUB_TOKEN=", + "--container-daemon-socket", + "-", + "--container-options", + "--cap-drop=ALL" + ], + cd: repo, + env: env, + stderr_to_stdout: true + ) + + %{ + workflow: file, + job: id, + exit: status, + outcome: receipt(output, id, status), + output: output + } + end + end) + after + File.rm_rf!(scratch) + end + end + + @doc "Read act jobResult fields for the exact job and every observed matrix member." + def receipt(output, id, exit) do + results = + output + |> String.split("\n", trim: true) + |> Enum.flat_map(fn line -> + case Jason.decode(line) do + {:ok, %{"jobID" => ^id, "jobResult" => result, "dryrun" => false} = row} + when result in ["success", "failure", "skipped"] -> + [{Map.get(row, "matrix", %{}), result}] + + _ -> + [] + end + end) + + cond do + exit != 0 -> :failed + results == [] -> :unobserved + Enum.any?(results, &(elem(&1, 1) == "failure")) -> :failed + Enum.any?(results, &(elem(&1, 1) == "skipped")) -> :skipped + true -> :completed + end end end diff --git a/lib/mix/tasks/hypatia.ci_changed.ex b/lib/mix/tasks/hypatia.ci_changed.ex index 6dd9f4ed..cdb47911 100644 --- a/lib/mix/tasks/hypatia.ci_changed.ex +++ b/lib/mix/tasks/hypatia.ci_changed.ex @@ -5,27 +5,32 @@ defmodule Mix.Tasks.Hypatia.CiChanged do @impl true def run(argv) do - {opts, args, invalid} = OptionParser.parse(argv, strict: [base: :string, plan: :boolean]) + {opts, args, invalid} = + OptionParser.parse(argv, strict: [base: :string, plan: :boolean, event: :string]) if args != [] or invalid != [], do: Mix.raise("usage: mix hypatia.ci_changed [--base REV] [--plan]") Mix.Task.run("compile") repo = File.cwd!() - plan = Hypatia.CiChanged.plan(repo, Keyword.get(opts, :base, "origin/main")) + plan = Hypatia.CiChanged.plan(repo, Keyword.get(opts, :base, "origin/main"), opts[:event]) if opts[:plan] do Mix.shell().info( Jason.encode!(Map.put(plan, :status, "planned, not executed"), pretty: true) ) else - findings = Hypatia.CiChanged.scan(repo, plan.files) + findings = + repo + |> Hypatia.CiChanged.scan(plan.files) + |> then(&Hypatia.CiChanged.apply_baseline(repo, &1)) - {_shown, blocking} = + {shown, blocking} = Hypatia.CLI.report_sets(findings, %{severity: "medium", advisory: false}) + if shown != [], do: Mix.shell().info(Jason.encode!(shown, pretty: true)) + if blocking != [] do - Mix.shell().info(Jason.encode!(findings, pretty: true)) Mix.raise("Hypatia found #{length(blocking)} blocking findings in changed files") end @@ -35,11 +40,14 @@ defmodule Mix.Tasks.Hypatia.CiChanged do results = Hypatia.CiChanged.execute_jobs(repo, plan.jobs) Enum.each(results, fn result -> Mix.shell().info(result.output) end) - if Enum.any?(results, &(&1.exit != 0)), - do: Mix.raise("changed workflow jobs failed; push is not ready") + if Enum.any?(results, &(&1.outcome != :completed)), + do: + Mix.raise( + "selected jobs are incomplete, skipped or need hosted validation; no execution receipt" + ) Mix.shell().info( - "#{length(plan.jobs)} changed jobs executed; Hypatia scan of #{length(plan.files)} changed files passed" + "#{length(plan.jobs)} selected jobs have successful observed receipts; Hypatia scan of #{length(plan.files)} changed files passed" ) end end diff --git a/test/ci_changed_receipt_test.exs b/test/ci_changed_receipt_test.exs new file mode 100644 index 00000000..d0d95a15 --- /dev/null +++ b/test/ci_changed_receipt_test.exs @@ -0,0 +1,445 @@ +# SPDX-License-Identifier: MPL-2.0 +defmodule Hypatia.CiChangedReceiptTest do + use ExUnit.Case, async: true + alias Hypatia.CiChanged + + defp repo do + directory = + Path.join( + System.tmp_dir!(), + "hypatia-ci-receipt-#{System.pid()}-#{System.os_time(:nanosecond)}" + ) + + File.mkdir_p!(Path.join(directory, ".github/workflows")) + on_exit(fn -> File.rm_rf!(directory) end) + git(directory, ["init", "-q"]) + directory + end + + defp git(repo, args) do + assert {_, 0} = System.cmd("git", args, cd: repo, stderr_to_stdout: true) + end + + defp commit(repo) do + git(repo, ["add", "."]) + + git(repo, [ + "-c", + "commit.gpgsign=false", + "-c", + "user.name=Test", + "-c", + "user.email=test@example.org", + "commit", + "--allow-empty", + "-qm", + "baseline" + ]) + end + + defp row(result, extra \\ %{}) do + Jason.encode!( + Map.merge( + %{"jobID" => "build", "jobResult" => result, "dryrun" => false, "matrix" => %{}}, + extra + ) + ) + end + + test "only observed, non-dry-run results for the selected job certify completion" do + assert CiChanged.receipt(row("success"), "build", 0) == :completed + assert CiChanged.receipt(row("skipped"), "build", 0) == :skipped + assert CiChanged.receipt("", "build", 0) == :unobserved + assert CiChanged.receipt(row("success", %{"jobID" => "other"}), "build", 0) == :unobserved + assert CiChanged.receipt(row("success", %{"dryrun" => true}), "build", 0) == :unobserved + assert CiChanged.receipt(row("success"), "build", 1) == :failed + + assert CiChanged.receipt( + row("success") <> "\n" <> row("skipped", %{"matrix" => %{"os" => "other"}}), + "build", + 0 + ) == :skipped + + message = Jason.encode!(%{"jobID" => "build", "msg" => row("success"), "dryrun" => false}) + assert CiChanged.receipt(message, "build", 0) == :unobserved + end + + test "source-only changes conservatively select existing jobs with their supported event" do + repo = repo() + + workflow = + "on: workflow_dispatch\npermissions: read-all\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v7\n - run: bash scripts/input.sh\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + File.mkdir_p!(Path.join(repo, "scripts")) + File.write!(Path.join(repo, "scripts/input.sh"), "exit 0\n") + commit(repo) + File.write!(Path.join(repo, "scripts/input.sh"), "exit 1\n") + + assert %{jobs: [%{job: "build", event: "workflow_dispatch", local_blocker: nil}]} = + CiChanged.plan(repo, "HEAD") + + assert_raise ArgumentError, fn -> CiChanged.plan(repo, "HEAD", "pull_request") end + end + + test "executor refuses every incomplete receipt outcome" do + jobs = [%{workflow: "ci.yml", job: "build", event: "push", local_blocker: nil}] + + for {output, exit, expected} <- [ + {row("success"), 1, :failed}, + {"", 0, :unobserved}, + {row("failure"), 0, :failed}, + {row("skipped"), 0, :skipped}, + {row("success", %{"dryrun" => true}), 0, :unobserved}, + {row("success"), 0, :completed} + ] do + assert [%{outcome: ^expected}] = + CiChanged.execute_jobs("/repo", jobs, fn _, _, _ -> {output, exit} end) + end + end + + test "transitive prerequisites retain hosted validation blockers" do + repo = repo() + + for ancestor <- [ + "permissions: {contents: write}\n steps: []", + "uses: owner/repo/.github/workflows/reuse.yml@main", + "steps: [{uses: actions/checkout@v7, with: {fetch-depth: 1}}]" + ] do + workflow = + "on: push\npermissions: read-all\njobs:\n setup:\n #{ancestor}\n middle:\n runs-on: ubuntu-latest\n needs: setup\n steps: [{uses: actions/checkout@v7}]\n build:\n runs-on: ubuntu-latest\n needs: middle\n steps: [{uses: actions/checkout@v7}, {run: echo before}]\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + commit(repo) + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + workflow + |> String.replace("echo before", "echo after") + |> String.replace("fetch-depth: 1", "fetch-depth: 0") + ) + + plan = CiChanged.plan(repo, "HEAD") + selected = Enum.filter(plan.jobs, &(&1.job == "build")) + assert [%{local_blocker: blocker}] = selected + assert blocker =~ "prerequisite setup" + + assert [%{outcome: :requires_hosted_receipt}] = + CiChanged.execute_jobs(repo, selected, fn _, _, _ -> + flunk("must not run prerequisites") + end) + end + end + + test "root composite action findings retain their actual path" do + repo = repo() + + File.write!( + Path.join(repo, "action.yml"), + "runs:\n using: composite\n steps:\n - uses: owner/action@main\n" + ) + + findings = CiChanged.scan(repo, ["action.yml"]) + assert Enum.any?(findings, &(&1.type == "RE006" and &1.file == "action.yml")) + end + + test "new checkout jobs and unreadable checkouts require hosted validation" do + repo = repo() + commit(repo) + + workflow = + "on: push\npermissions: read-all\njobs: {build: {runs-on: ubuntu-latest, steps: [{uses: actions/checkout@v7}, {run: echo ok}]}}\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "checkout" + + for permissions <- ["{}", "{contents: none}"] do + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + String.replace(workflow, "read-all", permissions) + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "contents: read" + end + end + + test "composite actions and direct event-file consumers cannot be locally certified" do + repo = repo() + + workflow = + "on: push\npermissions: read-all\njobs: {build: {runs-on: ubuntu-latest, steps: [{uses: actions/checkout@v7}, {run: echo before}]}}\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + commit(repo) + + for {step, expected} <- [ + {"uses: ./action", "composite"}, + {"run: cat $GITHUB_EVENT_PATH", "hosted-expression"} + ] do + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + String.replace(workflow, "run: echo before", step) + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ expected + end + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + String.replace(workflow, "ubuntu-latest", "self-hosted") + ) + + commit(repo) + File.write!(Path.join(repo, "input.txt"), "changed input\n") + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "host-native" + end + + test "checkout-less jobs, repository variables and matrices require hosted validation" do + repo = repo() + commit(repo) + + for {body, reason} <- [ + {"steps: [{run: echo ok}]", "checkout-less"}, + {"env: {FULL_TESTS: '${{ vars.FULL_TESTS }}'}\n steps: []", "repository-variable"}, + {"env: {FULL_TESTS: '${{ inputs.full_tests }}'}\n steps: []", "hosted-expression"}, + {"env: {EVENT: '${{ github.event_name }}'}\n steps: []", "hosted-expression"}, + {"strategy: {matrix: {os: [ubuntu-latest, windows-latest]}}\n runs-on: '${{ matrix.os }}'\n steps: []", + "matrix"} + ] do + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + "on: push\npermissions: read-all\njobs:\n build:\n #{body}\n" + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ reason + + assert [%{outcome: :requires_hosted_receipt}] = + CiChanged.execute_jobs(repo, CiChanged.plan(repo, "HEAD").jobs, fn _, _, _ -> + flunk("must not certify an incomplete local simulation") + end) + end + end + + test "changed prerequisites select their transitive consumers" do + workflow = + "on: push\njobs: {build: {steps: [{run: before}]}, test: {needs: build, steps: []}, deploy: {needs: test, steps: []}, unrelated: {steps: []}}\n" + + assert CiChanged.changed_jobs(workflow, String.replace(workflow, "before", "after")) == [ + "build", + "deploy", + "test" + ] + end + + test "changed reusable contracts select unchanged local callers" do + repo = repo() + + reusable = + "on: workflow_call\npermissions: read-all\njobs: {build: {steps: [{run: before}]}}\n" + + File.write!(Path.join(repo, ".github/workflows/reusable.yml"), reusable) + + File.write!( + Path.join(repo, ".github/workflows/caller.yml"), + "on: push\npermissions: read-all\njobs: {call: {uses: './.github/workflows/reusable.yml'}, consumer: {runs-on: ubuntu-latest, needs: call, steps: [{uses: actions/checkout@v7}]}}\n" + ) + + commit(repo) + + File.write!( + Path.join(repo, ".github/workflows/reusable.yml"), + String.replace(reusable, "before", "after") + ) + + jobs = CiChanged.plan(repo, "HEAD").jobs + + assert Enum.any?( + jobs, + &(&1.workflow == ".github/workflows/caller.yml" and &1.job == "call" and + &1.local_blocker =~ "reusable") + ) + + assert Enum.any?(jobs, &(&1.job == "consumer" and &1.local_blocker =~ "prerequisite call")) + end + + test "inherited permissions, conditions and secrets cannot earn local receipts" do + repo = repo() + commit(repo) + + for {permissions, body, expected} <- [ + {"", "steps: []", "inherited"}, + {"permissions: read-all\n", "steps: [{run: echo ok, if: false}]", "conditional"}, + {"permissions: read-all\n", "if: false\n steps: []", "conditional"}, + {"permissions: read-all\n", "env: {TOKEN: '${{ secrets.TOKEN }}'}\n steps: []", + "credential"}, + {"permissions: read-all\n", "steps: [{run: echo ok, continue-on-error: true}]", + "tolerated"} + ] do + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + "on: push\n#{permissions}jobs:\n build:\n #{body}\n" + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ expected + end + end + + test "removed and repositioned checkouts require hosted validation" do + repo = repo() + + workflow = + "on: push\npermissions: read-all\njobs: {build: {steps: [{uses: actions/checkout@v7}, {run: echo ok}]}}\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + commit(repo) + + for replacement <- [ + "steps: [{run: echo ok}]", + "steps: [{run: echo ok}, {uses: actions/checkout@v7}]" + ] do + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + String.replace( + workflow, + "steps: [{uses: actions/checkout@v7}, {run: echo ok}]", + replacement + ) + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "checkout" + end + end + + test "malformed current steps fail clearly and a corrected malformed base can be planned" do + repo = repo() + base = "on: push\npermissions: read-all\njobs: {build: {steps: null}}\n" + File.write!(Path.join(repo, ".github/workflows/ci.yml"), base) + commit(repo) + + assert_raise ArgumentError, ~r/steps must be a sequence/, fn -> + CiChanged.plan(repo, "HEAD") + end + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), String.replace(base, "null", "[]")) + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "checkout" + end + + test "whole secrets contexts and non-string scalar map keys are handled without JSON encoding" do + repo = repo() + commit(repo) + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + "on: push\npermissions: read-all\njobs: {build: {env: {TOKEN: '${{ toJSON(secrets) }}'}, steps: []}}\n" + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "credential" + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + "on: push\npermissions: read-all\njobs: {build: {env: {3: value}, steps: []}}\n" + ) + + assert [%{local_blocker: blocker}] = CiChanged.plan(repo, "HEAD").jobs + assert blocker =~ "checkout-less" + end + + test "deleting prerequisites and cyclic needs are rejected even when no surviving job changed" do + before = "on: push\npermissions: read-all\njobs: {a: {steps: []}, b: {needs: a, steps: []}}\n" + after_text = "on: push\npermissions: read-all\njobs: {b: {needs: a, steps: []}}\n" + assert_raise ArgumentError, fn -> CiChanged.changed_jobs(before, after_text) end + + assert_raise ArgumentError, fn -> + CiChanged.changed_jobs( + nil, + "on: push\npermissions: read-all\njobs: {a: {needs: b}, b: {needs: a}}\n" + ) + end + end + + test "changed checkout inputs require hosted validation rather than a skipped checkout receipt" do + repo = repo() + + workflow = + "on: push\npermissions: read-all\njobs:\n build:\n steps:\n - uses: actions/checkout@v7\n with: {fetch-depth: 1}\n - run: echo ok\n" + + File.write!(Path.join(repo, ".github/workflows/ci.yml"), workflow) + commit(repo) + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + String.replace(workflow, "fetch-depth: 1", "fetch-depth: 0") + ) + + plan = CiChanged.plan(repo, "HEAD") + assert [%{local_blocker: blocker}] = plan.jobs + assert blocker =~ "checkout" + + assert [%{outcome: :requires_hosted_receipt}] = + CiChanged.execute_jobs(repo, plan.jobs, fn _, _, _ -> + flunk("must not simulate changed checkout") + end) + end + + test "workflow-directory absence findings remain in scope" do + repo = repo() + + File.write!( + Path.join(repo, ".github/workflows/ci.yml"), + "on: push\npermissions: read-all\njobs: {build: {steps: [{run: echo ok}]}}\n" + ) + + findings = CiChanged.scan(repo, [".github/workflows/ci.yml"]) + assert Enum.any?(findings, &(&1.type == "missing_secret_history_scan")) + end + + @tag :baseline_helper + test "canonical baseline removes acknowledgements, applies overrides and rejects malformed input" do + repo = repo() + + helper = + System.get_env("HYPATIA_BASELINE_HELPER") || + Path.expand("../.standards/scripts/apply-baseline.sh", __DIR__) + + assert File.regular?(helper), "provision the governance-pinned baseline helper" + + baseline = %{ + severity: "high", + rule_module: "code_safety", + type: "elixir_atom_from_user", + file: "lib/input.ex", + expires_at: "2099-01-01", + note: "fixture" + } + + finding = %{ + severity: "high", + rule_module: "code_safety", + type: "elixir_atom_from_user", + file: "lib/input.ex", + reason: "fixture" + } + + path = Path.join(repo, ".hypatia-baseline.json") + File.write!(path, Jason.encode!([baseline])) + assert CiChanged.apply_baseline(repo, [finding], helper) == [] + File.write!(path, Jason.encode!([Map.put(baseline, :severity_override, "low")])) + assert [%{severity: "low"}] = CiChanged.apply_baseline(repo, [finding], helper) + File.write!(path, "{}") + assert_raise ArgumentError, fn -> CiChanged.apply_baseline(repo, [finding], helper) end + + assert_raise ArgumentError, fn -> + CiChanged.apply_baseline(repo, [finding], "/missing/helper") + end + end +end diff --git a/test/ci_changed_test.exs b/test/ci_changed_test.exs index ef0de319..70cc9d33 100644 --- a/test/ci_changed_test.exs +++ b/test/ci_changed_test.exs @@ -74,21 +74,42 @@ defmodule Hypatia.CiChangedTest do end test "executor preserves exact job argv and propagates failure" do - jobs = [%{workflow: ".github/workflows/ci.yml", job: "new-job"}] + jobs = [ + %{workflow: ".github/workflows/ci.yml", job: "new-job", event: "push", local_blocker: nil} + ] runner = fn "act", argv, opts -> - assert argv == [ - "pull_request", + assert [ + "push", "-W", ".github/workflows/ci.yml", "-j", "new-job", - "--bind", + "--json", + "--verbose", + "--secret-file", + empty, + "--env-file", + empty, + "--var-file", + empty, + "--input-file", + empty, + "--secret", + "GITHUB_TOKEN=", + "--container-daemon-socket", + "-", "--container-options", "--cap-drop=ALL" - ] + ] = argv assert opts[:cd] == "/repo" + assert File.read!(empty) == "" + env = Map.new(opts[:env]) + assert env["GH_TOKEN"] == nil + assert env["GITHUB_TOKEN"] == nil + assert Path.dirname(empty) == env["HOME"] + assert File.dir?(env["HOME"]) {"real job failed", 1} end diff --git a/test/test_helper.exs b/test/test_helper.exs index 70117db9..93cf7cc1 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -32,4 +32,7 @@ ExUnit.start() # Tests tagged :yaml_test_suite hold Hypatia.Kyaml to the YAML test suite; they # need a yaml-test-suite `data` checkout at YAML_TEST_SUITE, which the same job # pins. Run with: YAML_TEST_SUITE=... mix test --only yaml_test_suite. -ExUnit.configure(exclude: [:verisim_data, :kyq_differential, :yaml_test_suite], seed: 0) +ExUnit.configure( + exclude: [:verisim_data, :kyq_differential, :yaml_test_suite, :baseline_helper], + seed: 0 +)