diff --git a/free/ci/internal/nodes/registry/age_recipient.go b/free/ci/internal/nodes/registry/age_recipient.go new file mode 100644 index 000000000..7cb147b2c --- /dev/null +++ b/free/ci/internal/nodes/registry/age_recipient.go @@ -0,0 +1,57 @@ +package registry + +import ( + "context" + "errors" + "time" + + "github.com/nself-org/plugins/free/ci/internal/model" +) + +// AgeRecipient returns the recipient pinned by the first hello, if any. +func (r *Registry) AgeRecipient(ctx context.Context, id string) (*string, error) { + n, err := r.Get(ctx, id) + if err != nil { + return nil, err + } + return n.Capability.Identity.AgeRecipient.Value, nil +} + +// CheckAgeRecipient pins once, then flags a changed recipient without replacing it. +func (r *Registry) CheckAgeRecipient(ctx context.Context, id, presented, actor string) error { + if presented == "" { + return invalid("age recipient required") + } + n, err := r.Get(ctx, id) + if err != nil { + return err + } + if n.Capability.Identity.AgeRecipient.Value != nil && *n.Capability.Identity.AgeRecipient.Value != presented { + _, err = r.SetState(ctx, id, "maintenance", "age_recipient_changed", actor) + if err != nil { + return err + } + return &Error{Code: "E660", Reason: "age_recipient_changed"} + } + if n.Capability.Identity.AgeRecipient.Value != nil { + return nil + } + _, err = r.mutate(ctx, id, actor, "capability", func(n *Node) error { + if n.Capability.Identity.AgeRecipient.Value != nil { + if *n.Capability.Identity.AgeRecipient.Value != presented { + return &Error{Code: "E660", Reason: "age_recipient_changed"} + } + return nil + } + n.Capability.Identity.AgeRecipient = model.Fact[string]{Value: &presented, Source: "self", ObservedAt: time.Now().UTC(), Confidence: "known"} + return nil + }) + var changed *Error + if errors.As(err, &changed) && changed.Code == "E660" { + _, flagErr := r.SetState(ctx, id, "maintenance", "age_recipient_changed", actor) + if flagErr != nil { + return flagErr + } + } + return err +} diff --git a/free/ci/internal/nodes/registry/policy_guards.go b/free/ci/internal/nodes/registry/policy_guards.go new file mode 100644 index 000000000..e87a05d52 --- /dev/null +++ b/free/ci/internal/nodes/registry/policy_guards.go @@ -0,0 +1,52 @@ +package registry + +import ( + "strings" + + "github.com/nself-org/plugins/free/ci/internal/model" +) + +// PolicyActorPrefix marks a coordinator-side policy actor (the burst provider is "policy:burst"). Only such an +// actor may set or clear a node's hosted location or its nself.* labels; a node report may not (D-1402), because +// a hosted runner skips the AuthorizedProjects check in sched eligibility. +const PolicyActorPrefix = "policy:" + +func isHosted(f model.Fact[string]) bool { return f.Value != nil && *f.Value == "hosted" } + +// keepHostedKind returns the reported location kind, except that a report cannot make a node hosted or stop it +// being hosted: then the stored kind stays. +func keepHostedKind(old, reported model.Fact[string]) model.Fact[string] { + if isHosted(old) != isHosted(reported) { + return old + } + return reported +} + +// keepProviderLabels returns the reported labels with the nself.* ones replaced by the stored nself.* ones. +func keepProviderLabels(old, reported []string) []string { + const prefix = "nself." + var out []string + for _, l := range reported { + if !strings.HasPrefix(l, prefix) { + out = append(out, l) + } + } + for _, l := range old { + if strings.HasPrefix(l, prefix) { + out = append(out, l) + } + } + if reported == nil && out == nil { + return nil + } + return out +} + +func isolationRank(i model.Isolation) int { + for n, v := range model.EnumValues("Isolation") { + if v == string(i) { + return n + } + } + return -1 +} diff --git a/free/ci/internal/nodes/registry/registry.go b/free/ci/internal/nodes/registry/registry.go index c04236387..71dea80d8 100644 --- a/free/ci/internal/nodes/registry/registry.go +++ b/free/ci/internal/nodes/registry/registry.go @@ -171,51 +171,6 @@ func (r *Registry) UpdateCapability(ctx context.Context, id string, in model.Cap return n, drift, err } -// PolicyActorPrefix marks a coordinator-side policy actor (the burst provider is "policy:burst"). Only such an -// actor may set or clear a node's hosted location or its nself.* labels; a node report may not (D-1402), because -// a hosted runner skips the AuthorizedProjects check in sched eligibility. -const PolicyActorPrefix = "policy:" - -func isHosted(f model.Fact[string]) bool { return f.Value != nil && *f.Value == "hosted" } - -// keepHostedKind returns the reported location kind, except that a report cannot make a node hosted or stop it -// being hosted: then the stored kind stays. -func keepHostedKind(old, reported model.Fact[string]) model.Fact[string] { - if isHosted(old) != isHosted(reported) { - return old - } - return reported -} - -// keepProviderLabels returns the reported labels with the nself.* ones replaced by the stored nself.* ones. -func keepProviderLabels(old, reported []string) []string { - const prefix = "nself." - var out []string - for _, l := range reported { - if !strings.HasPrefix(l, prefix) { - out = append(out, l) - } - } - for _, l := range old { - if strings.HasPrefix(l, prefix) { - out = append(out, l) - } - } - if reported == nil && out == nil { - return nil - } - return out -} - -func isolationRank(i model.Isolation) int { - for n, v := range model.EnumValues("Isolation") { - if v == string(i) { - return n - } - } - return -1 -} - // SetTrust is the only authority for coordinator-assigned trust fields. func (r *Registry) SetTrust(ctx context.Context, id string, t model.CapabilityTrust, authority store.AdminAuthority) (Node, error) { return r.mutate(ctx, id, authority.Actor(), "trust", func(n *Node) error { @@ -292,51 +247,3 @@ func (r *Registry) SetState(ctx context.Context, id, state, reason, actor string return nil }) } - -// AgeRecipient returns the recipient pinned by the first hello, if any. -func (r *Registry) AgeRecipient(ctx context.Context, id string) (*string, error) { - n, err := r.Get(ctx, id) - if err != nil { - return nil, err - } - return n.Capability.Identity.AgeRecipient.Value, nil -} - -// CheckAgeRecipient pins once, then flags a changed recipient without replacing it. -func (r *Registry) CheckAgeRecipient(ctx context.Context, id, presented, actor string) error { - if presented == "" { - return invalid("age recipient required") - } - n, err := r.Get(ctx, id) - if err != nil { - return err - } - if n.Capability.Identity.AgeRecipient.Value != nil && *n.Capability.Identity.AgeRecipient.Value != presented { - _, err = r.SetState(ctx, id, "maintenance", "age_recipient_changed", actor) - if err != nil { - return err - } - return &Error{Code: "E660", Reason: "age_recipient_changed"} - } - if n.Capability.Identity.AgeRecipient.Value != nil { - return nil - } - _, err = r.mutate(ctx, id, actor, "capability", func(n *Node) error { - if n.Capability.Identity.AgeRecipient.Value != nil { - if *n.Capability.Identity.AgeRecipient.Value != presented { - return &Error{Code: "E660", Reason: "age_recipient_changed"} - } - return nil - } - n.Capability.Identity.AgeRecipient = model.Fact[string]{Value: &presented, Source: "self", ObservedAt: time.Now().UTC(), Confidence: "known"} - return nil - }) - var changed *Error - if errors.As(err, &changed) && changed.Code == "E660" { - _, flagErr := r.SetState(ctx, id, "maintenance", "age_recipient_changed", actor) - if flagErr != nil { - return flagErr - } - } - return err -} diff --git a/free/github-runner/internal/trust.go b/free/github-runner/internal/trust.go index 87fda275e..de9c8f7bd 100644 --- a/free/github-runner/internal/trust.go +++ b/free/github-runner/internal/trust.go @@ -1,7 +1,6 @@ package internal import ( - "bytes" "context" "encoding/json" "errors" @@ -11,8 +10,6 @@ import ( "net/http" "net/url" "os" - "path/filepath" - "strconv" "strings" "time" ) @@ -160,149 +157,6 @@ func checkGroup(ctx context.Context, cfg TrustConfig) (*TrustResult, error) { "Restrict the group to private repositories in the organisation settings, or set GITHUB_RUNNER_ALLOW_PUBLIC_REPOS=true to accept the risk", g.Name)} } -// runnerRef identifies a runner: by id when GitHub issued one (the persisted -// registration), else by display name. -type runnerRef struct { - ID int64 - Name string -} - -func (r runnerRef) matches(o runnerRef) bool { - if r.ID != 0 { - return o.ID == r.ID - } - return r.Name != "" && strings.EqualFold(o.Name, r.Name) -} - -func (r runnerRef) String() string { - if r.Name != "" { - return strconv.Quote(r.Name) - } - return "#" + strconv.FormatInt(r.ID, 10) -} - -// persistedRunner reads the registration the runner binary wrote to -// RunnerDir/.runner. Absent means not registered (nil, nil). A file that -// exists but cannot be read or parsed is an error: the guard fails closed -// rather than trust the configured name alone. -func persistedRunner(dir string) (*runnerRef, error) { - if dir == "" { - return nil, nil - } - raw, err := os.ReadFile(filepath.Join(dir, ".runner")) - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("read %s: %w", filepath.Join(dir, ".runner"), err) - } - var f struct { - AgentID json.Number `json:"agentId"` - AgentName string `json:"agentName"` - } - // The runner writes the file with a UTF-8 BOM. - if err := json.Unmarshal(bytes.TrimPrefix(raw, []byte("\xef\xbb\xbf")), &f); err != nil { - return nil, fmt.Errorf("parse %s: %w", filepath.Join(dir, ".runner"), err) - } - ref := runnerRef{Name: f.AgentName} - if f.AgentID != "" { - if ref.ID, err = f.AgentID.Int64(); err != nil { - return nil, fmt.Errorf("parse agentId in %s: %w", filepath.Join(dir, ".runner"), err) - } - } - if ref.ID == 0 && ref.Name == "" { - return nil, fmt.Errorf("%s names no runner", filepath.Join(dir, ".runner")) - } - return &ref, nil -} - -// verifyRegisteredGroup refuses a runner that GitHub has registered outside -// the configured group: the guard reads the configured group, so a runner left -// in another (possibly public) group by a config change would otherwise pass. -// Both the configured name and the registration persisted in the runner dir -// are checked, so renaming the runner (GITHUB_RUNNER_NAME or the hostname) does -// not hide the installed one. A runner that is not registered at all passes (it -// is about to be). -func verifyRegisteredGroup(ctx context.Context, cfg TrustConfig, g *runnerGroup) error { - var refs []runnerRef - if cfg.RunnerName != "" { - refs = append(refs, runnerRef{Name: cfg.RunnerName}) - } - persisted, err := persistedRunner(cfg.RunnerDir) - if err != nil { - return &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, - Message: fmt.Sprintf("the installed runner registration cannot be read, so its runner group cannot be verified: %v", err)} - } - if persisted != nil { - refs = append(refs, *persisted) - } - if len(refs) == 0 { - return nil - } - if g.ID == 0 { - return &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, - Message: fmt.Sprintf("GitHub did not report the id of runner group %q", g.Name)} - } - org := url.PathEscape(cfg.Org) - inGroup, err := listRunners(ctx, cfg, fmt.Sprintf("%s/orgs/%s/actions/runner-groups/%d/runners", githubAPIBase, org, g.ID)) - if err != nil { - return err - } - var inOrg []runnerRef - orgLoaded := false - for _, ref := range refs { - if anyMatch(ref, inGroup) { - continue - } - if !orgLoaded { - if inOrg, err = listRunners(ctx, cfg, fmt.Sprintf("%s/orgs/%s/actions/runners", githubAPIBase, org)); err != nil { - return err - } - orgLoaded = true - } - if !anyMatch(ref, inOrg) { - continue - } - return &TrustError{Code: CodeRunnerGroupMismatch, Status: http.StatusConflict, - Message: fmt.Sprintf("runner %s is registered in a different runner group from the configured %q, so the check above does not cover it. "+ - "Re-register the runner (install again) so it joins %q, or set GITHUB_RUNNER_GROUP to the group it is in", ref, g.Name, g.Name)} - } - return nil -} - -func anyMatch(ref runnerRef, list []runnerRef) bool { - for _, r := range list { - if ref.matches(r) { - return true - } - } - return false -} - -// listRunners pages through a runner list. -func listRunners(ctx context.Context, cfg TrustConfig, endpoint string) ([]runnerRef, error) { - var all []runnerRef - for page := 1; page <= maxGroupPages; page++ { - var body struct { - Runners []struct { - ID int64 `json:"id"` - Name string `json:"name"` - } `json:"runners"` - } - if err := githubGet(ctx, cfg.PAT, fmt.Sprintf("%s?per_page=%d&page=%d", endpoint, groupsPageSize, page), &body); err != nil { - return nil, err - } - for _, r := range body.Runners { - all = append(all, runnerRef{ID: r.ID, Name: r.Name}) - } - if len(body.Runners) < groupsPageSize { - return all, nil - } - } - return nil, &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, - Message: "the runner list is too long to search"} -} - // stopsRunner reports whether a refusal with code must stop a running runner. func stopsRunner(code string) bool { return code == CodeRunnerGroupPublic || code == CodeRunnerGroupMismatch diff --git a/free/github-runner/internal/trust_registration.go b/free/github-runner/internal/trust_registration.go new file mode 100644 index 000000000..e2273d2a4 --- /dev/null +++ b/free/github-runner/internal/trust_registration.go @@ -0,0 +1,158 @@ +package internal + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" +) + +// runnerRef identifies a runner: by id when GitHub issued one (the persisted +// registration), else by display name. +type runnerRef struct { + ID int64 + Name string +} + +func (r runnerRef) matches(o runnerRef) bool { + if r.ID != 0 { + return o.ID == r.ID + } + return r.Name != "" && strings.EqualFold(o.Name, r.Name) +} + +func (r runnerRef) String() string { + if r.Name != "" { + return strconv.Quote(r.Name) + } + return "#" + strconv.FormatInt(r.ID, 10) +} + +// persistedRunner reads the registration the runner binary wrote to +// RunnerDir/.runner. Absent means not registered (nil, nil). A file that +// exists but cannot be read or parsed is an error: the guard fails closed +// rather than trust the configured name alone. +func persistedRunner(dir string) (*runnerRef, error) { + if dir == "" { + return nil, nil + } + raw, err := os.ReadFile(filepath.Join(dir, ".runner")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", filepath.Join(dir, ".runner"), err) + } + var f struct { + AgentID json.Number `json:"agentId"` + AgentName string `json:"agentName"` + } + // The runner writes the file with a UTF-8 BOM. + if err := json.Unmarshal(bytes.TrimPrefix(raw, []byte("\xef\xbb\xbf")), &f); err != nil { + return nil, fmt.Errorf("parse %s: %w", filepath.Join(dir, ".runner"), err) + } + ref := runnerRef{Name: f.AgentName} + if f.AgentID != "" { + if ref.ID, err = f.AgentID.Int64(); err != nil { + return nil, fmt.Errorf("parse agentId in %s: %w", filepath.Join(dir, ".runner"), err) + } + } + if ref.ID == 0 && ref.Name == "" { + return nil, fmt.Errorf("%s names no runner", filepath.Join(dir, ".runner")) + } + return &ref, nil +} + +// verifyRegisteredGroup refuses a runner that GitHub has registered outside +// the configured group: the guard reads the configured group, so a runner left +// in another (possibly public) group by a config change would otherwise pass. +// Both the configured name and the registration persisted in the runner dir +// are checked, so renaming the runner (GITHUB_RUNNER_NAME or the hostname) does +// not hide the installed one. A runner that is not registered at all passes (it +// is about to be). +func verifyRegisteredGroup(ctx context.Context, cfg TrustConfig, g *runnerGroup) error { + var refs []runnerRef + if cfg.RunnerName != "" { + refs = append(refs, runnerRef{Name: cfg.RunnerName}) + } + persisted, err := persistedRunner(cfg.RunnerDir) + if err != nil { + return &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, + Message: fmt.Sprintf("the installed runner registration cannot be read, so its runner group cannot be verified: %v", err)} + } + if persisted != nil { + refs = append(refs, *persisted) + } + if len(refs) == 0 { + return nil + } + if g.ID == 0 { + return &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, + Message: fmt.Sprintf("GitHub did not report the id of runner group %q", g.Name)} + } + org := url.PathEscape(cfg.Org) + inGroup, err := listRunners(ctx, cfg, fmt.Sprintf("%s/orgs/%s/actions/runner-groups/%d/runners", githubAPIBase, org, g.ID)) + if err != nil { + return err + } + var inOrg []runnerRef + orgLoaded := false + for _, ref := range refs { + if anyMatch(ref, inGroup) { + continue + } + if !orgLoaded { + if inOrg, err = listRunners(ctx, cfg, fmt.Sprintf("%s/orgs/%s/actions/runners", githubAPIBase, org)); err != nil { + return err + } + orgLoaded = true + } + if !anyMatch(ref, inOrg) { + continue + } + return &TrustError{Code: CodeRunnerGroupMismatch, Status: http.StatusConflict, + Message: fmt.Sprintf("runner %s is registered in a different runner group from the configured %q, so the check above does not cover it. "+ + "Re-register the runner (install again) so it joins %q, or set GITHUB_RUNNER_GROUP to the group it is in", ref, g.Name, g.Name)} + } + return nil +} + +func anyMatch(ref runnerRef, list []runnerRef) bool { + for _, r := range list { + if ref.matches(r) { + return true + } + } + return false +} + +// listRunners pages through a runner list. +func listRunners(ctx context.Context, cfg TrustConfig, endpoint string) ([]runnerRef, error) { + var all []runnerRef + for page := 1; page <= maxGroupPages; page++ { + var body struct { + Runners []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"runners"` + } + if err := githubGet(ctx, cfg.PAT, fmt.Sprintf("%s?per_page=%d&page=%d", endpoint, groupsPageSize, page), &body); err != nil { + return nil, err + } + for _, r := range body.Runners { + all = append(all, runnerRef{ID: r.ID, Name: r.Name}) + } + if len(body.Runners) < groupsPageSize { + return all, nil + } + } + return nil, &TrustError{Code: CodeTrustUnverifiable, Status: http.StatusBadGateway, + Message: "the runner list is too long to search"} +} diff --git a/scripts/verify-published-checksums-lib.sh b/scripts/verify-published-checksums-lib.sh index dcfe3581e..18cdc7270 100644 --- a/scripts/verify-published-checksums-lib.sh +++ b/scripts/verify-published-checksums-lib.sh @@ -49,3 +49,83 @@ asset_is_published() { grep -Fxq "$1" "$ASSET_MANIFEST"; } # Matches internal/plugin/arch.go's PlatformArch(), the same list scripts/build-tarballs.sh cross-compiles for a binaryName plugin. PLATFORMS="darwin-arm64 darwin-amd64 linux-amd64 linux-arm64 windows-amd64" BINARY_RE='^nself-[a-z][a-z0-9-]*$' + +# upsert_release_row NAME TARBALL_NAME SHA PLATFORMS_JSON BINARIES_JSON SIG_JSON PSIGS_JSON: print RELEASES_JSON with NAME's row rewritten +# from the published bytes (reads VERSION, TAG, REPO, SIGNATURES_FILE and RELEASES_JSON from the CLI). +upsert_release_row() { + local name="$1" tarball_name="$2" computed_sha="$3" platforms_json="$4" binaries_json="$5" sig_json="$6" psigs_json="$7" signed=false + [ -z "$SIGNATURES_FILE" ] || signed=true + jq --arg n "$name" --arg v "$VERSION" --arg tag "$TAG" --arg sha "$computed_sha" \ + --arg url "https://github.com/${REPO}/releases/download/${TAG}/${tarball_name}" \ + --argjson plat "$platforms_json" --argjson bins "$binaries_json" --argjson sig "$sig_json" \ + --argjson psig "$psigs_json" --argjson signed "$signed" ' + (.releases[$n] // {release_signature: null}) as $old + | .releases[$n] = ($old + | (if .sha256 != $sha then .release_signature = null else . end) + | (if $sig != null then .release_signature = $sig else . end) + | . + {version: $v, sha256: $sha, release_tag: $tag, tarball_url: $url} + | del(.tarball) + | if ($plat | length) > 0 then .platform_checksums = $plat else del(.platform_checksums) end + | .platform_signatures = (if $signed then $psig + elif $old.version == $v and .release_signature != null then + (($old.platform_signatures // {}) | with_entries(select(.key as $p | $plat[$p] != null and ($old.platform_checksums // {})[$p] == $plat[$p]))) + else {} end) + | if (.platform_signatures | length) == 0 then del(.platform_signatures) else . end + | if ($bins | length) > 0 then .binaries = $bins else del(.binaries) end)' <<<"$RELEASES_JSON" +} + +# write_releases_json: write RELEASES_JSON over RELEASES_FILE atomically, or exit 1 with releases.json unchanged. +write_releases_json() { + # `a && b` is exempt from errexit, so a failed write must be tested explicitly or the script would log Wrote and exit 0. + # A unique temp file beside releases.json (same filesystem, so mv is atomic); cleanup is nonrecursive so nothing pre-existing is deleted. + RELEASES_TMP="" + if ! RELEASES_TMP="$(mktemp "${RELEASES_FILE}.XXXXXX")" || ! jq -S --indent 2 . <<<"$RELEASES_JSON" > "$RELEASES_TMP" || ! mv "$RELEASES_TMP" "$RELEASES_FILE"; then + [ -z "$RELEASES_TMP" ] || rm -f "$RELEASES_TMP" + err "could not write ${RELEASES_FILE}; releases.json is unchanged." + exit 1 + fi +} + +# report_failures: explain each non-zero failure counter of the CLI (MISSING_ASSETS, DOWNLOAD_ERRORS, UNSIGNED, UNAPPROVED, UNREADABLE, ROLLBACKS). +report_failures() { + if [ "$MISSING_ASSETS" -gt 0 ]; then + err "${MISSING_ASSETS} expected asset(s) are genuinely absent from release ${TAG}." + err "That is a release defect: rebuild and upload the missing artifact(s)." + fi + if [ "$DOWNLOAD_ERRORS" -gt 0 ]; then + err "${DOWNLOAD_ERRORS} asset(s) are published but could not be downloaded." + err "That is transient (network or GitHub rate limiting), NOT a missing artifact." + err "Re-run this script; do not rebuild or re-upload anything on account of it." + fi + if [ "$UNSIGNED" -gt 0 ]; then + err "${UNSIGNED} row(s) have no signature over the published bytes; the signing job must sign what is on the release." + fi + if [ "$UNAPPROVED" -gt 0 ]; then + err "${UNAPPROVED} tarball(s) differ from the publish job's approved manifest; the Release changed after approval. Nothing is written." + fi + if [ "$UNREADABLE" -gt 0 ]; then + err "${UNREADABLE} linux-amd64 archive(s) are published but corrupt; rebuild and upload them." + fi + if [ "$ROLLBACKS" -gt 0 ]; then + err "${ROLLBACKS} row(s) already hold a newer version than ${TAG}; releases.json is not rolled back." + fi +} + +# fetch_release_assets: list TAG's assets into ASSET_MANIFEST and download them all into WORK_DIR, or exit 1 (see the CLI for why once). +fetch_release_assets() { + local dl_err + ASSET_MANIFEST="${WORK_DIR}/.asset-manifest" + if ! gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[].name' > "$ASSET_MANIFEST" 2>/dev/null; then + err "could not list assets on release $TAG" + exit 1 + fi + log "Release $TAG lists $(wc -l < "$ASSET_MANIFEST" | tr -d ' ') asset(s); downloading them once." + + dl_err="${WORK_DIR}/.download-stderr" + if ! gh release download "$TAG" --repo "$REPO" --dir "$WORK_DIR" --clobber >/dev/null 2>"$dl_err"; then + err "bulk download of $TAG's assets failed; cannot verify against published bytes." + err "gh said: $(tr '\n' ' ' < "$dl_err" | cut -c1-400)" + exit 1 + fi + rm -f "$dl_err" +} diff --git a/scripts/verify-published-checksums.sh b/scripts/verify-published-checksums.sh index 9f494cf11..66f0a2af2 100755 --- a/scripts/verify-published-checksums.sh +++ b/scripts/verify-published-checksums.sh @@ -123,20 +123,7 @@ WRITTEN=0 # secondary rate limit and, with the output discarded, made a throttled download look like a MISSING asset (a false alarm # seen on v1.2.1 for flags/linux-arm64). So: list once, download everything once, then work from local files. An asset absent # from the list is MISSING (a real release defect); one in the list with no local file is a DOWNLOAD ERROR (transient). -ASSET_MANIFEST="${WORK_DIR}/.asset-manifest" -if ! gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[].name' > "$ASSET_MANIFEST" 2>/dev/null; then - err "could not list assets on release $TAG" - exit 1 -fi -log "Release $TAG lists $(wc -l < "$ASSET_MANIFEST" | tr -d ' ') asset(s); downloading them once." - -dl_err="${WORK_DIR}/.download-stderr" -if ! gh release download "$TAG" --repo "$REPO" --dir "$WORK_DIR" --clobber >/dev/null 2>"$dl_err"; then - err "bulk download of $TAG's assets failed; cannot verify against published bytes." - err "gh said: $(tr '\n' ' ' < "$dl_err" | cut -c1-400)" - exit 1 -fi -rm -f "$dl_err" +fetch_release_assets # Plugins to process, in a stable order: rows already on TAG, plus (with --write) manifests at this version with no row on TAG. # With --signatures the signed set decides instead: every signed source tarball must land as a row (or fail as MISSING), whatever @@ -282,23 +269,7 @@ while IFS= read -r name; do binaries_json="$( { printf '%s\n' "$listing" | awk -F/ '{print $NF}' | grep -E "$BINARY_RE" || true; } | sort -u | jq -R . | jq -sc .)" [ "$binaries_json" != "[]" ] || log "WARNING: ${name}-${VERSION}-linux-amd64.tar.gz holds no nself-* binary; the row gets no binaries list" fi - RELEASES_JSON="$(jq --arg n "$name" --arg v "$VERSION" --arg tag "$TAG" --arg sha "$computed_sha" \ - --arg url "https://github.com/${REPO}/releases/download/${TAG}/${tarball_name}" \ - --argjson plat "$platforms_json" --argjson bins "$binaries_json" --argjson sig "$sig_json" \ - --argjson psig "$psigs_json" --argjson signed "$([ -n "$SIGNATURES_FILE" ] && echo true || echo false)" ' - (.releases[$n] // {release_signature: null}) as $old - | .releases[$n] = ($old - | (if .sha256 != $sha then .release_signature = null else . end) - | (if $sig != null then .release_signature = $sig else . end) - | . + {version: $v, sha256: $sha, release_tag: $tag, tarball_url: $url} - | del(.tarball) - | if ($plat | length) > 0 then .platform_checksums = $plat else del(.platform_checksums) end - | .platform_signatures = (if $signed then $psig - elif $old.version == $v and .release_signature != null then - (($old.platform_signatures // {}) | with_entries(select(.key as $p | $plat[$p] != null and ($old.platform_checksums // {})[$p] == $plat[$p]))) - else {} end) - | if (.platform_signatures | length) == 0 then del(.platform_signatures) else . end - | if ($bins | length) > 0 then .binaries = $bins else del(.binaries) end)' <<<"$RELEASES_JSON")" + RELEASES_JSON="$(upsert_release_row "$name" "$tarball_name" "$computed_sha" "$platforms_json" "$binaries_json" "$sig_json" "$psigs_json")" WRITTEN=$((WRITTEN + 1)) fi done <<<"$(printf '%s\n' "$candidates" | sort -u)" @@ -306,41 +277,14 @@ done <<<"$(printf '%s\n' "$candidates" | sort -u)" if [ "$MISSING_ASSETS" -gt 0 ] || [ "$DOWNLOAD_ERRORS" -gt 0 ] || [ "$UNSIGNED" -gt 0 ] || [ "$UNAPPROVED" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] || [ "$ROLLBACKS" -gt 0 ]; then [ "$WRITE" != "true" ] || log "releases.json NOT written: an expected asset is missing, failed to download, is unsigned, unapproved or unreadable." elif [ "$WRITE" = "true" ]; then - # `a && b` is exempt from errexit, so a failed write must be tested explicitly or the script would log Wrote and exit 0. - # A unique temp file beside releases.json (same filesystem, so mv is atomic); cleanup is nonrecursive so nothing pre-existing is deleted. - RELEASES_TMP="" - if ! RELEASES_TMP="$(mktemp "${RELEASES_FILE}.XXXXXX")" || ! jq -S --indent 2 . <<<"$RELEASES_JSON" > "$RELEASES_TMP" || ! mv "$RELEASES_TMP" "$RELEASES_FILE"; then - [ -z "$RELEASES_TMP" ] || rm -f "$RELEASES_TMP" - err "could not write ${RELEASES_FILE}; releases.json is unchanged." - exit 1 - fi + write_releases_json log "Wrote ${WRITTEN} row(s) into releases.json from ${TAG}'s published assets (checked ${CHECKED} asset(s))." log "Next: scripts/plugin-counts.sh regenerates registry.json, catalog.json and counts.json from it." else log "Checked ${CHECKED} published asset(s) against releases.json for ${TAG}. Mismatches: ${MISMATCHES}." fi -if [ "$MISSING_ASSETS" -gt 0 ]; then - err "${MISSING_ASSETS} expected asset(s) are genuinely absent from release ${TAG}." - err "That is a release defect: rebuild and upload the missing artifact(s)." -fi -if [ "$DOWNLOAD_ERRORS" -gt 0 ]; then - err "${DOWNLOAD_ERRORS} asset(s) are published but could not be downloaded." - err "That is transient (network or GitHub rate limiting), NOT a missing artifact." - err "Re-run this script; do not rebuild or re-upload anything on account of it." -fi -if [ "$UNSIGNED" -gt 0 ]; then - err "${UNSIGNED} row(s) have no signature over the published bytes; the signing job must sign what is on the release." -fi -if [ "$UNAPPROVED" -gt 0 ]; then - err "${UNAPPROVED} tarball(s) differ from the publish job's approved manifest; the Release changed after approval. Nothing is written." -fi -if [ "$UNREADABLE" -gt 0 ]; then - err "${UNREADABLE} linux-amd64 archive(s) are published but corrupt; rebuild and upload them." -fi -if [ "$ROLLBACKS" -gt 0 ]; then - err "${ROLLBACKS} row(s) already hold a newer version than ${TAG}; releases.json is not rolled back." -fi +report_failures if [ "$MISMATCHES" -gt 0 ] || [ "$MISSING_ASSETS" -gt 0 ] || [ "$DOWNLOAD_ERRORS" -gt 0 ] || [ "$UNSIGNED" -gt 0 ] || [ "$UNAPPROVED" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] || [ "$ROLLBACKS" -gt 0 ]; then exit 1 fi