diff --git a/free/ci/internal/config/migrate.go b/free/ci/internal/config/migrate.go index 73d73fa6b..bcc97acbc 100644 --- a/free/ci/internal/config/migrate.go +++ b/free/ci/internal/config/migrate.go @@ -165,6 +165,9 @@ func Migrate(root string) (*Migration, error) { return nil, untranslatable(rel, 0, "gateway_routes", "gateway route probes are not jobs; remove the field or move the probes into a job run argv") } dir := filepath.ToSlash(filepath.Dir(rel)) + if dir != "." && len(lf.Stages) > 0 { + m.Notes = append(m.Notes, rel+": the legacy runner ran every stage on every run; the migrated jobs get when.paths "+dir+"/** and run only when a file under that directory changed, so remove the when block from a job that must always run") + } for i, st := range lf.Stages { field := fmt.Sprintf("stages[%d]", i) id := jobID(lf.Plugin, st.Name, taken) diff --git a/free/ci/internal/config/migrate_render.go b/free/ci/internal/config/migrate_render.go index 2b57fc854..cea74eb4b 100644 --- a/free/ci/internal/config/migrate_render.go +++ b/free/ci/internal/config/migrate_render.go @@ -99,6 +99,10 @@ func (m *Migration) Diff(root string) (string, error) { return b.String(), nil } +// testHookBeforeCreate runs between the destination check and the create; +// tests use it to swap a path, as a concurrent local writer could. +var testHookBeforeCreate = func(string) {} + // Write creates root/.nself/ci.yaml. It refuses to replace a file that holds // other content (merge by hand) and succeeds quietly when the file is identical. func (m *Migration) Write(root string) error { @@ -114,10 +118,18 @@ func (m *Migration) Write(root string) error { } else if !os.IsNotExist(err) { return &Error{Code: CodeInvalid, Problems: []Problem{{File: FileName, Message: err.Error()}}} } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + testHookBeforeCreate(root) + // checkDestination looked at the path; a symlink swapped in since then must + // not carry the write out of the repository. os.Root confines every step. + r, err := os.OpenRoot(root) + if err != nil { + return err + } + defer func() { _ = r.Close() }() + if err := r.MkdirAll(filepath.ToSlash(filepath.Dir(FileName)), 0o755); err != nil { return err } - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + f, err := r.OpenFile(filepath.ToSlash(FileName), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) if err != nil { return err } diff --git a/free/ci/internal/config/migrate_test.go b/free/ci/internal/config/migrate_test.go index 17041a86c..6dcc1f165 100644 --- a/free/ci/internal/config/migrate_test.go +++ b/free/ci/internal/config/migrate_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/nself-org/plugins/free/ci/internal/legacy" ) // copyFixture copies testdata//input into a temp repo root, turning each @@ -334,3 +336,92 @@ func TestOracleSkipsManifestsWithoutAPluginKey(t *testing.T) { } } } + +// A .nself directory swapped for a symlink after checkDestination must not +// carry the write out of the repository. +func TestWriteRefusesSymlinkSwappedAfterCheck(t *testing.T) { + root := copyFixture(t, filepath.Join("legacy", "gauth")) + outside := t.TempDir() + m, err := Migrate(root) + if err != nil { + t.Fatal(err) + } + testHookBeforeCreate = func(r string) { + testHookBeforeCreate = func(string) {} + if err := os.Symlink(outside, filepath.Join(r, ".nself")); err != nil { + t.Error(err) + } + } + t.Cleanup(func() { testHookBeforeCreate = func(string) {} }) + if err := m.Write(root); err == nil { + t.Fatal("wrote through a .nself symlink swapped in after the check") + } + if entries, _ := os.ReadDir(outside); len(entries) != 0 { + t.Fatalf("file created outside the repo: %v", entries) + } +} + +// A manifest below the root migrates to path-gated jobs the legacy runner +// never had; the migration must say so. +func TestMigrateNotesPathSelection(t *testing.T) { + m, err := Migrate(copyFixture(t, filepath.Join("legacy", "multi"))) + if err != nil { + t.Fatal(err) + } + var got []string + for _, n := range m.Notes { + if strings.Contains(n, "when.paths") { + got = append(got, n) + } + } + if len(got) != 1 || !strings.HasPrefix(got[0], "free/other/.ci.yaml: ") || !strings.Contains(got[0], "ran every stage on every run") { + t.Errorf("path-selection note: %v (all notes %v)", got, m.Notes) + } +} + +// The path-selection note says the legacy runner ran every stage on every run, +// in the manifest's own directory. Run the real legacy runner twice over the +// same unchanged tree and check both halves of that claim. +func TestLegacyRunnerRanEveryStageEveryRun(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "free", "other") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // Block-list cmd: the legacy line scanner does not read flow lists. + manifest := "plugin: other\nstages:\n" + + " - name: first\n cmd:\n - sh\n - -c\n - echo x >> first.log\n" + + " - name: second\n cmd:\n - sh\n - -c\n - echo x >> second.log\n" + if err := os.WriteFile(filepath.Join(dir, ".ci.yaml"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + for run := 1; run <= 2; run++ { + results, err := legacy.DiscoverAndRunPipeline(root, "", 30, false) + if err != nil { + t.Fatal(err) + } + if len(results) != 2 { + t.Fatalf("run %d: %d results, want one per stage", run, len(results)) + } + for _, log := range []string{"first.log", "second.log"} { + data, err := os.ReadFile(filepath.Join(dir, log)) + if err != nil { + t.Fatalf("run %d: %s not written in the manifest directory: %v", run, log, err) + } + if got := strings.Count(string(data), "x"); got != run { + t.Errorf("run %d: %s has %d lines; the stage must run once per run with no change filter", run, log, got) + } + } + } + m, err := Migrate(root) + if err != nil { + t.Fatal(err) + } + found := false + for _, n := range m.Notes { + found = found || strings.Contains(n, "ran every stage on every run") + } + if !found { + t.Errorf("the migration dropped the every-run note: %v", m.Notes) + } +} diff --git a/free/ci/internal/config/readfile.go b/free/ci/internal/config/readfile.go index 6151c7290..7130736c2 100644 --- a/free/ci/internal/config/readfile.go +++ b/free/ci/internal/config/readfile.go @@ -6,7 +6,9 @@ package config // file or exceeds MaxFileBytes. // Constraints: a symlink is followed only to a regular file; a device (a link // to /dev/zero) or a FIFO is refused before it is opened, so a read can -// neither fill memory nor block forever. +// neither fill memory nor block forever. The path check is only an early +// exit: the file is opened without blocking and its type is checked again on +// the open descriptor, so a swap between the check and the open cannot hang. import ( "fmt" @@ -17,6 +19,10 @@ import ( // MaxFileBytes bounds a config or legacy manifest. const MaxFileBytes = 1 << 20 +// testHookBeforeOpen runs between the path check and the open; tests use it +// to swap the file, as a concurrent local writer could. +var testHookBeforeOpen = func(string) {} + func readRegular(path string) ([]byte, error) { info, err := os.Lstat(path) if err != nil { @@ -33,11 +39,17 @@ func readRegular(path string) ([]byte, error) { if !info.Mode().IsRegular() { return nil, fmt.Errorf("%s is not a regular file", path) } - f, err := os.Open(path) + testHookBeforeOpen(path) + f, err := openNonBlocking(path) if err != nil { return nil, err } defer func() { _ = f.Close() }() + if fi, err := f.Stat(); err != nil { + return nil, err + } else if !fi.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", path) + } data, err := io.ReadAll(io.LimitReader(f, MaxFileBytes+1)) if err != nil { return nil, err diff --git a/free/ci/internal/config/readfile_other.go b/free/ci/internal/config/readfile_other.go new file mode 100644 index 000000000..b46c433b1 --- /dev/null +++ b/free/ci/internal/config/readfile_other.go @@ -0,0 +1,8 @@ +//go:build !unix + +package config + +import "os" + +// openNonBlocking opens path read-only; this platform has no FIFOs to block on. +func openNonBlocking(path string) (*os.File, error) { return os.Open(path) } diff --git a/free/ci/internal/config/readfile_unix.go b/free/ci/internal/config/readfile_unix.go new file mode 100644 index 000000000..468bfdf3f --- /dev/null +++ b/free/ci/internal/config/readfile_unix.go @@ -0,0 +1,14 @@ +//go:build unix + +package config + +import ( + "os" + "syscall" +) + +// openNonBlocking opens path read-only without blocking on a FIFO and without +// taking a terminal as the controlling one. +func openNonBlocking(path string) (*os.File, error) { + return os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK|syscall.O_NOCTTY, 0) +} diff --git a/free/ci/internal/config/readfile_unix_test.go b/free/ci/internal/config/readfile_unix_test.go index eb0736f22..371d45f46 100644 --- a/free/ci/internal/config/readfile_unix_test.go +++ b/free/ci/internal/config/readfile_unix_test.go @@ -52,3 +52,32 @@ func TestMigrateDestinationFIFO(t *testing.T) { t.Fatal("Diff blocked on a FIFO") } } + +// A regular file swapped for a FIFO after the path check must be refused on +// the open descriptor, not hang the open. +func TestReadRegularFIFOSwappedAfterCheck(t *testing.T) { + path := filepath.Join(t.TempDir(), "ci.yaml") + if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil { + t.Fatal(err) + } + testHookBeforeOpen = func(p string) { + testHookBeforeOpen = func(string) {} + if err := os.Remove(p); err != nil { + t.Error(err) + } + if err := syscall.Mkfifo(p, 0o644); err != nil { + t.Error(err) + } + } + t.Cleanup(func() { testHookBeforeOpen = func(string) {} }) + done := make(chan error, 1) + go func() { _, err := readRegular(path); done <- err }() + select { + case err := <-done: + if err == nil { + t.Fatal("FIFO swapped in after the check was read") + } + case <-time.After(5 * time.Second): + t.Fatal("readRegular blocked on a FIFO swapped in after the check") + } +} diff --git "a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles\nnp_mediap_jobs\nnp_mediap_job_outputs\nnp_mediap_hls_manifests\nnp_mediap_subtitles\nnp_mediap_trickplay\nnp_mediap_webhook_events\nnp_mediap_watcher_events\nnp_mediap_uploads.yaml" b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles.yaml similarity index 83% rename from "free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles\nnp_mediap_jobs\nnp_mediap_job_outputs\nnp_mediap_hls_manifests\nnp_mediap_subtitles\nnp_mediap_trickplay\nnp_mediap_webhook_events\nnp_mediap_watcher_events\nnp_mediap_uploads.yaml" rename to free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles.yaml index fbce70e3e..5c91f67c4 100644 --- "a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles\nnp_mediap_jobs\nnp_mediap_job_outputs\nnp_mediap_hls_manifests\nnp_mediap_subtitles\nnp_mediap_trickplay\nnp_mediap_webhook_events\nnp_mediap_watcher_events\nnp_mediap_uploads.yaml" +++ b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_encoding_profiles.yaml @@ -1,14 +1,6 @@ - table: schema: public name: np_mediap_encoding_profiles -np_mediap_jobs -np_mediap_job_outputs -np_mediap_hls_manifests -np_mediap_subtitles -np_mediap_trickplay -np_mediap_webhook_events -np_mediap_watcher_events -np_mediap_uploads select_permissions: - role: user permission: diff --git a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_hls_manifests.yaml b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_hls_manifests.yaml new file mode 100644 index 000000000..5f53d1e6d --- /dev/null +++ b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_hls_manifests.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_mediap_hls_manifests + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_job_outputs.yaml b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_job_outputs.yaml new file mode 100644 index 000000000..5ee7784b3 --- /dev/null +++ b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_job_outputs.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_mediap_job_outputs + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_jobs.yaml b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_jobs.yaml new file mode 100644 index 000000000..e2796b41b --- /dev/null +++ b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_jobs.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_mediap_jobs + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_subtitles.yaml b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_subtitles.yaml new file mode 100644 index 000000000..b56b87bb3 --- /dev/null +++ b/free/media-processing/hasura/metadata/databases/default/tables/np_mediap_subtitles.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_mediap_subtitles + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/media-processing/hasura/metadata/databases/default/tables/tables.yaml b/free/media-processing/hasura/metadata/databases/default/tables/tables.yaml index 08f806529..60c6b8331 100644 --- a/free/media-processing/hasura/metadata/databases/default/tables/tables.yaml +++ b/free/media-processing/hasura/metadata/databases/default/tables/tables.yaml @@ -1,9 +1,5 @@ -- "!include np_mediap_encoding_profiles -np_mediap_jobs -np_mediap_job_outputs -np_mediap_hls_manifests -np_mediap_subtitles -np_mediap_trickplay -np_mediap_webhook_events -np_mediap_watcher_events -np_mediap_uploads.yaml" +- "!include np_mediap_encoding_profiles.yaml" +- "!include np_mediap_jobs.yaml" +- "!include np_mediap_job_outputs.yaml" +- "!include np_mediap_hls_manifests.yaml" +- "!include np_mediap_subtitles.yaml" diff --git a/free/nself-image/README.md b/free/nself-image/README.md index 6b0c5be84..8f9871271 100644 --- a/free/nself-image/README.md +++ b/free/nself-image/README.md @@ -8,7 +8,7 @@ Server-side image processing plugin for nSelf: resize, crop, format conversion ( - **Category:** media - **Tier:** pro -- **Language:** rust +- **Language:** go - **Port:** 3852 - **License:** MIT diff --git a/free/nself-image/SPEC.md b/free/nself-image/SPEC.md index c67c25e57..2d180748e 100644 --- a/free/nself-image/SPEC.md +++ b/free/nself-image/SPEC.md @@ -2,7 +2,7 @@ **Bundle:** nFamily **Tier:** $0.99/mo (nFamily bundle) or ɳSelf+ $3.99/mo -**Language:** Rust (perf-justified — see §1 Architecture) +**Language:** Go (see §1 Architecture) **Port:** 3827 **Target version:** v1.2.0 **Repo path:** `plugins-pro/paid/nself-image/` @@ -12,16 +12,15 @@ ## §1 Architecture -### Language Justification (Rust) +### Language (Go) -Per Plugin Language Policy, Rust is permitted for CPU-intensive / encoding / media-processing work where a measurable performance need exists. This plugin qualifies: +This plugin is written in Go, like the other nSelf plugin services. An earlier draft chose Rust on the grounds of a "Plugin Language Policy" and an unbenchmarked SIMD speed-up. No such policy document exists, and no benchmark backs the claim, so the draft is withdrawn. The draft's precedent (the `nclaw` `libs/libnclaw` Rust crate) is also not a policy: it is an orphaned crate. -- The `image` + `fast_image_resize` Rust crates use SIMD intrinsics for resize operations — 2-5x faster than Go's `golang.org/x/image` or Node Sharp in CPU-bound benchmarks. -- AVIF encoding via `ravif` crate avoids spawning external processes (libavif). -- WebP encoding via `libwebp-sys2` bindings — battle-tested, zero overhead vs Sharp's libwebp path. -- Single statically-linked binary output. No runtime libvips or libvips-dev dependency on the host. +Rust stays off the table until someone attaches a benchmark on CX21 hardware that shows Go cannot meet the targets in §8 and records an ADR 0031 exception. Until then: -This is the same exception class as the existing `nclaw/libs` Rust FFI crate: CPU-bound media work with documented SIMD gain. +- Resize and crop use the Go standard library plus `golang.org/x/image` (or a maintained resizing library chosen in the implementation sprint). +- WebP and AVIF encoders are the open risk: `golang.org/x/image` has no encoder for either. The implementation sprint picks between a cgo binding to libwebp/libavif and a pure-Go or WebAssembly encoder, and records the choice. Prefer `CGO_ENABLED=0` so the output stays one static binary. +- The performance targets in §8 are targets to measure, not claims. If Go misses them, that is the evidence for an ADR 0031 exception. ### Service Shape @@ -39,7 +38,7 @@ All traffic via Nginx reverse proxy when accessed from outside the host. No dire 1. Receive job via `POST /process`: `{input, output, ops[]}` 2. Validate HMAC token (`IMAGE_SHARED_SECRET`) — reject if invalid 3. Stream-download input object from MinIO in chunks (no full-file buffer for large inputs) -4. Decode image using `image` crate (JPEG, PNG, WebP, GIF, BMP, TIFF) +4. Decode the image with Go image decoders (JPEG, PNG, WebP, GIF, BMP, TIFF) 5. Apply ops in declared order: resize → crop → strip_exif → format_convert 6. Stream-upload output to MinIO output bucket 7. Optionally record audit row in `np_image.jobs` if `IMAGE_AUDIT_ENABLED=true` @@ -155,7 +154,7 @@ Returns `{"status":"ok","version":"0.1.0"}`. Used by `nself doctor`. | Op type | Params | Notes | |---|---|---| -| `resize` | `width`, `height`, `fit` (cover/contain/fill) | SIMD-accelerated via fast_image_resize | +| `resize` | `width`, `height`, `fit` (cover/contain/fill) | Library chosen in the implementation sprint (see §1) | | `crop` | `x`, `y`, `width`, `height` | Pixel-coordinate crop | | `format` | `to` (webp/avif/jpeg/png), `quality` (1-100) | AVIF capped at 1MP input by default — see open questions | | `strip_exif` | (no params) | Recommended for all user uploads; removes GPS, device info | @@ -193,7 +192,7 @@ For the `service` role, no row filter applies (full table access for internal au ```yaml name: nself-image version: 0.1.0 -language: rust +language: go bundle: nfamily visibility: public port: 3827 @@ -257,12 +256,12 @@ arch_support: ## §8 Testing Plan -### Unit tests (Rust, `cargo test`) +### Unit tests (Go, `go test`) - Each op type with known input (JPEG 4K) → assert output dimensions/format/size - `strip_exif`: assert GPS + device metadata removed from output - Oversized input → 413 error response -- Corrupt image (truncated) → graceful error, no panic +- Corrupt image (truncated) → graceful error, no panic or recovered panic - Unsupported format (MP4, PDF) → 400 error response - HMAC validation: valid token → pass; missing token → 401; tampered token → 401 @@ -274,6 +273,8 @@ arch_support: ### Performance baseline +Targets, not measured yet. A miss is the evidence an ADR 0031 exception needs (see §1). + - 256×256 WebP from 4K JPEG in < 100ms on CX21 (2vCPU, 4GB) - 1MP JPEG resize to 512×512 PNG in < 200ms on CX21 @@ -292,14 +293,14 @@ arch_support: **Why nself-image is different:** - **Self-hosted:** no per-transform cost, no vendor API key, no data leaving the server. Cloudinary charges per transformation; nself-image is flat-rate via the nFamily bundle ($0.99/mo). -- **Rust speed:** imgproxy (the closest open-source competitor) is Go; nself-image uses SIMD-accelerated Rust crates (`fast_image_resize`, `ravif`) measurably faster on CPU-bound workloads. +- **Single static binary:** no runtime libvips dependency on the host. No speed advantage over imgproxy (the closest open-source competitor, also Go) is claimed. - **nSelf-native:** directly integrated with MinIO (nSelf's storage layer) and Hasura Actions — zero glue code needed for nSelf apps vs Cloudinary's SDK requirement. --- ## §10 Rollout -1. Build Rust binary via CI cross-compilation matrix (see §14) +1. Build the Go binary via the CI cross-compilation matrix (see §14) 2. Package as Docker image `nself/plugin-nself-image:0.1.0` 3. Install: `nself plugin install nself-image` (requires nFamily bundle license) 4. Run `nself plugin migrate nself-image` to create `np_image` schema (migration is no-op if `IMAGE_AUDIT_ENABLED=false`) @@ -340,70 +341,49 @@ nself-image is part of the nFamily bundle. It also benefits nChat (attachment We --- -## §14 CI Matrix — Rust Cross-Compilation +## §14 CI Matrix — Go Cross-Compilation -Per SIEGE T2-15: nself-image requires a cross-compilation CI matrix because Rust must be compiled per target architecture. +Per SIEGE T2-15: nself-image ships per-architecture binaries, so CI builds a GOOS/GOARCH matrix. Go cross-compiles natively; no Docker-based cross toolchain is needed while the build stays `CGO_ENABLED=0` (see §1 for the encoder decision that could change that). ### Toolchain -- **Cross-compilation tool:** [cross-rs](https://github.com/cross-rs/cross) (Docker-based Rust cross-compiler) -- **Rust toolchain:** stable + beta + MSRV (see below) -- **MSRV:** 1.75.0 (required for `fast_image_resize` 3.x SIMD feature set) +- **Go toolchain:** the version pinned in `go.mod` (the repo standard); no beta or MSRV leg. +- **Build:** `CGO_ENABLED=0 GOOS= GOARCH= go build -mod=vendor -trimpath` ### Build matrix -| Target triple | OS | Arch | Tier | CI trigger | +| GOOS/GOARCH | OS | Arch | Tier | CI trigger | |---|---|---|---|---| -| `x86_64-unknown-linux-gnu` | Linux (Hetzner CX-series) | amd64 | Primary | PR + release | -| `aarch64-unknown-linux-gnu` | Linux (Hetzner CAX-series) | arm64 | Primary | PR + release | -| `x86_64-apple-darwin` | macOS (Intel) | amd64 | Secondary | Release only | -| `aarch64-apple-darwin` | macOS (Apple Silicon) | arm64 | Secondary | PR + release (local dev) | -| `x86_64-pc-windows-msvc` | Windows | amd64 | Future | Release only (when Windows support ships) | - -### Toolchain version matrix - -| Toolchain | Tests | Purpose | -|---|---|---| -| `stable` | Full test suite + clippy | Primary correctness gate | -| `beta` | Build + `cargo check` | Early warning for upcoming Rust breakage | -| `1.75.0` (MSRV) | Build + unit tests | Ensure we don't accidentally raise MSRV | +| `linux/amd64` | Linux (Hetzner CX-series) | amd64 | Primary | PR + release | +| `linux/arm64` | Linux (Hetzner CAX-series) | arm64 | Primary | PR + release | +| `darwin/amd64` | macOS (Intel) | amd64 | Secondary | Release only | +| `darwin/arm64` | macOS (Apple Silicon) | arm64 | Secondary | PR + release (local dev) | +| `windows/amd64` | Windows | amd64 | Future | Release only (when Windows support ships) | ### GitHub Actions workflow excerpt ```yaml strategy: matrix: - rust: [stable, beta, "1.75.0"] - target: - - x86_64-unknown-linux-gnu - - aarch64-unknown-linux-gnu - - aarch64-apple-darwin include: - - rust: stable - target: x86_64-pc-windows-msvc - os: windows-latest - exclude: - - rust: beta - target: x86_64-pc-windows-msvc - - rust: "1.75.0" - target: x86_64-pc-windows-msvc + - { goos: linux, goarch: amd64 } + - { goos: linux, goarch: arm64 } + - { goos: darwin, goarch: arm64 } steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master + - uses: actions/setup-go@v5 with: - toolchain: ${{ matrix.rust }} - targets: ${{ matrix.target }} - - uses: taiki-e/install-action@cross - - run: cross build --target ${{ matrix.target }} --release - - run: cross test --target ${{ matrix.target }} # Linux targets only + go-version-file: go.mod + - run: CGO_ENABLED=0 GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -mod=vendor -trimpath ./... + - run: go vet -mod=vendor ./... && go test -mod=vendor ./... # linux/amd64 leg only ``` -**Note:** Tests run only on Linux targets in CI (cross-rs Docker). Darwin builds compile-check only (no Docker-in-Docker for macOS targets). macOS tests run on local dev machines. +**Note:** Tests run on the native runner leg only. Other targets are compile-checked. ### Cross-compilation CI trigger rules -- **PR:** `x86_64-unknown-linux-gnu` (stable + MSRV) + `aarch64-unknown-linux-gnu` (stable) + `aarch64-apple-darwin` (stable, build-only) +- **PR:** `linux/amd64` + `linux/arm64` + `darwin/arm64` (build-only) - **Release (tag push):** Full matrix including Windows --- @@ -418,7 +398,7 @@ steps: - Hasura Actions custom-service pattern documented in `web/docs` **Pre-ship checklist:** -- [ ] Rust binary builds cleanly for `x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu` +- [ ] Go binary builds cleanly for `linux/amd64` and `linux/arm64` - [ ] All unit + integration tests pass - [ ] perf baseline verified on CX21 (256×256 WebP < 100ms) - [ ] `nself doctor` check passes on clean install diff --git a/free/nself-image/plugin.json b/free/nself-image/plugin.json index d92ee542f..f2117bff3 100644 --- a/free/nself-image/plugin.json +++ b/free/nself-image/plugin.json @@ -66,7 +66,7 @@ ], "health_endpoint": "/health", "homepage": "https://nself.org/plugins/nself-image", - "language": "rust", + "language": "go", "license": "free", "licenseType": "free", "license_spdx": "MIT", @@ -128,8 +128,6 @@ "avif", "format-conversion", "minio", - "rust", - "simd", "nfamily", "avatar", "photo" diff --git a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_categories.yaml b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_categories.yaml new file mode 100644 index 000000000..bfcc6d7dd --- /dev/null +++ b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_categories.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_podcast_categories + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_episodes.yaml b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_episodes.yaml new file mode 100644 index 000000000..840478d5c --- /dev/null +++ b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_episodes.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_podcast_episodes + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_playback_positions.yaml b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_playback_positions.yaml new file mode 100644 index 000000000..2c89e0649 --- /dev/null +++ b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_playback_positions.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_podcast_playback_positions + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git "a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts\nnp_podcast_episodes\nnp_podcast_subscriptions\nnp_podcast_playback_positions\nnp_podcast_categories.yaml" b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts.yaml similarity index 89% rename from "free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts\nnp_podcast_episodes\nnp_podcast_subscriptions\nnp_podcast_playback_positions\nnp_podcast_categories.yaml" rename to free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts.yaml index 677f4af3f..b68167a8f 100644 --- "a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts\nnp_podcast_episodes\nnp_podcast_subscriptions\nnp_podcast_playback_positions\nnp_podcast_categories.yaml" +++ b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_podcasts.yaml @@ -1,10 +1,6 @@ - table: schema: public name: np_podcast_podcasts -np_podcast_episodes -np_podcast_subscriptions -np_podcast_playback_positions -np_podcast_categories select_permissions: - role: user permission: diff --git a/free/podcast/hasura/metadata/databases/default/tables/np_podcast_subscriptions.yaml b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_subscriptions.yaml new file mode 100644 index 000000000..0f04bcf84 --- /dev/null +++ b/free/podcast/hasura/metadata/databases/default/tables/np_podcast_subscriptions.yaml @@ -0,0 +1,36 @@ +- table: + schema: public + name: np_podcast_subscriptions + select_permissions: + - role: user + permission: + columns: '*' + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + allow_aggregations: true + - role: nself_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + insert_permissions: + - role: user + permission: + columns: '*' + check: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + columns: '*' + check: {} + delete_permissions: + - role: user + permission: + filter: + source_account_id: + _eq: X-Hasura-Source-Account-Id + - role: nself_admin + permission: + filter: {} diff --git a/free/podcast/hasura/metadata/databases/default/tables/tables.yaml b/free/podcast/hasura/metadata/databases/default/tables/tables.yaml index 90dbb903a..bd7531111 100644 --- a/free/podcast/hasura/metadata/databases/default/tables/tables.yaml +++ b/free/podcast/hasura/metadata/databases/default/tables/tables.yaml @@ -1,5 +1,5 @@ -- "!include np_podcast_podcasts -np_podcast_episodes -np_podcast_subscriptions -np_podcast_playback_positions -np_podcast_categories.yaml" +- "!include np_podcast_podcasts.yaml" +- "!include np_podcast_episodes.yaml" +- "!include np_podcast_subscriptions.yaml" +- "!include np_podcast_playback_positions.yaml" +- "!include np_podcast_categories.yaml" diff --git a/registry.json b/registry.json index 0666903f7..5ddd5cfd7 100644 --- a/registry.json +++ b/registry.json @@ -5114,7 +5114,7 @@ ], "description": "Server-side image processing plugin for nSelf: resize, crop, format conversion (WebP/AVIF/JPEG/PNG), EXIF strip, and MinIO-integrated upload pipeline. Replaces per-app Sharp/Node.js usage across nFamily, nChat, and any consumer app needing image normalization.", "download_url": "https://plugins.nself.org/plugins/nself-image/tarball", - "language": "rust", + "language": "go", "license": "MIT", "minNselfVersion": "1.2.0", "min_nself_version": "1.2.0", @@ -5137,8 +5137,6 @@ "avif", "format-conversion", "minio", - "rust", - "simd", "nfamily", "avatar", "photo" diff --git a/scripts/conformance/fixtures.sh b/scripts/conformance/fixtures.sh index 0eb959d40..3ba62ade5 100644 --- a/scripts/conformance/fixtures.sh +++ b/scripts/conformance/fixtures.sh @@ -52,6 +52,8 @@ plant unvendored-build alpha build "rm -rf vendor" plant sock-in-named-fragment beta docker-socket "mkdir deploy; mv docker-compose.plugin.yml deploy/plugin.yml; jqe '.service.compose = \"deploy/plugin.yml\"'; sock deploy/plugin.yml" plant sock-in-other-yaml beta docker-socket "mkdir extra; printf 'services:\n x:\n volumes:\n - /var/run/docker.sock:/s\n' > extra/override.yaml" plant sock-var-run beta docker-socket "mkdir extra; printf 'services:\n x:\n volumes:\n - \"/var/run:/host-run\"\n' > extra/run.yml" +plant sock-in-newline-filename beta docker-socket "mkdir extra; printf 'services:\n x:\n volumes:\n - /var/run/docker.sock:/s\n' > 'extra/a +b.yaml'" # Rule port-bind (D-0453): every published port binds the literal 127.0.0.1. PLANT_MSG pins the failure to a real finding, # not a tool or parse error. PLANT_MSG='publishes|network_mode' diff --git a/scripts/conformance/rules/docker-socket.sh b/scripts/conformance/rules/docker-socket.sh index 08790eee6..d7d60b282 100644 --- a/scripts/conformance/rules/docker-socket.sh +++ b/scripts/conformance/rules/docker-socket.sh @@ -7,16 +7,14 @@ set -u . "$(dirname "$0")/../lib.sh" dir=$1; name=$(plugin_name "$dir"); exc=${2:-$CONF_DIR/exceptions.yaml}; rc=0 frag=$(jq -r '.service.compose // empty' "$dir/plugin.json" 2>/dev/null) -files=$( { [ -n "$frag" ] && [ -f "$dir/$frag" ] && printf '%s\n' "$dir/$frag" - find "$dir" \( -name vendor -o -name node_modules -o -name third_party \) -prune -o \ - \( -name '*.yml' -o -name '*.yaml' -o -name 'Dockerfile*' \) -type f -print; } | sort -u ) -while IFS= read -r f; do +# NUL-separated: a tracked file name may hold a newline (D-0628; port-bind has been NUL-safe since D-0453). +while IFS= read -r -d '' f; do [ -n "$f" ] || continue if grep -nE 'docker\.sock|/var/run/docker|(^|[[:space:]"'"'"'-])/var/run/?([:"'"'"'[:space:]]|$)' "$f" >/dev/null 2>&1; then if exception_active "$exc" docker-socket "$name"; then continue; fi fail docker-socket "$name" "${f#$dir/} mounts the Docker socket (ADR 0027) with no unexpired exception"; rc=1 fi -done <