diff --git a/CHANGELOG.md b/CHANGELOG.md index 6236cecd..811bf460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,14 @@ changes; the linked API and deployment documentation contains operational detail already signed with. See `docs/tag-spec.md` §10. ### Added +- Source-built `dbl-studio` authoring tool with a typed workflow compiler, + local visual editor integration and MCP tools. Workflows support network + actions, processing, conditions, bounded repetition and custom Go blocks. + See [workflow authoring](docs/development/workflow-authoring.md). +- Coordinated workflow experiments for two or three executors, with typed + send/await channels and retained connection manifests for the console's + measurement map. Peer messaging requires directly reachable IPv4 TCP + listeners and is plaintext; see [execution limits](docs/development/workflows.md). - The executor answers pre-disclosure tag verification queries over the control session (`VerifyTags`): for one group of captured packets of one epoch it reports which candidate run, if any, reproduces every tag, without diff --git a/Makefile b/Makefile index 21bc4b82..5195755a 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,9 @@ CI_RACE_PACKAGES = \ ./internal/executor/ratelimit/fallback \ ./internal/executor/scheduler/memory \ ./internal/executor/transport/rpc \ - ./internal/readiness + ./internal/readiness \ + ./internal/studio \ + ./pkg/workflow/guest CI_RACE_TIMEOUT ?= 5m CI_RACE_BUDGET_SECONDS ?= 240 CI_RACE_PACKAGE_PARALLEL ?= 2 diff --git a/NOTICE b/NOTICE index fd559ab0..5c15b249 100644 --- a/NOTICE +++ b/NOTICE @@ -16,6 +16,7 @@ github.com/google/uuid v1.6.0 - BSD-3-Clause github.com/hashicorp/yamux v0.1.2 - MPL-2.0 github.com/labstack/echo/v4 v4.15.4 - MIT github.com/maxmind/mmdbwriter v1.2.0 - Apache-2.0 OR MIT +github.com/modelcontextprotocol/go-sdk v1.7.0 - Apache-2.0 AND MIT (see module LICENSE) github.com/netsec-ethz/scion-apps v0.6.0 - Apache-2.0 github.com/oschwald/maxminddb-golang/v2 v2.6.0 - ISC github.com/pelletier/go-toml/v2 v2.2.4 - MIT @@ -25,6 +26,7 @@ github.com/soheilhy/cmux v0.1.5 - Apache-2.0 github.com/tetratelabs/wazero v1.12.0 - Apache-2.0 go.uber.org/zap v1.27.0 - MIT golang.org/x/crypto v0.57.0 - BSD-3-Clause +golang.org/x/net v0.60.0 - BSD-3-Clause golang.org/x/oauth2 v0.36.0 - BSD-3-Clause golang.org/x/sync v0.23.0 - BSD-3-Clause golang.org/x/sys v0.48.0 - BSD-3-Clause diff --git a/README.md b/README.md index e2d41754..612b1a8a 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ The versioned guides in [`docs/`](docs/README.md) are published at the [document | Use the CLI | [CLI reference](docs/cli.md) | | Write a Go application client | [Go client library](docs/client.md) | | Write a debuglet | [Write a debuglet](docs/debuglets.md) | +| Build visually or with an MCP client | [Workflow and experiment authoring](docs/development/workflow-authoring.md) | | Integrate with the HTTP API | [OpenAPI contract](api/openapi.yaml) and [API guide](docs/api.md) | | Understand the code and protocol | [Architecture](docs/architecture.md) | | Operate a service | [Managed services](docs/operations/services.md) | diff --git a/cmd/dbl-studio/main.go b/cmd/dbl-studio/main.go new file mode 100644 index 00000000..dde4b05a --- /dev/null +++ b/cmd/dbl-studio/main.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// Command dbl-studio edits and compiles visual measurement workflows locally. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/netsec-ethz/debuglet/internal/studio" + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "dbl-studio:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + return errors.New("usage: dbl-studio catalog|validate|build|validate-experiment|build-experiment|serve|mcp [options]") + } + command := args[0] + if command == "--help" || command == "-h" || command == "help" { + _, err := fmt.Fprintln(stdout, "usage: dbl-studio catalog|validate|build|validate-experiment|build-experiment|serve|mcp [options]\nUse -h for options.") + return err + } + flags := flag.NewFlagSet("dbl-studio "+command, flag.ContinueOnError) + flags.SetOutput(stderr) + sdk := flags.String("sdk", "", "trusted Debuglet source checkout (required for compilation)") + input := flags.String("input", "-", "workflow JSON file, or - for stdin") + output := flags.String("output", "-", "bundle JSON file, or - for stdout") + address := flags.String("listen", "127.0.0.1:8090", "loopback HTTP address") + ui := flags.String("ui", "", "built console directory containing index.html") + dispatcher := flags.String("dispatcher", "", "owned loopback development dispatcher origin for the console API proxy") + caFile := flags.String("ca-file", "", "additional PEM CA for the fixed dispatcher HTTPS connection") + workspace := flags.String("workspace", "", "directory for MCP workflow and bundle files") + if err := flags.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + if flags.NArg() != 0 { + return errors.New("unexpected positional arguments") + } + var builder *studio.Builder + if *sdk != "" { + var err error + builder, err = studio.NewBuilder(*sdk) + if err != nil { + return err + } + } + switch command { + case "catalog": + return json.NewEncoder(stdout).Encode(workflow.Catalog()) + case "serve": + return studio.Serve(ctx, builder, studio.ServerConfig{Address: *address, UI: *ui, Dispatcher: *dispatcher, CAFile: *caFile}, func(address string) { fmt.Fprintln(stderr, "Open", address) }) + case "mcp": + if *workspace == "" { + return errors.New("mcp requires --workspace for editable workflows and build bundles") + } + return studio.MCP(ctx, builder, *workspace) + case "validate", "build", "validate-experiment", "build-experiment": + var reader io.Reader = os.Stdin + if *input != "-" { + file, err := os.Open(*input) + if err != nil { + return err + } + defer file.Close() + reader = file + } + var result any + var err error + if command == "validate-experiment" || command == "build-experiment" { + experiment, decodeErr := studio.DecodeExperiment(reader) + if decodeErr != nil { + return decodeErr + } + if command == "validate-experiment" { + diagnostics := workflow.ValidateExperiment(experiment) + if diagnostics == nil { + diagnostics = []workflow.ExperimentDiagnostic{} + } + if err := json.NewEncoder(stdout).Encode(map[string]any{"diagnostics": diagnostics}); err != nil { + return err + } + if len(diagnostics) > 0 { + return errors.New("experiment is invalid") + } + return nil + } + if builder == nil { + return errors.New("build-experiment requires --sdk") + } + result, err = builder.BuildExperiment(ctx, experiment) + } else { + wf, decodeErr := studio.DecodeWorkflow(reader) + if decodeErr != nil { + return decodeErr + } + if command == "validate" { + diagnostics := workflow.Validate(wf) + if diagnostics == nil { + diagnostics = []workflow.Diagnostic{} + } + if err := json.NewEncoder(stdout).Encode(map[string]any{"diagnostics": diagnostics}); err != nil { + return err + } + if len(diagnostics) > 0 { + return errors.New("workflow is invalid") + } + return nil + } + if builder == nil { + return errors.New("build requires --sdk pointing to the matching Debuglet checkout") + } + result, err = builder.Build(ctx, wf) + } + if err != nil { + return err + } + var writer io.Writer = stdout + if *output != "-" { + file, err := os.OpenFile(*output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer file.Close() + writer = file + } + return json.NewEncoder(writer).Encode(result) + default: + return fmt.Errorf("unknown command %q (use --help for commands)", command) + } +} diff --git a/cmd/dbl-studio/main_test.go b/cmd/dbl-studio/main_test.go new file mode 100644 index 00000000..0e7406ca --- /dev/null +++ b/cmd/dbl-studio/main_test.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestValidateExperimentCommand(t *testing.T) { + for _, name := range []string{"three-node.json", "request-reply.json"} { + var output, diagnostics bytes.Buffer + input := filepath.Join("..", "..", "examples", "workflows", "experiments", name) + err := run(context.Background(), []string{"validate-experiment", "--input", input}, &output, &diagnostics) + if err != nil || output.String() != "{\"diagnostics\":[]}\n" { + t.Fatalf("%s: %v %s %s", name, err, &output, &diagnostics) + } + } +} diff --git a/docs/README.md b/docs/README.md index 1b870578..4712039d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,8 @@ This directory is the versioned source for the [Debuglet documentation site](htt - [Portable results](results.md) — export retained output and admission facts for offline analysis. - [Controlled latency experiment](research/latency-evaluation.md) — compare native and WASM probes under known local network faults. - [Write a debuglet](debuglets.md) — execution model and Go authoring interface. +- [Build workflows and experiments](development/workflow-authoring.md) — visual editing, local compilation and MCP authoring. +- [Workflow blocks and execution](development/workflows.md) — typed connections, reusable blocks and communication between executors. - [HTTP API](api.md) — public contract, authentication, compatibility, and deprecation. - [Versions and compatibility](versions.md) — tested combinations, support policy, and breaking changes. - [Linux client container](operations/client-container.md) — remote client with persistent credentials. diff --git a/docs/development/ci.md b/docs/development/ci.md index c8aeff3c..cebff49e 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -24,6 +24,9 @@ make ci-package Use `make ci-race` for concurrency changes. `make ci-local` exercises an installed package with local dispatcher and executor roles. `make ci-kernel` needs an isolated Linux host with the required kernel capabilities. +The race lane includes the local workflow authoring service and the guest +experiment transport, alongside the dispatcher and executor lifecycle packages. + CI checks generated code, formatting, tests, race-sensitive code, package installation, local roles, and kernel integration. A passing workflow validates the checked revision on its test platform; it does not establish production readiness or remote deployment compatibility. The workflow uses GitHub-hosted runners, read-only permissions, and no deployment secrets. See [CI images](ci-images.md) when changing the build environment. diff --git a/docs/development/workflow-authoring.md b/docs/development/workflow-authoring.md new file mode 100644 index 00000000..a50a198b --- /dev/null +++ b/docs/development/workflow-authoring.md @@ -0,0 +1,156 @@ +# Build workflows and experiments + +Choose **Workflow builder** to write one executor's program. Choose +**Experiment builder** to connect two or three participants, each with its own +workflow. The same graph can be edited visually, authored by an MCP client or +compiled from the command line with `dbl-studio`. + +Conditions, calculations and repeats execute inside the compiled WASI program. +The dispatcher coordinates experiment readiness and stores results; it does not +compile source or orchestrate individual blocks. See the +[workflow guide](workflows.md) for HTTP bodies, processing blocks, reusable +visual groups and custom Go blocks. + +`dbl-studio` is currently built from source; it is not part of the release or apt +packages. A hosted console can edit and export graphs and import compiled bundles. +Compilation needs Go and a trusted checkout of the matching SDK on the authoring +machine. The local service can combine editing and compilation in one browser tab. + +## Start the editor + +On Linux, install Go 1.26.9, clone this version of Debuglet, and build the +authoring tool from the repository root: + +```sh +go build -o dbl-studio ./cmd/dbl-studio +``` + +In the [dashboard repository](https://gitlab.inf.ethz.ch/OU-PERRIG/yimin/debuglet/debuglet-dashboard), +follow its **Run locally** prerequisites first: Node.js 24.14.0, `npm ci`, and +both template builds with their pinned Go versions. Then build the console with +its path explicitly set: + +```sh +VITE_BASE_PATH=/console npm run build +``` + +This sets both the application router and its asset URLs. A build for `/` will +not work under the studio's `/console/` route. Point the service at that build +and your owned development dispatcher origin: + +```sh +./dbl-studio serve \ + --sdk /path/to/debuglet \ + --ui /path/to/debuglet-dashboard/dist \ + --dispatcher https://127.0.0.1:8000 \ + --listen 127.0.0.1:8090 +``` + +Open the exact address printed by the command. The editor and its API use the same +loopback origin. `/api/` forwards to the one configured loopback dispatcher. +`--dispatcher` accepts a loopback IP or `localhost`, without an API path. For an owned +development dispatcher with local account sign-in enabled, this supports the +existing console login and submission flow. GitHub/CILogon sign-in through this +loopback proxy is not supported: provider callbacks and their cookies belong to +the hosted console's configured origin. For a provider-only dispatcher, build +locally, import the bundle into its hosted console, and sign in there normally. +The service never supplies login credentials or changes dispatcher policy. Use `--ca-file /path/to/dispatcher-ca.pem` when its HTTPS certificate is +signed by your own CA; certificate and hostname verification remain enabled. Do not +expose or tunnel this service to an untrusted network. + +Without `--sdk`, the service can show the editor and validate graphs but cannot +compile. Without `--dispatcher`, editing and compilation work, while sign-in and +submission are unavailable. `--ui` must contain the dashboard's built `index.html`. + +## Build and review a bundle + +Inspect the compiler's exact block catalog with `./dbl-studio catalog`. For +example, from the core repository: + +```sh +./dbl-studio validate --input examples/workflows/http-status.json +./dbl-studio build --sdk /path/to/debuglet \ + --input examples/workflows/http-status.json --output http-status.bundle.json +``` + +A bundle contains the editable graph, its canonical JSON, generated Go, compiled +WASM, destination hints and SHA-256 digests. Import it into the visual editor to +inspect the program, then choose executors and review the normal submission +policy. Editing the graph invalidates the compiled artifact and requires rebuilding. +A digest detects changes; it does not certify that a third-party WASM implements +its claimed source. Review the bundle's origin and rebuild unfamiliar source. + +Compilation uses a disposable module, the explicitly configured SDK checkout, +Go 1.26.9 and an offline build with a two-minute deadline. No graph can choose +build flags, fetch a dependency or run a generation command. One build runs at a +time. Workflows are limited to 1 MiB and bundles to 24 MiB. Install the toolchain +and prepare the trusted SDK before starting an offline authoring session. + +For a multi-executor experiment, use the experiment editor or these commands: + +```sh +./dbl-studio validate-experiment \ + --input examples/workflows/experiments/three-node.json +./dbl-studio build-experiment --sdk /path/to/debuglet \ + --input examples/workflows/experiments/three-node.json \ + --output three-node.experiment-bundle.json +``` + +The bundle includes the editable experiment, generated source and ordinary +WASM bundle for each participant. Import it into the experiment editor, bind +participants to distinct reachable executors, then choose **Review and run**. +Review the grouped measurement's policies, timing and TEST quote before +submitting. Compilation and import do not submit a measurement. +See [experiment communication](workflows.md#experiments-with-several-executors) +for listener requirements, message limits and request/reply semantics. + +The resulting **Measurement group** page shows a participant map. Select a node +to see its reported values; connection details and raw traces remain expandable. +New experiment programs include a connection manifest in their output, so the +map can load after a page reload. It describes declared communication, not +observed message delivery. For older runs, **More → Load experiment map** accepts +the compiled experiment bundle when every participant's program hash matches +the retained run. That imported map is kept only for the current page visit. + +## Connect an MCP client + +Configure a stdio MCP server in your client's settings: + +```json +{ + "mcpServers": { + "debuglet": { + "command": "/path/to/dbl-studio", + "args": [ + "mcp", "--sdk", "/path/to/debuglet", + "--workspace", "/path/to/my-workflows" + ] + } + } +} +``` + +Available tools: + +| Tool | Purpose | +| --- | --- | +| `catalog` | Inspect primitive blocks, typed ports and control outputs. | +| `validate` | Check a workflow using the compiler's validator. | +| `save_workflow` | Save the editable graph under a simple name in the workspace. | +| `get_workflow` | Read that named graph for further editing. | +| `build` | Compile the graph and return a bundle path and artifact metadata. | +| `validate_experiment` | Check participant graphs and their directed message channels. | +| `build_experiment` | Compile the whole experiment and return a portable bundle path. | +| `read_artifact` | Inspect a bundle created in the current session without returning WASM bytes. | + +Ask the client to compose a graph, validate it and save it. Open the saved workflow +or bundle in the visual editor, inspect its branches and source, and change it as +needed. The tools have no submission, payment or executor-management action. +They accept workflow names rather than arbitrary paths. A session retains up to +16 build artifact references; files remain in your chosen workspace after exit. + +Custom Go blocks remain visible function bodies with declared ports. They execute +with the enclosing guest's permissions and limits, not in an additional per-block +sandbox. Runtime destination permissions and operator network policy still apply; +compiler destination hints do not grant network access. See +[Workflow format](workflows.md) for supported blocks and control-flow rules. diff --git a/docs/development/workflows.md b/docs/development/workflows.md new file mode 100644 index 00000000..7b5e271d --- /dev/null +++ b/docs/development/workflows.md @@ -0,0 +1,201 @@ +# Visual measurement programs + +A workflow is a versioned JSON program that can be edited visually or through +an authoring tool. It compiles to one Go WASI debuglet. Network operations, +processing, branches and bounded repetition execute on the selected executor; +they do not require a round trip through an AI client or dispatcher. + +An experiment connects several of these workflows. In the console, +**Workflow builder** edits one program; **Experiment builder** selects a +participant, edits its program and connects it to the others. Start with the +[authoring guide](workflow-authoring.md) to build the local editor, compile +bundles or connect an MCP client. + +Start with [HTTP status branching](../../examples/workflows/http-status.json), +[Ping and RTT analysis](../../examples/workflows/ping-branch.json), +[posting JSON](../../examples/workflows/http-json-post.json), or +[a custom processing block](../../examples/workflows/custom-transform.json). +The network examples use loopback placeholders: choose an endpoint you control +and permit its address in the normal submission policy. The JSON POST example +expects its `/echo` target to return JSON. On a remote executor, +loopback refers to that executor. Ping also requires its ICMP capability. + +## Blocks and connections + +Blocks have named, typed inputs and outputs. An input receives either a literal +or one data connection. Types are strings, finite numbers, booleans, number +lists, string lists, objects and arbitrary JSON values. An `any` connection is +checked against the receiving type when used; strings are never automatically +converted into numbers or booleans. + +Data connections carry values. Control connections determine when actions run. +A data connection to an HTTP response does not send another request. Reading an +action that has not run on the current control path is an error. Pure analysis +blocks evaluate when their value is needed and can appear more than once in the +trace. An If block receives a boolean from a comparison or other producer and +runs its `true` or `false` control branch. A second target uses another ordinary +network block with different inputs. + +HTTP's `ok` indicates a completed response, including 3xx, 4xx and 5xx statuses. +Redirects are not followed. The optional `body` is text, and `headers` is an +object whose values are strings: for example `{"Content-Type":"application/json"}`. +Connect Stringify JSON to `body` to send a structured value. The HTTP block sets +Content-Length from the body's byte length and keeps the URL's host as the +request authority. Host, framing, connection, compression and upgrade headers +cannot be overridden. Request bodies are at most 16 KiB; caller-supplied request headers and received +response headers are limited to 16 KiB and 64 names. Response `headers` uses lowercase +names and lists of strings, preserving repeated values such as Set-Cookie. + +Check `ok` for transport errors and compare `status` +to express the status policy you need. Ping's `rtts_ms` contains only successful +replies and `loss` is a percentage. Check `received` before calculating a mean; +analysis of an empty sample set fails rather than reporting a misleading zero. + +Data blocks support Parse JSON and Stringify JSON, Get field and Set field, +List item and Append item. Set field and Append item return new values without +changing their inputs. List item uses a zero-based integer index and fails when +the item is absent. Text blocks offer Concatenate, Contains, Split and Join. +Contains is case-sensitive; an empty substring matches. Split requires a +nonempty separator and preserves empty pieces. Join of an empty list is empty +text. All produced values retain the normal string and list limits. + +Mean, Minimum, Maximum, Median and Percentile require a nonempty number list. +Median averages the middle pair for even-sized lists. Percentile accepts a value +from 0 to 100 and linearly interpolates sorted samples at `p/100 × (n−1)`; this is +not the nearest-rank convention. Add, Subtract, Multiply and Divide use finite +numbers; division by zero and nonfinite results fail explicitly. + +## Reusable visual groups + +The editor can fold a selection into one named block. Its interface comes from +connections crossing the selection and its unconnected ports. Double-click +the group to edit its members, then close it to return to the compact view. +Saved groups are templates: inserting one copies its nodes, internal wires and +custom block definitions, so editing an instance does not change other instances. + +The portable workflow keeps these ordinary nodes and wires, plus optional +`groups` metadata containing `id`, `label`, `node_ids` and `position`. Grouping +does not introduce a function call or change execution order. Metadata is retained +in compiled bundles and included in the workflow checksum. Groups currently +contain at least two nodes and cannot overlap or nest; ungrouping preserves the +underlying program. + +## Experiments with several executors + +An experiment contains two or three ordinary participant workflows and named, +directed channels. Participant names are labels, not fixed roles. Choose a +different executor for each participant when reviewing the grouped submission. +Every executor must advertise a directly reachable IPv4 TCP listener and permit +the other participants' addresses in its measurement policy. Channel names are +chosen by the author to identify messages, such as `reading` or `reply`; they +are not built-in operations. + +The [three-node example](../../examples/workflows/experiments/three-node.json) +sends 21 from A to B, doubles it on B, adds one on C and returns 43 to A. +Each participant first publishes its listener through the existing readiness +exchange and waits for the common requested start. This is a clock-dependent +start time, not atomic simultaneous execution. The dispatcher coordinates setup; +the messages then travel directly between executors. + +Send message selects a destination and channel, and supplies a typed JSON value. +Await message selects the source and channel. Both accept a bounded `timeout_ms` +and return `ok` and `error`. Test `ok` before using an Await's `value`; failure +returns `null`, not a successful zero or empty result. Send success means the +socket write completed, not that the receiver processed it. + +A channel with `response_type` permits one reply per request. The +[request/reply example](../../examples/workflows/experiments/request-reply.json) +wires the initial Send's `request_id` into the sender's Await, and the receiver's +Await `request_id` into its replying Send. A late reply cannot satisfy a different +request. One-way channels leave `request_id` empty. There are no automatic +retries, participant replacement or repeated global barriers. + +Messages are plaintext TCP, limited to 8 KiB including framing fields; the inbox +is limited to 64 KiB and 128 messages. Group identifiers and request correlation +prevent accidental mixing, but do not authenticate a peer socket or encrypt its +data. Use controlled experiments without confidential payloads. A participant +only receives while executing Await: long-running blocks can delay its peers, +and cyclic waits can time out. Normal executor lifetime and traffic limits remain +authoritative. + +The experiment source is at most 4 MiB, with at most 64 channels. Compilation +injects the participant ID, whole-experiment fingerprint and channel manifest +into each program's canonical workflow and digest. Build the whole experiment +after editing any participant; individual compiled programs cannot be rebuilt +as standalone workflows. Bundles are limited to 32 MiB, and the console checks +the final grouped request against the API's 32 MiB limit before submission. + +## Repetition + +Repeat has a `body` and a `done` output. The body runs sequentially until its +control path ends, then starts again. Its action outputs are cleared between +iterations. Repeat's `iteration` output starts at 1. After the requested count, +execution continues at `done`. A body cannot also lead into that continuation +or be entered by another control path. Arbitrary cycles are rejected. + +Programs support up to 128 nodes, 100 repeat iterations and 1,000 action steps. +A single wait or network timeout is at most 60 seconds. Graph control limits +complement the ordinary executor's runtime, traffic and destination limits. + +## Custom blocks + +A custom block declares its inputs, outputs and a Go function body. It receives +`inputs map[string]any` and returns `(map[string]any, error)`. JSON numbers are +`float64`, arrays are `[]any` and objects are `map[string]any`. Returned Go values +are normalized to JSON and checked against the declared ports. Custom blocks +are actions with a `next` control output; they execute only when control reaches +them. Their complete source remains visible in the workflow. + +The generated program makes `fmt`, `json`, `os` and `guest` available. Referring +to `debuglet`, `time`, `strings` or `math` selects those approved packages too. +`debuglet` is the existing guest SDK, and `guest` exposes the built-in operations +such as `HTTP` and `Ping`. Arbitrary imports, additional source files, compiler +directives and dependency downloads are not accepted. For example: + +```go +samples := inputs["samples"].([]any) +if len(samples) == 0 { + return nil, fmt.Errorf("need at least one sample") +} +var total float64 +for _, sample := range samples { + total += sample.(float64) +} +return map[string]any{"value": total / float64(len(samples))}, nil +``` + +Custom code shares the guest's resource and network policy. It is not isolated +per block and can loop or use the SDK directly; the executor's overall runtime +limit remains necessary. Typed source validation does not prove arbitrary code +safe or establish that a supplied Wasm artifact matches that source. + +## Results and artifact identity + +The guest emits JSON lines with schema `debuglet.workflow.v1`, +`workflow_sha256`, `node_id`, `block`, `status`, `outputs` and optional `error`. +`completed` means the block ran: a network operation can complete with +`outputs.ok: false`. A fatal graph/type/custom-code error emits a `failed` event +and ends the run. An Emit block publishes `name` and `value` in its outputs. + +An experiment program first emits one `debuglet.experiment.v1` JSON line, +before joining its peers. This bounded manifest contains its workflow and +experiment digests, participant ID, ordered membership and declared channels. +The console uses agreeing manifests from the group to restore its connection +map. If output is missing or the manifests disagree, the automatic map is +unavailable. These are program declarations, not evidence that messages were +delivered. +Consumers of block events should filter by the `debuglet.workflow.v1` schema. + +Individual strings are limited to 16 KiB and lists to 1,024 entries. Each event +is at most 64 KiB; a run emits at most 4,096 events and 4 MiB of trace output. +The experiment manifest is an additional record of at most 64 KiB. +Large responses fail explicitly rather than silently producing complete-looking +measurements. + +The workflow digest hashes its canonical Go JSON encoding, including layout. +A changed graph must be rebuilt. Compiler destination hints cover literal +network targets and directly connected Constant blocks. Computed or custom +network targets require the submitter to supply the intended destination policy +and enable ICMP when their custom code uses it; +the hints never grant network access. The normal dispatcher and executor checks +apply to every compiled workflow just as they do to hand-written debuglets. diff --git a/examples/workflows/catalog.json b/examples/workflows/catalog.json new file mode 100644 index 00000000..feb26075 --- /dev/null +++ b/examples/workflows/catalog.json @@ -0,0 +1,1014 @@ +[ + { + "id": "http", + "label": "HTTP request", + "category": "Network", + "description": "Send one HTTP request. ok describes transport completion; compare status to choose an HTTP success policy. Body is text; request headers are string values. Response headers use lowercase names and lists of values. Redirects are returned, not followed.", + "pure": false, + "inputs": [ + { + "name": "url", + "type": "string", + "required": true + }, + { + "name": "method", + "type": "string", + "default": "GET" + }, + { + "name": "body", + "type": "string", + "default": "" + }, + { + "name": "headers", + "type": "object", + "default": {} + }, + { + "name": "timeout_ms", + "type": "number", + "default": 2000 + } + ], + "outputs": [ + { + "name": "ok", + "type": "boolean" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "status", + "type": "number" + }, + { + "name": "body", + "type": "string" + }, + { + "name": "headers", + "type": "object" + }, + { + "name": "duration_ms", + "type": "number" + } + ], + "control_outputs": [ + "next" + ] + }, + { + "id": "tcp_connect", + "label": "TCP connect", + "category": "Network", + "description": "Measure one TCP connection to host:port.", + "pure": false, + "inputs": [ + { + "name": "target", + "type": "string", + "required": true + }, + { + "name": "timeout_ms", + "type": "number", + "default": 2000 + } + ], + "outputs": [ + { + "name": "ok", + "type": "boolean" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "duration_ms", + "type": "number" + } + ], + "control_outputs": [ + "next" + ] + }, + { + "id": "ping", + "label": "Ping", + "category": "Network", + "description": "Send up to ten ICMPv4 echo requests. rtts_ms contains successful replies; loss is a percentage.", + "pure": false, + "inputs": [ + { + "name": "target", + "type": "string", + "required": true + }, + { + "name": "count", + "type": "number", + "default": 3 + }, + { + "name": "timeout_ms", + "type": "number", + "default": 2000 + } + ], + "outputs": [ + { + "name": "ok", + "type": "boolean" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "rtts_ms", + "type": "numbers" + }, + { + "name": "sent", + "type": "number" + }, + { + "name": "received", + "type": "number" + }, + { + "name": "loss", + "type": "number" + } + ], + "control_outputs": [ + "next" + ] + }, + { + "id": "send_message", + "label": "Send message", + "category": "Experiment", + "description": "Send a bounded plaintext JSON message to a participant. Success means the socket write completed, not that the peer processed it. Requires an experiment.", + "pure": false, + "inputs": [ + { + "name": "destination", + "type": "string", + "required": true + }, + { + "name": "channel", + "type": "string", + "required": true + }, + { + "name": "value", + "type": "any", + "required": true + }, + { + "name": "request_id", + "type": "string", + "default": "" + }, + { + "name": "timeout_ms", + "type": "number", + "default": 2000 + } + ], + "outputs": [ + { + "name": "ok", + "type": "boolean" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "request_id", + "type": "string" + } + ], + "control_outputs": [ + "next" + ] + }, + { + "id": "await_message", + "label": "Await message", + "category": "Experiment", + "description": "Wait for one selected incoming message or reply until its deadline. Test ok before using value. Requires an experiment.", + "pure": false, + "inputs": [ + { + "name": "source", + "type": "string", + "required": true + }, + { + "name": "channel", + "type": "string", + "required": true + }, + { + "name": "request_id", + "type": "string", + "default": "" + }, + { + "name": "timeout_ms", + "type": "number", + "default": 2000 + } + ], + "outputs": [ + { + "name": "ok", + "type": "boolean" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "value", + "type": "any" + }, + { + "name": "request_id", + "type": "string" + }, + { + "name": "source", + "type": "string" + } + ], + "control_outputs": [ + "next" + ] + }, + { + "id": "constant", + "label": "Constant", + "category": "Data", + "description": "Provide a JSON value.", + "pure": true, + "inputs": [ + { + "name": "value", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [] + }, + { + "id": "field", + "label": "Get field", + "category": "Data", + "description": "Read one named field from an object; missing fields are errors.", + "pure": true, + "inputs": [ + { + "name": "object", + "type": "object", + "required": true + }, + { + "name": "name", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [] + }, + { + "id": "parse_json", + "label": "Parse JSON", + "category": "Data", + "description": "Parse a JSON string into a value.", + "pure": true, + "inputs": [ + { + "name": "text", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [] + }, + { + "id": "stringify_json", + "label": "Stringify JSON", + "category": "Data", + "description": "Encode a JSON value as text, for example an HTTP request body.", + "pure": true, + "inputs": [ + { + "name": "value", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "string" + } + ], + "control_outputs": [] + }, + { + "id": "set_field", + "label": "Set field", + "category": "Data", + "description": "Return a copy of an object with one field added or replaced.", + "pure": true, + "inputs": [ + { + "name": "object", + "type": "object", + "required": true + }, + { + "name": "name", + "type": "string", + "required": true + }, + { + "name": "value", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "object" + } + ], + "control_outputs": [] + }, + { + "id": "list_item", + "label": "List item", + "category": "Data", + "description": "Read one list item at a zero-based integer index. An out-of-range index is an error.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "any", + "required": true + }, + { + "name": "index", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [] + }, + { + "id": "append", + "label": "Append item", + "category": "Data", + "description": "Return a new list with one item appended.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "any", + "required": true + }, + { + "name": "value", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [] + }, + { + "id": "concat", + "label": "Concatenate", + "category": "Text", + "description": "Join two strings without adding a separator.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "string", + "required": true + }, + { + "name": "b", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "string" + } + ], + "control_outputs": [] + }, + { + "id": "contains", + "label": "Contains", + "category": "Text", + "description": "Case-sensitive substring search. An empty substring matches every string.", + "pure": true, + "inputs": [ + { + "name": "text", + "type": "string", + "required": true + }, + { + "name": "substring", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "split", + "label": "Split", + "category": "Text", + "description": "Split text at each occurrence of a nonempty separator, keeping empty pieces.", + "pure": true, + "inputs": [ + { + "name": "text", + "type": "string", + "required": true + }, + { + "name": "separator", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "strings" + } + ], + "control_outputs": [] + }, + { + "id": "join", + "label": "Join", + "category": "Text", + "description": "Join a list of strings with a separator. An empty list produces empty text.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "strings", + "required": true + }, + { + "name": "separator", + "type": "string", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "string" + } + ], + "control_outputs": [] + }, + { + "id": "median", + "label": "Median", + "category": "Analysis", + "description": "Median of a nonempty number list; an even-sized list uses the mean of its middle pair.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "numbers", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "percentile", + "label": "Percentile", + "category": "Analysis", + "description": "Percentile from 0 to 100 of a nonempty number list, with linear interpolation at rank p/100 × (n−1).", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "numbers", + "required": true + }, + { + "name": "percentile", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "mean", + "label": "Mean", + "category": "Analysis", + "description": "Arithmetic mean of a nonempty number list.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "numbers", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "min", + "label": "Minimum", + "category": "Analysis", + "description": "Smallest value in a nonempty number list.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "numbers", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "max", + "label": "Maximum", + "category": "Analysis", + "description": "Largest value in a nonempty number list.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "numbers", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "count", + "label": "Count", + "category": "Analysis", + "description": "Number of items in a JSON list.", + "pure": true, + "inputs": [ + { + "name": "values", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "add", + "label": "Add", + "category": "Math", + "description": "Add two numbers.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "subtract", + "label": "Subtract", + "category": "Math", + "description": "Subtract b from a.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "multiply", + "label": "Multiply", + "category": "Math", + "description": "Multiply two numbers. Nonfinite results are errors.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "divide", + "label": "Divide", + "category": "Math", + "description": "Divide a by b. Division by zero and nonfinite results are errors.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "number" + } + ], + "control_outputs": [] + }, + { + "id": "greater_than", + "label": "Greater than", + "category": "Logic", + "description": "Whether a is greater than b.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "less_than", + "label": "Less than", + "category": "Logic", + "description": "Whether a is less than b.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "number", + "required": true + }, + { + "name": "b", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "equals", + "label": "Equals", + "category": "Logic", + "description": "Compare JSON values without coercion.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "any", + "required": true + }, + { + "name": "b", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "and", + "label": "And", + "category": "Logic", + "description": "Both conditions are true.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "boolean", + "required": true + }, + { + "name": "b", + "type": "boolean", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "or", + "label": "Or", + "category": "Logic", + "description": "At least one condition is true.", + "pure": true, + "inputs": [ + { + "name": "a", + "type": "boolean", + "required": true + }, + { + "name": "b", + "type": "boolean", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "not", + "label": "Not", + "category": "Logic", + "description": "Invert a condition.", + "pure": true, + "inputs": [ + { + "name": "value", + "type": "boolean", + "required": true + } + ], + "outputs": [ + { + "name": "value", + "type": "boolean" + } + ], + "control_outputs": [] + }, + { + "id": "if", + "label": "If", + "category": "Control", + "description": "Run exactly the selected branch.", + "pure": false, + "inputs": [ + { + "name": "condition", + "type": "boolean", + "required": true + } + ], + "outputs": [], + "control_outputs": [ + "true", + "false" + ] + }, + { + "id": "repeat", + "label": "Repeat", + "category": "Control", + "description": "Run the body sequentially up to 100 times, then continue on done. The body ends at an unconnected control output.", + "pure": false, + "inputs": [ + { + "name": "count", + "type": "number", + "required": true + } + ], + "outputs": [ + { + "name": "iteration", + "type": "number" + } + ], + "control_outputs": [ + "body", + "done" + ] + }, + { + "id": "wait", + "label": "Wait", + "category": "Control", + "description": "Wait between actions, at most 60 seconds per call.", + "pure": false, + "inputs": [ + { + "name": "duration_ms", + "type": "number", + "required": true + } + ], + "outputs": [], + "control_outputs": [ + "next" + ] + }, + { + "id": "emit", + "label": "Emit result", + "category": "Results", + "description": "Publish a named JSON result.", + "pure": false, + "inputs": [ + { + "name": "name", + "type": "string", + "required": true + }, + { + "name": "value", + "type": "any", + "required": true + } + ], + "outputs": [ + { + "name": "name", + "type": "string" + }, + { + "name": "value", + "type": "any" + } + ], + "control_outputs": [ + "next" + ] + } +] diff --git a/examples/workflows/custom-transform.json b/examples/workflows/custom-transform.json new file mode 100644 index 00000000..1646023a --- /dev/null +++ b/examples/workflows/custom-transform.json @@ -0,0 +1,14 @@ +{ + "version":1, + "name":"Custom processing block", + "entry":"transform", + "nodes":[ + {"id":"transform","block":"spread","inputs":{"samples":[12,15,18]},"position":{"x":100,"y":100}}, + {"id":"result","block":"emit","inputs":{"name":"spread_ms"},"position":{"x":420,"y":100}} + ], + "edges":[ + {"kind":"control","from":{"node":"transform","port":"next"},"to":{"node":"result","port":"in"}}, + {"kind":"data","from":{"node":"transform","port":"spread"},"to":{"node":"result","port":"value"}} + ], + "custom_blocks":[{"id":"spread","label":"Sample spread","inputs":[{"name":"samples","type":"numbers","required":true}],"outputs":[{"name":"spread","type":"number"}],"code":"samples := inputs[\"samples\"].([]any)\nif len(samples) == 0 { return nil, fmt.Errorf(\"need at least one sample\") }\nlo, hi := samples[0].(float64), samples[0].(float64)\nfor _, sample := range samples {\n n := sample.(float64)\n if n < lo { lo = n }; if n > hi { hi = n }\n}\nreturn map[string]any{\"spread\": hi-lo}, nil"}] +} diff --git a/examples/workflows/experiments/request-reply.json b/examples/workflows/experiments/request-reply.json new file mode 100644 index 00000000..06d1ca5e --- /dev/null +++ b/examples/workflows/experiments/request-reply.json @@ -0,0 +1,69 @@ +{ + "version":1,"kind":"experiment","name":"Correlated request and reply", + "participants":[ + { + "id":"A","label":"Start", + "workflow":{ + "version":1,"name":"Send a request and await its reply","entry":"send", + "nodes":[ + {"id":"send","block":"send_message","inputs":{"destination":"B","channel":"request","value":21,"timeout_ms":5000}}, + {"id":"sent","block":"if","inputs":{}}, + {"id":"await","block":"await_message","inputs":{"source":"B","channel":"request","timeout_ms":8000}}, + {"id":"received","block":"if","inputs":{}}, + {"id":"result","block":"emit","inputs":{"name":"answer"}}, + {"id":"send_error","block":"emit","inputs":{"name":"send_error"}}, + {"id":"receive_error","block":"emit","inputs":{"name":"receive_error"}} + ], + "edges":[ + {"kind":"control","from":{"node":"send","port":"next"},"to":{"node":"sent","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"ok"},"to":{"node":"sent","port":"condition"}}, + {"kind":"control","from":{"node":"sent","port":"true"},"to":{"node":"await","port":"in"}}, + {"kind":"control","from":{"node":"sent","port":"false"},"to":{"node":"send_error","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"error"},"to":{"node":"send_error","port":"value"}}, + {"kind":"control","from":{"node":"await","port":"next"},"to":{"node":"received","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"ok"},"to":{"node":"received","port":"condition"}}, + {"kind":"control","from":{"node":"received","port":"true"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"received","port":"false"},"to":{"node":"receive_error","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"value"},"to":{"node":"result","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"error"},"to":{"node":"receive_error","port":"value"}}, + {"kind":"data","from":{"node":"send","port":"request_id"},"to":{"node":"await","port":"request_id"}} + ] + } + }, + { + "id":"B","label":"Double", + "workflow":{ + "version":1,"name":"Double the value and reply","entry":"await", + "nodes":[ + {"id":"await","block":"await_message","inputs":{"source":"A","channel":"request","timeout_ms":8000}}, + {"id":"received","block":"if","inputs":{}}, + {"id":"calculate","block":"multiply","inputs":{"b":2}}, + {"id":"send","block":"send_message","inputs":{"destination":"A","channel":"request","timeout_ms":5000}}, + {"id":"sent","block":"if","inputs":{}}, + {"id":"result","block":"emit","inputs":{"name":"sent_value"}}, + {"id":"send_error","block":"emit","inputs":{"name":"send_error"}}, + {"id":"receive_error","block":"emit","inputs":{"name":"receive_error"}} + ], + "edges":[ + {"kind":"control","from":{"node":"await","port":"next"},"to":{"node":"received","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"ok"},"to":{"node":"received","port":"condition"}}, + {"kind":"control","from":{"node":"received","port":"true"},"to":{"node":"send","port":"in"}}, + {"kind":"control","from":{"node":"received","port":"false"},"to":{"node":"receive_error","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"error"},"to":{"node":"receive_error","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"value"},"to":{"node":"calculate","port":"a"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"send","port":"value"}}, + {"kind":"control","from":{"node":"send","port":"next"},"to":{"node":"sent","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"ok"},"to":{"node":"sent","port":"condition"}}, + {"kind":"control","from":{"node":"sent","port":"true"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"sent","port":"false"},"to":{"node":"send_error","port":"in"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"result","port":"value"}}, + {"kind":"data","from":{"node":"send","port":"error"},"to":{"node":"send_error","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"request_id"},"to":{"node":"send","port":"request_id"}} + ] + } + } + ], + "channels":[ + {"id":"request","from":"A","to":"B","type":"number","response_type":"number"} + ] +} diff --git a/examples/workflows/experiments/three-node.json b/examples/workflows/experiments/three-node.json new file mode 100644 index 00000000..ce89e959 --- /dev/null +++ b/examples/workflows/experiments/three-node.json @@ -0,0 +1,100 @@ +{ + "version":1,"kind":"experiment","name":"Three-node calculation", + "participants":[ + { + "id":"A","label":"Start", + "workflow":{ + "version":1,"name":"Send a value and await its result","entry":"send", + "nodes":[ + {"id":"send","block":"send_message","inputs":{"destination":"B","channel":"a_to_b","value":21,"timeout_ms":5000}}, + {"id":"sent","block":"if","inputs":{}}, + {"id":"await","block":"await_message","inputs":{"source":"C","channel":"c_to_a","timeout_ms":8000}}, + {"id":"received","block":"if","inputs":{}}, + {"id":"result","block":"emit","inputs":{"name":"answer"}}, + {"id":"send_error","block":"emit","inputs":{"name":"send_error"}}, + {"id":"receive_error","block":"emit","inputs":{"name":"receive_error"}} + ], + "edges":[ + {"kind":"control","from":{"node":"send","port":"next"},"to":{"node":"sent","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"ok"},"to":{"node":"sent","port":"condition"}}, + {"kind":"control","from":{"node":"sent","port":"true"},"to":{"node":"await","port":"in"}}, + {"kind":"control","from":{"node":"sent","port":"false"},"to":{"node":"send_error","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"error"},"to":{"node":"send_error","port":"value"}}, + {"kind":"control","from":{"node":"await","port":"next"},"to":{"node":"received","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"ok"},"to":{"node":"received","port":"condition"}}, + {"kind":"control","from":{"node":"received","port":"true"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"received","port":"false"},"to":{"node":"receive_error","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"value"},"to":{"node":"result","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"error"},"to":{"node":"receive_error","port":"value"}} + ] + } + }, + { + "id":"B","label":"Double", + "workflow":{ + "version":1,"name":"Double the value","entry":"await", + "nodes":[ + {"id":"await","block":"await_message","inputs":{"source":"A","channel":"a_to_b","timeout_ms":8000}}, + {"id":"received","block":"if","inputs":{}}, + {"id":"calculate","block":"multiply","inputs":{"b":2}}, + {"id":"send","block":"send_message","inputs":{"destination":"C","channel":"b_to_c","timeout_ms":5000}}, + {"id":"sent","block":"if","inputs":{}}, + {"id":"result","block":"emit","inputs":{"name":"sent_value"}}, + {"id":"send_error","block":"emit","inputs":{"name":"send_error"}}, + {"id":"receive_error","block":"emit","inputs":{"name":"receive_error"}} + ], + "edges":[ + {"kind":"control","from":{"node":"await","port":"next"},"to":{"node":"received","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"ok"},"to":{"node":"received","port":"condition"}}, + {"kind":"control","from":{"node":"received","port":"true"},"to":{"node":"send","port":"in"}}, + {"kind":"control","from":{"node":"received","port":"false"},"to":{"node":"receive_error","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"error"},"to":{"node":"receive_error","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"value"},"to":{"node":"calculate","port":"a"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"send","port":"value"}}, + {"kind":"control","from":{"node":"send","port":"next"},"to":{"node":"sent","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"ok"},"to":{"node":"sent","port":"condition"}}, + {"kind":"control","from":{"node":"sent","port":"true"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"sent","port":"false"},"to":{"node":"send_error","port":"in"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"result","port":"value"}}, + {"kind":"data","from":{"node":"send","port":"error"},"to":{"node":"send_error","port":"value"}} + ] + } + }, + { + "id":"C","label":"Add one", + "workflow":{ + "version":1,"name":"Add one","entry":"await", + "nodes":[ + {"id":"await","block":"await_message","inputs":{"source":"B","channel":"b_to_c","timeout_ms":8000}}, + {"id":"received","block":"if","inputs":{}}, + {"id":"calculate","block":"add","inputs":{"b":1}}, + {"id":"send","block":"send_message","inputs":{"destination":"A","channel":"c_to_a","timeout_ms":5000}}, + {"id":"sent","block":"if","inputs":{}}, + {"id":"result","block":"emit","inputs":{"name":"sent_value"}}, + {"id":"send_error","block":"emit","inputs":{"name":"send_error"}}, + {"id":"receive_error","block":"emit","inputs":{"name":"receive_error"}} + ], + "edges":[ + {"kind":"control","from":{"node":"await","port":"next"},"to":{"node":"received","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"ok"},"to":{"node":"received","port":"condition"}}, + {"kind":"control","from":{"node":"received","port":"true"},"to":{"node":"send","port":"in"}}, + {"kind":"control","from":{"node":"received","port":"false"},"to":{"node":"receive_error","port":"in"}}, + {"kind":"data","from":{"node":"await","port":"error"},"to":{"node":"receive_error","port":"value"}}, + {"kind":"data","from":{"node":"await","port":"value"},"to":{"node":"calculate","port":"a"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"send","port":"value"}}, + {"kind":"control","from":{"node":"send","port":"next"},"to":{"node":"sent","port":"in"}}, + {"kind":"data","from":{"node":"send","port":"ok"},"to":{"node":"sent","port":"condition"}}, + {"kind":"control","from":{"node":"sent","port":"true"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"sent","port":"false"},"to":{"node":"send_error","port":"in"}}, + {"kind":"data","from":{"node":"calculate","port":"value"},"to":{"node":"result","port":"value"}}, + {"kind":"data","from":{"node":"send","port":"error"},"to":{"node":"send_error","port":"value"}} + ] + } + } + ], + "channels":[ + {"id":"a_to_b","from":"A","to":"B","type":"number"}, + {"id":"b_to_c","from":"B","to":"C","type":"number"}, + {"id":"c_to_a","from":"C","to":"A","type":"number"} + ] +} diff --git a/examples/workflows/http-json-post.json b/examples/workflows/http-json-post.json new file mode 100644 index 00000000..d56e6bfb --- /dev/null +++ b/examples/workflows/http-json-post.json @@ -0,0 +1,170 @@ +{ + "version": 1, + "name": "Post JSON and read the response", + "entry": "request", + "nodes": [ + { + "id": "payload", + "block": "stringify_json", + "inputs": { + "value": { + "samples": [ + 10, + 20, + 30 + ], + "source": "debuglet-workflow" + } + }, + "position": { + "x": 0, + "y": 0 + } + }, + { + "id": "request", + "block": "http", + "inputs": { + "url": "http://127.0.0.1:8080/echo", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-Experiment": "workflow-post" + }, + "timeout_ms": 2000 + }, + "position": { + "x": 320, + "y": 0 + } + }, + { + "id": "transport", + "block": "if", + "position": { + "x": 640, + "y": 0 + } + }, + { + "id": "response", + "block": "parse_json", + "position": { + "x": 640, + "y": 300 + } + }, + { + "id": "result", + "block": "emit", + "inputs": { + "name": "response" + }, + "position": { + "x": 960, + "y": 0 + } + }, + { + "id": "error", + "block": "emit", + "inputs": { + "name": "transport_error" + }, + "position": { + "x": 960, + "y": 300 + } + } + ], + "edges": [ + { + "kind": "data", + "from": { + "node": "payload", + "port": "value" + }, + "to": { + "node": "request", + "port": "body" + } + }, + { + "kind": "control", + "from": { + "node": "request", + "port": "next" + }, + "to": { + "node": "transport", + "port": "in" + } + }, + { + "kind": "data", + "from": { + "node": "request", + "port": "ok" + }, + "to": { + "node": "transport", + "port": "condition" + } + }, + { + "kind": "control", + "from": { + "node": "transport", + "port": "true" + }, + "to": { + "node": "result", + "port": "in" + } + }, + { + "kind": "control", + "from": { + "node": "transport", + "port": "false" + }, + "to": { + "node": "error", + "port": "in" + } + }, + { + "kind": "data", + "from": { + "node": "request", + "port": "body" + }, + "to": { + "node": "response", + "port": "text" + } + }, + { + "kind": "data", + "from": { + "node": "response", + "port": "value" + }, + "to": { + "node": "result", + "port": "value" + } + }, + { + "kind": "data", + "from": { + "node": "request", + "port": "error" + }, + "to": { + "node": "error", + "port": "value" + } + } + ] +} diff --git a/examples/workflows/http-status.json b/examples/workflows/http-status.json new file mode 100644 index 00000000..1dad5558 --- /dev/null +++ b/examples/workflows/http-status.json @@ -0,0 +1,27 @@ +{ + "version": 1, + "name": "HTTP status branch", + "entry": "request", + "nodes": [ + {"id":"request","block":"http","inputs":{"url":"http://127.0.0.1:8088/","timeout_ms":2000},"position":{"x":80,"y":120}}, + {"id":"status","block":"greater_than","inputs":{"b":299},"position":{"x":360,"y":340}}, + {"id":"branch","block":"if","position":{"x":630,"y":120}}, + {"id":"problem","block":"emit","inputs":{"name":"http_problem"},"position":{"x":900,"y":40}}, + {"id":"response","block":"emit","inputs":{"name":"http_status"},"position":{"x":900,"y":280}}, + {"id":"transport","block":"if","position":{"x":350,"y":100}}, + {"id":"failed","block":"emit","inputs":{"name":"transport_error"},"position":{"x":630,"y":560}} + ], + "edges": [ + {"kind":"control","from":{"node":"request","port":"next"},"to":{"node":"transport","port":"in"}}, + {"kind":"data","from":{"node":"request","port":"status"},"to":{"node":"status","port":"a"}}, + {"kind":"data","from":{"node":"status","port":"value"},"to":{"node":"branch","port":"condition"}}, + {"kind":"control","from":{"node":"branch","port":"true"},"to":{"node":"problem","port":"in"}}, + {"kind":"control","from":{"node":"branch","port":"false"},"to":{"node":"response","port":"in"}}, + {"kind":"data","from":{"node":"request","port":"status"},"to":{"node":"problem","port":"value"}}, + {"kind":"data","from":{"node":"request","port":"status"},"to":{"node":"response","port":"value"}}, + {"kind":"data","from":{"node":"request","port":"ok"},"to":{"node":"transport","port":"condition"}}, + {"kind":"control","from":{"node":"transport","port":"true"},"to":{"node":"branch","port":"in"}}, + {"kind":"control","from":{"node":"transport","port":"false"},"to":{"node":"failed","port":"in"}}, + {"kind":"data","from":{"node":"request","port":"error"},"to":{"node":"failed","port":"value"}} + ] +} diff --git a/examples/workflows/ping-branch.json b/examples/workflows/ping-branch.json new file mode 100644 index 00000000..91e043be --- /dev/null +++ b/examples/workflows/ping-branch.json @@ -0,0 +1,31 @@ +{ + "version":1, + "name":"Ping, average, compare, branch", + "entry":"probe", + "nodes":[ + {"id":"probe","block":"ping","inputs":{"target":"127.0.0.1","count":3,"timeout_ms":1000},"position":{"x":40,"y":100}}, + {"id":"replied","block":"greater_than","inputs":{"b":0},"position":{"x":300,"y":340}}, + {"id":"have_samples","block":"if","position":{"x":560,"y":100}}, + {"id":"average","block":"mean","position":{"x":560,"y":340}}, + {"id":"slow","block":"greater_than","inputs":{"b":50},"position":{"x":820,"y":340}}, + {"id":"branch","block":"if","position":{"x":1080,"y":100}}, + {"id":"second_probe","block":"ping","inputs":{"target":"127.0.0.1","count":3,"timeout_ms":1000},"position":{"x":1340,"y":40}}, + {"id":"result","block":"emit","inputs":{"name":"mean_rtt_ms"},"position":{"x":1340,"y":340}}, + {"id":"no_samples","block":"emit","inputs":{"name":"probe_error"},"position":{"x":820,"y":560}} + ], + "edges":[ + {"kind":"control","from":{"node":"probe","port":"next"},"to":{"node":"have_samples","port":"in"}}, + {"kind":"data","from":{"node":"probe","port":"received"},"to":{"node":"replied","port":"a"}}, + {"kind":"data","from":{"node":"replied","port":"value"},"to":{"node":"have_samples","port":"condition"}}, + {"kind":"control","from":{"node":"have_samples","port":"true"},"to":{"node":"branch","port":"in"}}, + {"kind":"control","from":{"node":"have_samples","port":"false"},"to":{"node":"no_samples","port":"in"}}, + {"kind":"data","from":{"node":"probe","port":"error"},"to":{"node":"no_samples","port":"value"}}, + {"kind":"data","from":{"node":"probe","port":"rtts_ms"},"to":{"node":"average","port":"values"}}, + {"kind":"data","from":{"node":"average","port":"value"},"to":{"node":"slow","port":"a"}}, + {"kind":"data","from":{"node":"slow","port":"value"},"to":{"node":"branch","port":"condition"}}, + {"kind":"control","from":{"node":"branch","port":"true"},"to":{"node":"second_probe","port":"in"}}, + {"kind":"control","from":{"node":"branch","port":"false"},"to":{"node":"result","port":"in"}}, + {"kind":"control","from":{"node":"second_probe","port":"next"},"to":{"node":"result","port":"in"}}, + {"kind":"data","from":{"node":"average","port":"value"},"to":{"node":"result","port":"value"}} + ] +} diff --git a/go.mod b/go.mod index baee28e6..a4bdb888 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/hashicorp/yamux v0.1.2 github.com/labstack/echo/v4 v4.15.4 github.com/maxmind/mmdbwriter v1.2.0 + github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/netsec-ethz/scion-apps v0.6.0 github.com/oschwald/maxminddb-golang/v2 v2.6.0 github.com/pelletier/go-toml/v2 v2.2.4 @@ -22,6 +23,7 @@ require ( github.com/tetratelabs/wazero v1.12.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.57.0 + golang.org/x/net v0.60.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 @@ -50,6 +52,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect @@ -78,6 +81,8 @@ require ( github.com/quic-go/quic-go v0.43.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/samber/lo v1.49.1 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/segmentio/encoding v0.5.4 // indirect github.com/sethvargo/go-retry v0.4.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.3 // indirect @@ -88,13 +93,13 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect golang.org/x/mod v0.41.0 // indirect - golang.org/x/net v0.60.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.49.0 // indirect modernc.org/libc v1.74.3 // indirect diff --git a/go.sum b/go.sum index 3c098bd3..150021de 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= @@ -101,6 +103,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -189,6 +193,8 @@ github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/modelcontextprotocol/go-sdk v1.7.0 h1:yqjY2dsbKAC0LSuWZVBMrHgiG8ukXv6NRo0JiALay44= +github.com/modelcontextprotocol/go-sdk v1.7.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -236,6 +242,10 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= github.com/scionproto/scion v0.11.1-0.20240610170620-50b971ca2d4b h1:n4jkPRHGsPC4xNCkVXhOy2qzqUGuMrO6A1zuY1W/2FY= github.com/scionproto/scion v0.11.1-0.20240610170620-50b971ca2d4b/go.mod h1:paxrF6VreownCN7E7Rdri6ifXMkiq3leFGoP6n/BFC4= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -275,6 +285,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -313,8 +325,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -328,8 +338,6 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74= -golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -354,8 +362,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -368,8 +374,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -399,22 +403,16 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/internal/studio/build.go b/internal/studio/build.go new file mode 100644 index 00000000..540759af --- /dev/null +++ b/internal/studio/build.go @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// Package studio builds workflow guests without adding a compiler to the dispatcher. +package studio + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +const ( + MaxWorkflowBytes = 1 << 20 + MaxBundleBytes = 24 << 20 + maxWasmBytes = 16 << 20 + toolchain = "go1.26.9" +) + +type Bundle struct { + Version int `json:"version"` + Workflow workflow.Workflow `json:"workflow"` + WorkflowJSON string `json:"workflow_json"` + Source string `json:"source"` + WorkflowSHA256 string `json:"workflow_sha256"` + WasmSHA256 string `json:"wasm_sha256"` + Wasm []byte `json:"wasm"` + Destinations []string `json:"destinations"` + RequireICMP bool `json:"require_icmp"` +} + +type ValidationError struct { + Diagnostics []workflow.Diagnostic `json:"diagnostics"` +} + +func (e *ValidationError) Error() string { return "workflow has validation errors" } + +type Builder struct { + sdk string + goCommand string + slot chan struct{} +} + +// NewBuilder accepts the trusted SDK location chosen by the operator, never by a graph. +func NewBuilder(sdk string) (*Builder, error) { + path, err := filepath.Abs(sdk) + if err != nil || sdk == "" { + return nil, errors.New("--sdk must name the Debuglet source checkout containing pkg/workflow") + } + module, err := os.ReadFile(filepath.Join(path, "go.mod")) + if err != nil { + return nil, fmt.Errorf("read SDK module: %w", err) + } + if !strings.HasPrefix(string(module), "module github.com/netsec-ethz/debuglet\n") { + return nil, errors.New("--sdk must name a Debuglet SDK checkout") + } + if _, err := os.Stat(filepath.Join(path, "pkg", "workflow", "guest")); err != nil { + return nil, errors.New("SDK checkout lacks pkg/workflow/guest; use the same version as dbl-studio") + } + goCommand, err := exec.LookPath("go") + if err != nil { + return nil, fmt.Errorf("install Go %s to compile workflows: %w", strings.TrimPrefix(toolchain, "go"), err) + } + return &Builder{sdk: path, goCommand: goCommand, slot: make(chan struct{}, 1)}, nil +} + +func (b *Builder) Build(ctx context.Context, wf workflow.Workflow) (*Bundle, error) { + if diagnostics := workflow.Validate(wf); len(diagnostics) != 0 { + return nil, &ValidationError{diagnostics} + } + raw, err := json.Marshal(wf) + if err != nil || len(raw) > MaxWorkflowBytes { + return nil, errors.New("workflow exceeds the 1 MiB limit or is not valid JSON") + } + program, err := workflow.Generate(wf) + if err != nil { + return nil, err + } + return b.buildProgram(ctx, wf, raw, program) +} + +func (b *Builder) buildProgram(ctx context.Context, wf workflow.Workflow, raw []byte, program workflow.Program) (*Bundle, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + select { + case b.slot <- struct{}{}: + defer func() { <-b.slot }() + case <-ctx.Done(): + return nil, ctx.Err() + } + if err := ctx.Err(); err != nil { + return nil, err + } + dir, err := os.MkdirTemp("", "debuglet-workflow-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + env := buildEnv() + version := exec.CommandContext(ctx, b.goCommand, "env", "GOVERSION") + version.Env = env + version.Dir = dir + actual, err := version.Output() + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err != nil || strings.TrimSpace(string(actual)) != toolchain { + return nil, fmt.Errorf("workflow builds require %s installed locally (automatic toolchain downloads are disabled)", toolchain) + } + module := "module debuglet.workflow.build\n\ngo 1.26.9\n\nrequire github.com/netsec-ethz/debuglet v0.0.0\nreplace github.com/netsec-ethz/debuglet => " + strconv.Quote(filepath.ToSlash(b.sdk)) + "\n" + for name, data := range map[string][]byte{"go.mod": []byte(module), "main.go": []byte(program.Source)} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + return nil, err + } + } + command := exec.CommandContext(ctx, b.goCommand, "build", "-mod=readonly", "-buildvcs=false", "-trimpath", "-ldflags=-s -w", "-o", "program.wasm", ".") + command.Dir, command.Env = dir, env + configureBuildProcess(command) + output := &boundedOutput{limit: 64 << 10} + command.Stdout, command.Stderr = output, output + if err := command.Run(); err != nil { + if ctx.Err() != nil { + return nil, fmt.Errorf("workflow compilation stopped: %w", ctx.Err()) + } + message := strings.ReplaceAll(output.String(), dir, "") + message = strings.ReplaceAll(message, b.sdk, "") + if strings.TrimSpace(message) == "" { + message = err.Error() + } + return nil, fmt.Errorf("compile workflow: %s", strings.TrimSpace(message)) + } + file, err := os.Open(filepath.Join(dir, "program.wasm")) + if err != nil { + return nil, err + } + defer file.Close() + wasm, err := io.ReadAll(io.LimitReader(file, maxWasmBytes+1)) + if err != nil { + return nil, err + } + if len(wasm) > maxWasmBytes { + return nil, errors.New("compiled workflow exceeds the 16 MiB module limit") + } + bundle := &Bundle{Version: 1, Workflow: wf, WorkflowJSON: string(raw), Source: program.Source, WorkflowSHA256: program.WorkflowSHA256, WasmSHA256: digest(wasm), Wasm: wasm, Destinations: program.Destinations, RequireICMP: program.RequireICMP} + encoded, err := json.Marshal(bundle) + if err != nil { + return nil, err + } + if len(encoded) > MaxBundleBytes { + return nil, errors.New("compiled bundle exceeds the 24 MiB limit") + } + return bundle, nil +} + +func digest(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + +func buildEnv() []string { + env := []string{"GOOS=wasip1", "GOARCH=wasm", "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOPROXY=off", "GOSUMDB=off", "GOWORK=off", "GOENV=off", "GOFLAGS="} + for _, key := range []string{"PATH", "HOME", "TMPDIR", "GOCACHE", "GOMODCACHE"} { + if value, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+value) + } + } + return env +} + +type boundedOutput struct { + bytes.Buffer + limit int +} + +func (b *boundedOutput) Write(p []byte) (int, error) { + n := len(p) + if left := b.limit - b.Len(); left > 0 { + if len(p) > left { + p = p[:left] + } + _, _ = b.Buffer.Write(p) + } + return n, nil +} + +func DecodeWorkflow(r io.Reader) (workflow.Workflow, error) { + var wf workflow.Workflow + data, err := io.ReadAll(io.LimitReader(r, MaxWorkflowBytes+1)) + if err != nil { + return wf, err + } + if len(data) > MaxWorkflowBytes { + return wf, errors.New("workflow exceeds the 1 MiB limit") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wf); err != nil { + return wf, fmt.Errorf("read workflow JSON: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return wf, errors.New("expected one workflow JSON object") + } + return wf, nil +} diff --git a/internal/studio/experiment.go b/internal/studio/experiment.go new file mode 100644 index 00000000..7eee16bf --- /dev/null +++ b/internal/studio/experiment.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +const MaxExperimentBytes = 4 << 20 +const MaxExperimentBundleBytes = 32 << 20 + +type ExperimentBundle struct { + Version int `json:"version"` + Kind string `json:"kind"` + Experiment workflow.Experiment `json:"experiment"` + ExperimentJSON string `json:"experiment_json"` + ExperimentSHA256 string `json:"experiment_sha256"` + Participants []ParticipantBundle `json:"participants"` + RequireTCPListener bool `json:"require_tcp_listener"` + ProgramBytes int `json:"program_bytes"` + EncodedProgramBytes int `json:"encoded_program_bytes"` +} +type ParticipantBundle struct { + ID string `json:"id"` + Label string `json:"label,omitempty"` + Bundle *Bundle `json:"bundle"` +} +type ExperimentValidationError struct { + Diagnostics []workflow.ExperimentDiagnostic `json:"diagnostics"` +} + +func (e *ExperimentValidationError) Error() string { return "experiment has validation errors" } + +func DecodeExperiment(r io.Reader) (workflow.Experiment, error) { + var experiment workflow.Experiment + data, err := io.ReadAll(io.LimitReader(r, MaxExperimentBytes+1)) + if err != nil { + return experiment, err + } + if len(data) > MaxExperimentBytes { + return experiment, errors.New("experiment exceeds 4 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&experiment); err != nil { + return experiment, fmt.Errorf("read experiment JSON: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return experiment, errors.New("expected one experiment JSON object") + } + for _, p := range experiment.Participants { + if _, err := boundedWorkflow(p.Workflow); err != nil { + return experiment, fmt.Errorf("participant %s: %w", p.ID, err) + } + } + return experiment, nil +} + +// BuildExperiment compiles the participants for one existing grouped submission. +// No program is submitted and no permission is inferred from channel metadata. +func (b *Builder) BuildExperiment(ctx context.Context, e workflow.Experiment) (*ExperimentBundle, error) { + if ds := workflow.ValidateExperiment(e); len(ds) > 0 { + return nil, &ExperimentValidationError{ds} + } + raw, err := json.Marshal(e) + if err != nil { + return nil, err + } + if len(raw) > MaxExperimentBytes { + return nil, errors.New("experiment exceeds 4 MiB") + } + result := &ExperimentBundle{Version: 1, Kind: "experiment_bundle", Experiment: e, ExperimentJSON: string(raw), ExperimentSHA256: digest(raw), RequireTCPListener: true, Participants: []ParticipantBundle{}} + for _, p := range e.Participants { + if _, err := boundedWorkflow(p.Workflow); err != nil { + return nil, fmt.Errorf("participant %s: %w", p.ID, err) + } + wf, program, err := workflow.GenerateParticipant(e, p.ID) + if err != nil { + return nil, err + } + canonical, err := json.Marshal(wf) + if err != nil { + return nil, err + } + bundle, err := b.buildProgram(ctx, wf, canonical, program) + if err != nil { + return nil, fmt.Errorf("participant %s: %w", p.ID, err) + } + result.Participants = append(result.Participants, ParticipantBundle{p.ID, p.Label, bundle}) + result.ProgramBytes += len(bundle.Wasm) + result.EncodedProgramBytes += base64.StdEncoding.EncodedLen(len(bundle.Wasm)) + // Policies, identities and other fields are added by the console; its final + // exact request-size check remains authoritative for the 32 MiB API limit. + if result.EncodedProgramBytes >= 32<<20 { + return nil, errors.New("participant programs exceed the grouped API's 32 MiB request limit; reduce the programs or participant count") + } + } + encoded, err := json.Marshal(result) + if err != nil { + return nil, err + } + if len(encoded) > MaxExperimentBundleBytes { + return nil, errors.New("compiled experiment bundle exceeds 32 MiB") + } + return result, nil +} + +func experimentRoutes(mux *http.ServeMux, builder *Builder, token string) { + handler := func(build bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Studio-Token") != token { + writeError(w, 403, errors.New("open the studio page again to obtain its session token")) + return + } + content, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || content != "application/json" { + writeError(w, 415, errors.New("Content-Type must be application/json")) + return + } + r.Body = http.MaxBytesReader(w, r.Body, MaxExperimentBytes) + e, err := DecodeExperiment(r.Body) + if err != nil { + writeError(w, 400, err) + return + } + if !build { + ds := workflow.ValidateExperiment(e) + if ds == nil { + ds = []workflow.ExperimentDiagnostic{} + } + writeJSON(w, 200, map[string]any{"diagnostics": ds}) + return + } + if builder == nil { + writeError(w, 503, errors.New("compiler unavailable; configure --sdk")) + return + } + bundle, err := builder.BuildExperiment(r.Context(), e) + if err != nil { + writeError(w, 422, err) + return + } + writeJSON(w, 200, bundle) + } + } + mux.HandleFunc("POST /studio/api/experiments/validate", handler(false)) + mux.HandleFunc("POST /studio/api/experiments/build", handler(true)) +} diff --git a/internal/studio/experiment_mcp.go b/internal/studio/experiment_mcp.go new file mode 100644 index 00000000..968dcc14 --- /dev/null +++ b/internal/studio/experiment_mcp.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +type experimentInput struct { + Experiment workflow.Experiment `json:"experiment"` +} + +func experimentTools(server *mcp.Server, builder *Builder, files *store) { + mcp.AddTool(server, &mcp.Tool{Name: "validate_experiment", Description: "Validate participant workflows and declared directed message channels. Direct TCP requires pairwise reachability and listener permissions; messages are plaintext."}, func(ctx context.Context, req *mcp.CallToolRequest, input experimentInput) (*mcp.CallToolResult, map[string]any, error) { + raw, err := json.Marshal(input.Experiment) + if err != nil { + return nil, nil, err + } + if len(raw) > MaxExperimentBytes { + return nil, nil, errors.New("experiment exceeds 4 MiB") + } + diagnostics := workflow.ValidateExperiment(input.Experiment) + if diagnostics == nil { + diagnostics = []workflow.ExperimentDiagnostic{} + } + return nil, map[string]any{"diagnostics": diagnostics}, nil + }) + mcp.AddTool(server, &mcp.Tool{Name: "build_experiment", Description: "Compile 2–3 participant workflows into a portable experiment bundle, with source, exact shared context and program digests. Does not submit, allocate executors or spend funds. Import the bundle into the experiment editor to inspect or edit it."}, func(ctx context.Context, req *mcp.CallToolRequest, input experimentInput) (*mcp.CallToolResult, metadata, error) { + if builder == nil { + return nil, metadata{}, errors.New("compiler unavailable: configure --sdk") + } + bundle, err := builder.BuildExperiment(ctx, input.Experiment) + if err != nil { + return nil, metadata{}, err + } + encoded, err := json.Marshal(bundle) + if err != nil { + return nil, metadata{}, err + } + hash := digest(encoded) + id := bundle.ExperimentSHA256 + "-" + hash[:12] + filename := id + ".experiment-bundle.json" + files.mu.Lock() + defer files.mu.Unlock() + if _, exists := files.artifacts[id]; !exists && len(files.artifacts) >= 16 { + return nil, metadata{}, errors.New("this session has 16 artifacts; start a fresh MCP session before building more") + } + if err := files.write(filename, encoded); err != nil { + return nil, metadata{}, err + } + meta := metadata{Kind: "experiment_bundle", ArtifactID: id, BundlePath: filepath.Join(files.directory, filename), ExperimentSHA256: bundle.ExperimentSHA256, BundleSHA256: hash} + files.artifacts[id] = meta + return nil, meta, nil + }) +} + +func inspectExperimentArtifact(data []byte, meta metadata) (map[string]any, error) { + var bundle ExperimentBundle + if err := json.Unmarshal(data, &bundle); err != nil { + return nil, err + } + if digest([]byte(bundle.ExperimentJSON)) != meta.ExperimentSHA256 { + return nil, errors.New("experiment content changed since build") + } + sources := map[string]string{} + size := len(bundle.ExperimentJSON) + for _, p := range bundle.Participants { + if p.Bundle == nil || digest(p.Bundle.Wasm) != p.Bundle.WasmSHA256 || digest([]byte(p.Bundle.WorkflowJSON)) != p.Bundle.WorkflowSHA256 { + return nil, errors.New("participant content changed since build") + } + sources[p.ID] = p.Bundle.Source + size += len(p.Bundle.Source) + } + if size > 128<<10 { + return nil, errors.New("artifact text exceeds 128 KiB; inspect the bundle in the experiment editor") + } + return map[string]any{"metadata": meta, "experiment": bundle.Experiment, "sources": sources}, nil +} diff --git a/internal/studio/experiment_test.go b/internal/studio/experiment_test.go new file mode 100644 index 00000000..fb3c9434 --- /dev/null +++ b/internal/studio/experiment_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "bytes" + "context" + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func TestExperimentHTTPUsesStudioSessionAndRejectsUnknownFields(t *testing.T) { + handler, err := NewHandler(nil, ServerConfig{}, "http://127.0.0.1:8090") + if err != nil { + t.Fatal(err) + } + session := httptest.NewRecorder() + handler.ServeHTTP(session, httptest.NewRequest("GET", "http://127.0.0.1:8090/studio/api/session", nil)) + var auth map[string]string + if err = json.Unmarshal(session.Body.Bytes(), &auth); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile("../../examples/workflows/experiments/three-node.json") + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + token, path string + body []byte + status int + }{{"", "validate", body, 403}, {auth["token"], "validate", body, 200}, {auth["token"], "build", body, 503}, {auth["token"], "validate", []byte(`{"version":1,"unknown":true}`), 400}} { + request := httptest.NewRequest("POST", "http://127.0.0.1:8090/studio/api/experiments/"+test.path, bytes.NewReader(test.body)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Studio-Token", test.token) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != test.status { + t.Fatalf("%s: %d %s", test.path, response.Code, response.Body) + } + } + for _, raw := range []string{string(body) + " {}", strings.Repeat(" ", MaxExperimentBytes+1)} { + if _, err := DecodeExperiment(strings.NewReader(raw)); err == nil { + t.Fatal("accepted invalid input") + } + } +} + +func TestBuildExperimentPreservesContextAndCompilesEveryParticipant(t *testing.T) { + if testing.Short() { + t.Skip("WASI compilation") + } + source, err := os.Open("../../examples/workflows/experiments/three-node.json") + if err != nil { + t.Fatal(err) + } + defer source.Close() + e, err := DecodeExperiment(source) + if err != nil { + t.Fatal(err) + } + sdk, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + builder, err := NewBuilder(sdk) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + server, closeServer, err := MCPServer(builder, t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer closeServer() + serverTransport, clientTransport := mcp.NewInMemoryTransports() + ss, err := server.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatal(err) + } + defer ss.Close() + client := mcp.NewClient(&mcp.Implementation{Name: "experiment-test", Version: "1"}, nil) + cs, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatal(err) + } + defer cs.Close() + built, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: "build_experiment", Arguments: experimentInput{e}}) + if err != nil || built.IsError { + t.Fatalf("build experiment: %v %+v", err, built) + } + metaJSON, err := json.Marshal(built.StructuredContent) + if err != nil { + t.Fatal(err) + } + var meta metadata + if err := json.Unmarshal(metaJSON, &meta); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(meta.BundlePath) + if err != nil { + t.Fatal(err) + } + var bundle ExperimentBundle + if err := json.Unmarshal(data, &bundle); err != nil { + t.Fatal(err) + } + if bundle.ExperimentSHA256 != digest([]byte(bundle.ExperimentJSON)) || len(bundle.Participants) != 3 || !bundle.RequireTCPListener { + t.Fatal("incomplete experiment bundle") + } + for _, p := range bundle.Participants { + if p.Bundle.Workflow.Experiment.Fingerprint != bundle.ExperimentSHA256 || p.Bundle.Workflow.Experiment.Participant != p.ID || p.Bundle.WorkflowSHA256 != digest([]byte(p.Bundle.WorkflowJSON)) || p.Bundle.WasmSHA256 != digest(p.Bundle.Wasm) { + t.Fatalf("participant %s identity mismatch", p.ID) + } + if len(p.Bundle.Wasm) < 8 || string(p.Bundle.Wasm[:4]) != "\x00asm" { + t.Fatalf("participant %s is not WASM", p.ID) + } + } + if len(data) > MaxExperimentBundleBytes || bundle.EncodedProgramBytes >= 32<<20 { + t.Fatal("oversized experiment") + } + read, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: "read_artifact", Arguments: artifactInput{meta.ArtifactID}}) + if err != nil || read.IsError { + t.Fatalf("read experiment artifact: %v %+v", err, read) + } +} diff --git a/internal/studio/http.go b/internal/studio/http.go new file mode 100644 index 00000000..66f88415 --- /dev/null +++ b/internal/studio/http.go @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "io" + "mime" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +type ServerConfig struct { + Address string + UI string + Dispatcher string + CAFile string +} + +// Serve binds only a literal loopback address. The browser and authoring API share one origin. +func Serve(ctx context.Context, builder *Builder, config ServerConfig, ready func(string)) error { + host, _, err := net.SplitHostPort(config.Address) + if err != nil || net.ParseIP(host) == nil || !net.ParseIP(host).IsLoopback() { + return errors.New("--listen must be a literal loopback address, for example 127.0.0.1:8090") + } + listener, err := net.Listen("tcp", config.Address) + if err != nil { + return err + } + defer listener.Close() + origin := "http://" + listener.Addr().String() + handler, err := NewHandler(builder, config, origin) + if err != nil { + return err + } + server := &http.Server{Handler: handler, BaseContext: func(net.Listener) context.Context { return ctx }, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 7 * time.Minute, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 16 << 10} + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + stop, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = server.Shutdown(stop) + case <-done: + } + }() + if ready != nil { + ready(origin + "/console/builder") + } + err = server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func NewHandler(builder *Builder, config ServerConfig, origin string) (http.Handler, error) { + parsed, err := url.Parse(origin) + if err != nil || parsed.Host == "" { + return nil, errors.New("invalid studio origin") + } + tokenBytes := make([]byte, 32) + if _, err := rand.Read(tokenBytes); err != nil { + return nil, err + } + token := hex.EncodeToString(tokenBytes) + mux := http.NewServeMux() + experimentRoutes(mux, builder, token) + mux.HandleFunc("GET /studio/api/session", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]string{"token": token}) }) + mux.HandleFunc("GET /studio/api/catalog", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, workflow.Catalog()) }) + mutate := func(fn func(http.ResponseWriter, *http.Request, workflow.Workflow)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Studio-Token") != token { + writeError(w, 403, errors.New("open the studio page again to obtain its session token")) + return + } + mediaType, _, mediaErr := mime.ParseMediaType(r.Header.Get("Content-Type")) + if mediaErr != nil || mediaType != "application/json" { + writeError(w, 415, errors.New("Content-Type must be application/json")) + return + } + r.Body = http.MaxBytesReader(w, r.Body, MaxWorkflowBytes) + wf, err := DecodeWorkflow(r.Body) + if err != nil { + writeError(w, 400, err) + return + } + fn(w, r, wf) + } + } + mux.HandleFunc("POST /studio/api/validate", mutate(func(w http.ResponseWriter, r *http.Request, wf workflow.Workflow) { + diagnostics := workflow.Validate(wf) + if diagnostics == nil { + diagnostics = []workflow.Diagnostic{} + } + writeJSON(w, 200, map[string]any{"diagnostics": diagnostics}) + })) + mux.HandleFunc("POST /studio/api/build", mutate(func(w http.ResponseWriter, r *http.Request, wf workflow.Workflow) { + if builder == nil { + writeError(w, 503, errors.New("compiler is unavailable; restart dbl-studio with --sdk pointing to the matching Debuglet checkout")) + return + } + bundle, err := builder.Build(r.Context(), wf) + if err != nil { + writeError(w, 422, err) + return + } + writeJSON(w, 200, bundle) + })) + if config.Dispatcher != "" { + target, err := url.Parse(config.Dispatcher) + if err != nil || (target.Scheme != "http" && target.Scheme != "https") || target.Host == "" || target.User != nil || target.RawQuery != "" || target.Fragment != "" || (target.Path != "" && target.Path != "/") { + return nil, errors.New("--dispatcher must be an http(s) origin without credentials, path, query or fragment") + } + upstreamIP := net.ParseIP(target.Hostname()) + if target.Hostname() != "localhost" && (upstreamIP == nil || !upstreamIP.IsLoopback()) { + return nil, errors.New("--dispatcher supports an owned loopback development dispatcher only; import the built bundle into a public dispatcher’s hosted console to sign in and submit there") + } + proxy := httputil.NewSingleHostReverseProxy(target) + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + if host == "localhost" { + address = net.JoinHostPort("127.0.0.1", port) + } + return (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext(ctx, network, address) + } + if config.CAFile != "" { + certs, err := os.ReadFile(config.CAFile) + if err != nil { + return nil, err + } + pool, err := x509.SystemCertPool() + if err != nil { + pool = x509.NewCertPool() + } + if !pool.AppendCertsFromPEM(certs) { + return nil, errors.New("--ca-file contains no PEM certificates") + } + transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} + } + proxy.Transport = transport + original := proxy.Director + proxy.Director = func(r *http.Request) { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") + if r.URL.RawPath != "" { + r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, "/api") + } + original(r) + r.Host = target.Host + r.Header.Del("X-Studio-Token") + if r.Header.Get("Origin") != "" { + r.Header.Set("Origin", target.Scheme+"://"+target.Host) + } + } + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + writeError(w, 502, errors.New("dispatcher is unreachable; check the configured dispatcher address")) + } + proxy.ModifyResponse = func(response *http.Response) error { + if response.ContentLength > 32<<20 { + return errors.New("dispatcher response exceeds 32 MiB") + } + response.Body = &limitedBody{ReadCloser: response.Body, remaining: 32 << 20} + // Cookies belong to the local console proxy, not the upstream hostname. + cookies := response.Cookies() + response.Header.Del("Set-Cookie") + for _, cookie := range cookies { + cookie.Domain = "" + response.Header.Add("Set-Cookie", cookie.String()) + } + return nil + } + mux.Handle("/api/", http.MaxBytesHandler(proxy, 32<<20)) + } else { + mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { + writeError(w, 503, errors.New("no dispatcher configured; start dbl-studio serve with --dispatcher to sign in and submit")) + }) + } + if config.UI != "" { + root, err := filepath.Abs(config.UI) + if err != nil { + return nil, err + } + info, err := os.Stat(filepath.Join(root, "index.html")) + if err != nil || info.IsDir() { + return nil, errors.New("--ui must name the built console directory containing index.html") + } + files := http.FileServer(http.Dir(root)) + mux.HandleFunc("GET /console/", func(w http.ResponseWriter, r *http.Request) { + relative := strings.TrimPrefix(r.URL.Path, "/console/") + path := filepath.Join(root, filepath.FromSlash(relative)) + if info, err := os.Stat(path); err == nil && !info.IsDir() { + copy := r.Clone(r.Context()) + copy.URL.Path = "/" + relative + files.ServeHTTP(w, copy) + return + } + if filepath.Ext(relative) != "" { + http.NotFound(w, r) + return + } + http.ServeFile(w, r, filepath.Join(root, "index.html")) + }) + } else { + mux.HandleFunc("GET /console/", func(w http.ResponseWriter, r *http.Request) { + writeError(w, 503, errors.New("build the Debuglet console and restart with --ui /dist to open the visual editor")) + }) + } + mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + http.Redirect(w, r, "/console/builder", http.StatusTemporaryRedirect) + }) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "no-referrer") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; font-src 'self'; object-src 'none'; base-uri 'self'; frame-ancestors 'none'") + if r.Host != parsed.Host || (r.Header.Get("Origin") != "" && r.Header.Get("Origin") != origin) || r.Header.Get("Sec-Fetch-Site") == "cross-site" { + writeError(w, 403, errors.New("studio requests must use its own loopback origin")) + return + } + mux.ServeHTTP(w, r) + }), nil +} + +type limitedBody struct { + io.ReadCloser + remaining int64 +} + +func (r *limitedBody) Read(p []byte) (int, error) { + if r.remaining <= 0 { + var probe [1]byte + n, err := r.ReadCloser.Read(probe[:]) + if n > 0 { + return 0, errors.New("dispatcher response exceeds 32 MiB") + } + return 0, err + } + if int64(len(p)) > r.remaining { + p = p[:r.remaining] + } + n, err := r.ReadCloser.Read(p) + r.remaining -= int64(n) + return n, err +} +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} +func writeError(w http.ResponseWriter, status int, err error) { + body := map[string]any{"error": err.Error()} + var invalid *ValidationError + if errors.As(err, &invalid) { + body["diagnostics"] = invalid.Diagnostics + } + var experimentInvalid *ExperimentValidationError + if errors.As(err, &experimentInvalid) { + body["diagnostics"] = experimentInvalid.Diagnostics + } + writeJSON(w, status, body) +} diff --git a/internal/studio/mcp.go b/internal/studio/mcp.go new file mode 100644 index 00000000..dfa68ef2 --- /dev/null +++ b/internal/studio/mcp.go @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sync" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +type workflowInput struct { + Workflow workflow.Workflow `json:"workflow"` +} +type savedInput struct { + Name string `json:"name" jsonschema:"A simple name containing letters, digits, hyphens or underscores (no path)."` + Workflow workflow.Workflow `json:"workflow"` +} +type nameInput struct { + Name string `json:"name"` +} +type artifactInput struct { + ArtifactID string `json:"artifact_id"` +} +type metadata struct { + Kind string `json:"kind,omitempty"` + ExperimentSHA256 string `json:"experiment_sha256,omitempty"` + ArtifactID string `json:"artifact_id"` + BundlePath string `json:"bundle_path"` + WorkflowSHA256 string `json:"workflow_sha256"` + WasmSHA256 string `json:"wasm_sha256"` + BundleSHA256 string `json:"bundle_sha256"` + Destinations []string `json:"destinations"` + RequireICMP bool `json:"require_icmp"` +} +type store struct { + root *os.Root + directory string + mu sync.Mutex + artifacts map[string]metadata +} + +var safeName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,47}$`) + +// MCP exposes authoring tools only. Submission and execution remain explicit console actions. +func MCP(ctx context.Context, builder *Builder, directory string) error { + server, close, err := MCPServer(builder, directory) + if err != nil { + return err + } + defer close() + return server.Run(ctx, &mcp.StdioTransport{}) +} + +func MCPServer(builder *Builder, directory string) (*mcp.Server, func(), error) { + directory, err := filepath.Abs(directory) + if err != nil { + return nil, nil, err + } + if err := os.MkdirAll(directory, 0700); err != nil { + return nil, nil, err + } + root, err := os.OpenRoot(directory) + if err != nil { + return nil, nil, err + } + files := &store{root: root, directory: directory, artifacts: make(map[string]metadata)} + server := mcp.NewServer(&mcp.Implementation{Name: "debuglet-studio", Version: "1.0.0"}, &mcp.ServerOptions{Instructions: "Author version-1 Debuglet workflows using catalog primitives. Data edges carry typed values; control edges execute actions. Save the exact workflow for human inspection in the visual editor. Build creates a bundle, but never submits it or spends funds. Custom Go code is visible in the graph and has the same guest permissions as its enclosing program."}) + mcp.AddTool(server, &mcp.Tool{Name: "catalog", Description: "List reusable block types, typed ports and control outputs."}, func(ctx context.Context, req *mcp.CallToolRequest, input struct{}) (*mcp.CallToolResult, map[string]any, error) { + return nil, map[string]any{"blocks": workflow.Catalog()}, nil + }) + mcp.AddTool(server, &mcp.Tool{Name: "validate", Description: "Check the same workflow graph used by the visual editor, including types and bounded control flow."}, func(ctx context.Context, req *mcp.CallToolRequest, input workflowInput) (*mcp.CallToolResult, map[string]any, error) { + if _, err := boundedWorkflow(input.Workflow); err != nil { + return nil, nil, err + } + diagnostics := workflow.Validate(input.Workflow) + if diagnostics == nil { + diagnostics = []workflow.Diagnostic{} + } + return nil, map[string]any{"diagnostics": diagnostics}, nil + }) + mcp.AddTool(server, &mcp.Tool{Name: "save_workflow", Description: "Save an editable workflow JSON in the configured studio workspace. Replaces a workflow with this simple name; no arbitrary filesystem paths."}, func(ctx context.Context, req *mcp.CallToolRequest, input savedInput) (*mcp.CallToolResult, map[string]any, error) { + if !safeName.MatchString(input.Name) { + return nil, nil, errors.New("name must contain 1–48 letters, digits, hyphens or underscores") + } + data, err := boundedWorkflow(input.Workflow) + if err != nil { + return nil, nil, err + } + filename := input.Name + ".workflow.json" + if err := files.write(filename, data); err != nil { + return nil, nil, err + } + return nil, map[string]any{"path": filepath.Join(directory, filename), "workflow_sha256": digest(data)}, nil + }) + mcp.AddTool(server, &mcp.Tool{Name: "get_workflow", Description: "Read a named workflow from the configured studio workspace, for inspection or editing."}, func(ctx context.Context, req *mcp.CallToolRequest, input nameInput) (*mcp.CallToolResult, map[string]any, error) { + if !safeName.MatchString(input.Name) { + return nil, nil, errors.New("invalid workflow name") + } + file, err := root.Open(input.Name + ".workflow.json") + if err != nil { + return nil, nil, err + } + defer file.Close() + wf, err := DecodeWorkflow(file) + return nil, map[string]any{"workflow": wf}, err + }) + mcp.AddTool(server, &mcp.Tool{Name: "build", Description: "Compile an inspected workflow to a WASI bundle using the configured trusted SDK. Returns artifact metadata and a file to import into the console; does not run the program."}, func(ctx context.Context, req *mcp.CallToolRequest, input workflowInput) (*mcp.CallToolResult, metadata, error) { + if builder == nil { + return nil, metadata{}, errors.New("compiler unavailable: configure --sdk") + } + bundle, err := builder.Build(ctx, input.Workflow) + if err != nil { + return nil, metadata{}, err + } + encoded, err := json.Marshal(bundle) + if err != nil { + return nil, metadata{}, err + } + id := bundle.WorkflowSHA256 + "-" + bundle.WasmSHA256[:12] + filename := id + ".bundle.json" + files.mu.Lock() + defer files.mu.Unlock() + if _, exists := files.artifacts[id]; !exists && len(files.artifacts) >= 16 { + return nil, metadata{}, errors.New("this session has 16 artifacts; start a fresh MCP session before building more") + } + if err := files.write(filename, encoded); err != nil { + return nil, metadata{}, err + } + meta := metadata{ArtifactID: id, BundlePath: filepath.Join(directory, filename), WorkflowSHA256: bundle.WorkflowSHA256, WasmSHA256: bundle.WasmSHA256, BundleSHA256: digest(encoded), Destinations: bundle.Destinations, RequireICMP: bundle.RequireICMP} + files.artifacts[id] = meta + return nil, meta, nil + }) + experimentTools(server, builder, files) + mcp.AddTool(server, &mcp.Tool{Name: "read_artifact", Description: "Inspect the graph, generated Go and metadata of a bundle created in this session. WASM bytes remain in the bundle file."}, func(ctx context.Context, req *mcp.CallToolRequest, input artifactInput) (*mcp.CallToolResult, map[string]any, error) { + files.mu.Lock() + meta, ok := files.artifacts[input.ArtifactID] + files.mu.Unlock() + if !ok { + return nil, nil, errors.New("artifact was not created by this session") + } + file, err := root.Open(filepath.Base(meta.BundlePath)) + if err != nil { + return nil, nil, err + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, MaxExperimentBundleBytes+1)) + if err != nil { + return nil, nil, err + } + if len(data) > MaxExperimentBundleBytes { + return nil, nil, errors.New("bundle exceeds size limit") + } + if digest(data) != meta.BundleSHA256 { + return nil, nil, errors.New("artifact content changed since build") + } + if meta.Kind == "experiment_bundle" { + result, err := inspectExperimentArtifact(data, meta) + return nil, result, err + } + var bundle Bundle + if err := json.Unmarshal(data, &bundle); err != nil { + return nil, nil, err + } + if digest(bundle.Wasm) != meta.WasmSHA256 || digest([]byte(bundle.WorkflowJSON)) != meta.WorkflowSHA256 { + return nil, nil, errors.New("artifact content changed since build") + } + if len(bundle.Source)+len(bundle.WorkflowJSON) > 128<<10 { + return nil, nil, fmt.Errorf("artifact text exceeds the 128 KiB inspection limit; inspect %s in the editor", meta.BundlePath) + } + return nil, map[string]any{"metadata": meta, "workflow": bundle.Workflow, "source": bundle.Source}, nil + }) + return server, func() { _ = root.Close() }, nil +} + +func boundedWorkflow(wf workflow.Workflow) ([]byte, error) { + data, err := json.Marshal(wf) + if err != nil { + return nil, err + } + if len(data) > MaxWorkflowBytes { + return nil, errors.New("workflow exceeds the 1 MiB limit") + } + return data, nil +} +func (s *store) write(name string, data []byte) error { + temp, err := os.CreateTemp(s.directory, ".write-*") + if err != nil { + return err + } + tempName := temp.Name() + defer os.Remove(tempName) + if _, err := temp.Write(data); err != nil { + _ = temp.Close() + return err + } + if err := temp.Close(); err != nil { + return err + } + return s.root.Rename(filepath.Base(tempName), name) +} diff --git a/internal/studio/process_other.go b/internal/studio/process_other.go new file mode 100644 index 00000000..b5eedebc --- /dev/null +++ b/internal/studio/process_other.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +//go:build !linux && !darwin && !freebsd && !openbsd && !netbsd + +package studio + +import ( + "os/exec" + "time" +) + +func configureBuildProcess(command *exec.Cmd) { command.WaitDelay = 5 * time.Second } diff --git a/internal/studio/process_unix.go b/internal/studio/process_unix.go new file mode 100644 index 00000000..ca6406f5 --- /dev/null +++ b/internal/studio/process_unix.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +//go:build linux || darwin || freebsd || openbsd || netbsd + +package studio + +import ( + "os" + "os/exec" + "syscall" + "time" +) + +func configureBuildProcess(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + command.Cancel = func() error { + err := syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + if err == syscall.ESRCH { + return os.ErrProcessDone + } + return err + } + command.WaitDelay = 5 * time.Second +} diff --git a/internal/studio/studio_test.go b/internal/studio/studio_test.go new file mode 100644 index 00000000..99503769 --- /dev/null +++ b/internal/studio/studio_test.go @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package studio + +import ( + "bytes" + "context" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/netsec-ethz/debuglet/pkg/workflow" +) + +func sampleWorkflow() workflow.Workflow { + return workflow.Workflow{Version: 1, Name: "Answer", Entry: "result", Nodes: []workflow.Node{{ID: "result", Block: "emit", Inputs: map[string]any{"name": "answer", "value": float64(42)}}}, Edges: []workflow.Edge{}} +} + +func TestHTTPRequiresOwnOriginAndSession(t *testing.T) { + handler, err := NewHandler(nil, ServerConfig{}, "http://127.0.0.1:8090") + if err != nil { + t.Fatal(err) + } + request := func(method, path, host, origin, token string, body []byte) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(method, "http://127.0.0.1:8090"+path, bytes.NewReader(body)) + r.Host = host + r.Header.Set("Origin", origin) + r.Header.Set("X-Studio-Token", token) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + for _, pair := range [][2]string{{"evil.example:8090", ""}, {"127.0.0.1:8090", "https://evil.example"}} { + if got := request("GET", "/studio/api/session", pair[0], pair[1], "", nil).Code; got != 403 { + t.Fatalf("foreign origin returned %d", got) + } + } + session := request("GET", "/studio/api/session", "127.0.0.1:8090", "", "", nil) + var auth struct { + Token string `json:"token"` + } + if err := json.Unmarshal(session.Body.Bytes(), &auth); err != nil || auth.Token == "" { + t.Fatal("session did not provide a token", err) + } + body, _ := json.Marshal(sampleWorkflow()) + if got := request("POST", "/studio/api/validate", "127.0.0.1:8090", "", "", body).Code; got != 403 { + t.Fatalf("missing token returned %d", got) + } + valid := request("POST", "/studio/api/validate", "127.0.0.1:8090", "http://127.0.0.1:8090", auth.Token, body) + if valid.Code != 200 || !strings.Contains(valid.Body.String(), `"diagnostics":[]`) { + t.Fatalf("validate: %d %s", valid.Code, valid.Body) + } + if got := request("POST", "/studio/api/build", "127.0.0.1:8090", "", auth.Token, body).Code; got != 503 { + t.Fatalf("missing compiler returned %d", got) + } + oversized := request("POST", "/studio/api/validate", "127.0.0.1:8090", "", auth.Token, bytes.Repeat([]byte(" "), MaxWorkflowBytes+1)) + if oversized.Code == 200 { + t.Fatal("accepted oversized workflow") + } +} + +func TestHTTPProxyIsFixedAndBoundsBodies(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/account" { + t.Errorf("unexpected path %s", r.URL.Path) + } + if r.Header.Get("X-Studio-Token") != "" { + t.Error("studio token forwarded upstream") + } + cookie, err := r.Cookie("session_csrf") + if err != nil || cookie.Value != "csrf-value" || r.Header.Get("X-Debuglet-CSRF") != "csrf-value" { + t.Error("CSRF cookie/header not preserved") + } + http.SetCookie(w, &http.Cookie{Name: "session_csrf", Value: "next", Domain: "upstream.example", Path: "/"}) + _, _ = io.WriteString(w, `{"ok":true}`) + })) + defer upstream.Close() + handler, err := NewHandler(nil, ServerConfig{Dispatcher: upstream.URL}, "http://127.0.0.1:8090") + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest("GET", "http://127.0.0.1:8090/api/account", nil) + request.Header.Set("X-Studio-Token", "private") + request.Header.Set("X-Debuglet-CSRF", "csrf-value") + request.AddCookie(&http.Cookie{Name: "session_csrf", Value: "csrf-value"}) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != 200 || response.Body.String() != `{"ok":true}` { + t.Fatalf("proxy: %d %s", response.Code, response.Body) + } + for _, cookie := range response.Result().Cookies() { + if cookie.Domain != "" { + t.Error("upstream cookie domain retained") + } + } + for _, target := range []string{"file:///etc/passwd", "http://user:password@example.test", "http://example.test/other", "https://dispatcher.example.test", "http://192.168.1.5:8000"} { + if _, err := NewHandler(nil, ServerConfig{Dispatcher: target}, "http://127.0.0.1:8090"); err == nil { + t.Errorf("accepted %s", target) + } + } +} + +func TestWorkflowDecoderRejectsUnknownTrailingAndOversizedJSON(t *testing.T) { + body, _ := json.Marshal(sampleWorkflow()) + for _, data := range [][]byte{append(append([]byte{}, body...), []byte(" {}")...), []byte(`{"version":1,"shell":"rm"}`), append(append([]byte{}, body...), bytes.Repeat([]byte(" "), MaxWorkflowBytes)...)} { + if _, err := DecodeWorkflow(bytes.NewReader(data)); err == nil { + t.Fatal("invalid workflow encoding accepted") + } + } +} + +func TestMCPPreservesEditableWorkflowAndLimitsFilesystem(t *testing.T) { + workspace := t.TempDir() + server, closeServer, err := MCPServer(nil, workspace) + if err != nil { + t.Fatal(err) + } + defer closeServer() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + serverTransport, clientTransport := mcp.NewInMemoryTransports() + ss, err := server.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatal(err) + } + defer ss.Close() + client := mcp.NewClient(&mcp.Implementation{Name: "studio-test", Version: "1"}, nil) + cs, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatal(err) + } + defer cs.Close() + call := func(name string, args any) *mcp.CallToolResult { + t.Helper() + result, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: name, Arguments: args}) + if err != nil { + t.Fatal(err) + } + return result + } + for _, name := range []string{"catalog", "validate"} { + args := any(map[string]any{}) + if name == "validate" { + args = workflowInput{sampleWorkflow()} + } + if result := call(name, args); result.IsError { + t.Fatalf("%s: %+v", name, result) + } + } + if result := call("save_workflow", savedInput{"example", sampleWorkflow()}); result.IsError { + t.Fatal(result) + } + result := call("get_workflow", nameInput{"example"}) + if result.IsError { + t.Fatal(result) + } + data, err := os.ReadFile(filepath.Join(workspace, "example.workflow.json")) + if err != nil { + t.Fatal(err) + } + expected, _ := json.Marshal(sampleWorkflow()) + if !bytes.Equal(data, expected) { + t.Fatal("MCP changed the saved graph") + } + if result := call("save_workflow", savedInput{"../outside", sampleWorkflow()}); !result.IsError { + t.Fatal("MCP accepted a path") + } + if result := call("read_artifact", artifactInput{"../outside"}); !result.IsError { + t.Fatal("MCP read an unknown artifact") + } + if result := call("build", workflowInput{sampleWorkflow()}); !result.IsError { + t.Fatal("MCP built without a compiler") + } + outside := t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "private.json"), expected, 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(outside, "private.json"), filepath.Join(workspace, "linked.workflow.json")); err != nil { + t.Fatal(err) + } + if result := call("get_workflow", nameInput{"linked"}); !result.IsError { + t.Fatal("MCP followed a symlink outside its workspace") + } +} + +func TestMCPBuildProducesOfflineWASIBundle(t *testing.T) { + sdk, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + builder, err := NewBuilder(sdk) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + server, closeServer, err := MCPServer(builder, t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer closeServer() + serverTransport, clientTransport := mcp.NewInMemoryTransports() + ss, err := server.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatal(err) + } + defer ss.Close() + client := mcp.NewClient(&mcp.Implementation{Name: "build-test", Version: "1"}, nil) + cs, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatal(err) + } + defer cs.Close() + built, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: "build", Arguments: workflowInput{sampleWorkflow()}}) + if err != nil { + t.Fatal(err) + } + if built.IsError { + t.Fatal(built.Content) + } + metaJSON, err := json.Marshal(built.StructuredContent) + if err != nil { + t.Fatal(err) + } + var meta metadata + if err := json.Unmarshal(metaJSON, &meta); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(meta.BundlePath) + if err != nil { + t.Fatal(err) + } + var bundle Bundle + if err := json.Unmarshal(data, &bundle); err != nil { + t.Fatal(err) + } + read, err := cs.CallTool(ctx, &mcp.CallToolParams{Name: "read_artifact", Arguments: artifactInput{meta.ArtifactID}}) + if err != nil || read.IsError { + t.Fatalf("read original artifact: %v %+v", err, read) + } + if !bytes.HasPrefix(bundle.Wasm, []byte{0, 'a', 's', 'm', 1, 0, 0, 0}) { + t.Fatal("not a WASM module") + } + if bundle.WasmSHA256 != digest(bundle.Wasm) || bundle.WorkflowSHA256 != digest([]byte(bundle.WorkflowJSON)) { + t.Fatal("bundle digest mismatch") + } + wf, err := DecodeWorkflow(strings.NewReader(bundle.WorkflowJSON)) + if err != nil { + t.Fatal(err) + } + program, err := workflow.Generate(wf) + if err != nil { + t.Fatal(err) + } + if program.Source != bundle.Source { + t.Fatal("bundle source differs from workflow") + } + bundle.Workflow.Name = "Changed independently of canonical graph" + changed, err := json.Marshal(bundle) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(meta.BundlePath, changed, 0600); err != nil { + t.Fatal(err) + } + read, err = cs.CallTool(ctx, &mcp.CallToolParams{Name: "read_artifact", Arguments: artifactInput{meta.ArtifactID}}) + if err != nil || !read.IsError { + t.Fatalf("modified artifact not rejected: %v %+v", err, read) + } + stopped, stop := context.WithCancel(context.Background()) + stop() + if _, err := builder.Build(stopped, sampleWorkflow()); err == nil { + t.Fatal("cancelled build succeeded") + } +} + +func TestProxyTrustsOnlyConfiguredCA(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + defer upstream.Close() + caFile := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(caFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: upstream.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + for _, fixture := range []struct { + ca string + want int + }{{"", 502}, {caFile, 204}} { + handler, err := NewHandler(nil, ServerConfig{Dispatcher: upstream.URL, CAFile: fixture.ca}, "http://127.0.0.1:8090") + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest("GET", "http://127.0.0.1:8090/api/account", nil)) + if response.Code != fixture.want { + t.Fatalf("proxy CA=%q status=%d want=%d", fixture.ca, response.Code, fixture.want) + } + } +} diff --git a/pkg/workflow/catalog.go b/pkg/workflow/catalog.go new file mode 100644 index 00000000..c1507401 --- /dev/null +++ b/pkg/workflow/catalog.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +func input(name, kind string) Port { return Port{Name: name, Type: kind, Required: true} } +func optional(name, kind string, value any) Port { return Port{Name: name, Type: kind, Default: value} } +func output(name, kind string) Port { return Port{Name: name, Type: kind} } + +// Catalog returns independent descriptions of the supported primitives. +func Catalog() []Block { + return []Block{ + {"http", "HTTP request", "Network", "Send one HTTP request. ok describes transport completion; compare status to choose an HTTP success policy. Body is text; request headers are string values. Response headers use lowercase names and lists of values. Redirects are returned, not followed.", false, []Port{input("url", "string"), optional("method", "string", "GET"), optional("body", "string", ""), optional("headers", "object", map[string]any{}), optional("timeout_ms", "number", float64(2000))}, []Port{output("ok", "boolean"), output("error", "string"), output("status", "number"), output("body", "string"), output("headers", "object"), output("duration_ms", "number")}, []string{"next"}}, + {"tcp_connect", "TCP connect", "Network", "Measure one TCP connection to host:port.", false, []Port{input("target", "string"), optional("timeout_ms", "number", float64(2000))}, []Port{output("ok", "boolean"), output("error", "string"), output("duration_ms", "number")}, []string{"next"}}, + {"ping", "Ping", "Network", "Send up to ten ICMPv4 echo requests. rtts_ms contains successful replies; loss is a percentage.", false, []Port{input("target", "string"), optional("count", "number", float64(3)), optional("timeout_ms", "number", float64(2000))}, []Port{output("ok", "boolean"), output("error", "string"), output("rtts_ms", "numbers"), output("sent", "number"), output("received", "number"), output("loss", "number")}, []string{"next"}}, + {"send_message", "Send message", "Experiment", "Send a bounded plaintext JSON message to a participant. Success means the socket write completed, not that the peer processed it. Requires an experiment.", false, []Port{input("destination", "string"), input("channel", "string"), input("value", "any"), optional("request_id", "string", ""), optional("timeout_ms", "number", float64(2000))}, []Port{output("ok", "boolean"), output("error", "string"), output("request_id", "string")}, []string{"next"}}, + {"await_message", "Await message", "Experiment", "Wait for one selected incoming message or reply until its deadline. Test ok before using value. Requires an experiment.", false, []Port{input("source", "string"), input("channel", "string"), optional("request_id", "string", ""), optional("timeout_ms", "number", float64(2000))}, []Port{output("ok", "boolean"), output("error", "string"), output("value", "any"), output("request_id", "string"), output("source", "string")}, []string{"next"}}, + + {"constant", "Constant", "Data", "Provide a JSON value.", true, []Port{input("value", "any")}, []Port{output("value", "any")}, []string{}}, + {"field", "Get field", "Data", "Read one named field from an object; missing fields are errors.", true, []Port{input("object", "object"), input("name", "string")}, []Port{output("value", "any")}, []string{}}, + {"parse_json", "Parse JSON", "Data", "Parse a JSON string into a value.", true, []Port{input("text", "string")}, []Port{output("value", "any")}, []string{}}, + {"stringify_json", "Stringify JSON", "Data", "Encode a JSON value as text, for example an HTTP request body.", true, []Port{input("value", "any")}, []Port{output("value", "string")}, []string{}}, + {"set_field", "Set field", "Data", "Return a copy of an object with one field added or replaced.", true, []Port{input("object", "object"), input("name", "string"), input("value", "any")}, []Port{output("value", "object")}, []string{}}, + {"list_item", "List item", "Data", "Read one list item at a zero-based integer index. An out-of-range index is an error.", true, []Port{input("values", "any"), input("index", "number")}, []Port{output("value", "any")}, []string{}}, + {"append", "Append item", "Data", "Return a new list with one item appended.", true, []Port{input("values", "any"), input("value", "any")}, []Port{output("value", "any")}, []string{}}, + {"concat", "Concatenate", "Text", "Join two strings without adding a separator.", true, []Port{input("a", "string"), input("b", "string")}, []Port{output("value", "string")}, []string{}}, + {"contains", "Contains", "Text", "Case-sensitive substring search. An empty substring matches every string.", true, []Port{input("text", "string"), input("substring", "string")}, []Port{output("value", "boolean")}, []string{}}, + {"split", "Split", "Text", "Split text at each occurrence of a nonempty separator, keeping empty pieces.", true, []Port{input("text", "string"), input("separator", "string")}, []Port{output("value", "strings")}, []string{}}, + {"join", "Join", "Text", "Join a list of strings with a separator. An empty list produces empty text.", true, []Port{input("values", "strings"), input("separator", "string")}, []Port{output("value", "string")}, []string{}}, + {"median", "Median", "Analysis", "Median of a nonempty number list; an even-sized list uses the mean of its middle pair.", true, []Port{input("values", "numbers")}, []Port{output("value", "number")}, []string{}}, + {"percentile", "Percentile", "Analysis", "Percentile from 0 to 100 of a nonempty number list, with linear interpolation at rank p/100 × (n−1).", true, []Port{input("values", "numbers"), input("percentile", "number")}, []Port{output("value", "number")}, []string{}}, + {"mean", "Mean", "Analysis", "Arithmetic mean of a nonempty number list.", true, []Port{input("values", "numbers")}, []Port{output("value", "number")}, []string{}}, + {"min", "Minimum", "Analysis", "Smallest value in a nonempty number list.", true, []Port{input("values", "numbers")}, []Port{output("value", "number")}, []string{}}, + {"max", "Maximum", "Analysis", "Largest value in a nonempty number list.", true, []Port{input("values", "numbers")}, []Port{output("value", "number")}, []string{}}, + {"count", "Count", "Analysis", "Number of items in a JSON list.", true, []Port{input("values", "any")}, []Port{output("value", "number")}, []string{}}, + {"add", "Add", "Math", "Add two numbers.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "number")}, []string{}}, + {"subtract", "Subtract", "Math", "Subtract b from a.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "number")}, []string{}}, + {"multiply", "Multiply", "Math", "Multiply two numbers. Nonfinite results are errors.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "number")}, []string{}}, + {"divide", "Divide", "Math", "Divide a by b. Division by zero and nonfinite results are errors.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "number")}, []string{}}, + {"greater_than", "Greater than", "Logic", "Whether a is greater than b.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "boolean")}, []string{}}, + {"less_than", "Less than", "Logic", "Whether a is less than b.", true, []Port{input("a", "number"), input("b", "number")}, []Port{output("value", "boolean")}, []string{}}, + {"equals", "Equals", "Logic", "Compare JSON values without coercion.", true, []Port{input("a", "any"), input("b", "any")}, []Port{output("value", "boolean")}, []string{}}, + {"and", "And", "Logic", "Both conditions are true.", true, []Port{input("a", "boolean"), input("b", "boolean")}, []Port{output("value", "boolean")}, []string{}}, + {"or", "Or", "Logic", "At least one condition is true.", true, []Port{input("a", "boolean"), input("b", "boolean")}, []Port{output("value", "boolean")}, []string{}}, + {"not", "Not", "Logic", "Invert a condition.", true, []Port{input("value", "boolean")}, []Port{output("value", "boolean")}, []string{}}, + {"if", "If", "Control", "Run exactly the selected branch.", false, []Port{input("condition", "boolean")}, []Port{}, []string{"true", "false"}}, + {"repeat", "Repeat", "Control", "Run the body sequentially up to 100 times, then continue on done. The body ends at an unconnected control output.", false, []Port{input("count", "number")}, []Port{output("iteration", "number")}, []string{"body", "done"}}, + {"wait", "Wait", "Control", "Wait between actions, at most 60 seconds per call.", false, []Port{input("duration_ms", "number")}, []Port{}, []string{"next"}}, + {"emit", "Emit result", "Results", "Publish a named JSON result.", false, []Port{input("name", "string"), input("value", "any")}, []Port{output("name", "string"), output("value", "any")}, []string{"next"}}, + } +} + +func catalogFor(w Workflow) map[string]Block { + m := map[string]Block{} + for _, b := range Catalog() { + m[b.ID] = b + } + for _, b := range w.CustomBlocks { + m[b.ID] = Block{b.ID, b.Label, "Custom", "Custom Go action", false, b.Inputs, b.Outputs, []string{"next"}} + } + return m +} diff --git a/pkg/workflow/experiment.go b/pkg/workflow/experiment.go new file mode 100644 index 00000000..f387e90b --- /dev/null +++ b/pkg/workflow/experiment.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "strings" +) + +// Experiment groups independently executed workflows. Executor selection and +// policy remain submission choices, not embedded guest authority. +type Experiment struct { + Version int `json:"version"` + Kind string `json:"kind"` + Name string `json:"name"` + Participants []Participant `json:"participants"` + Channels []Channel `json:"channels"` +} + +// ExperimentContext is injected by the experiment builder and covered by the +// participant workflow digest. It supplies no executor policy permissions. +type ExperimentContext struct { + Participant string `json:"participant"` + Fingerprint string `json:"fingerprint"` + Participants []string `json:"participants"` + Channels []Channel `json:"channels"` +} +type Participant struct { + ID string `json:"id"` + Label string `json:"label,omitempty"` + Workflow Workflow `json:"workflow"` +} + +// ResponseType makes the channel request/reply; otherwise it is one-way. +type Channel struct { + ID string `json:"id"` + From string `json:"from"` + To string `json:"to"` + Type string `json:"type"` + ResponseType string `json:"response_type,omitempty"` +} +type ExperimentDiagnostic struct { + Participant string `json:"participant,omitempty"` + Diagnostic +} + +// ValidateExperiment checks membership and communication boundaries in addition +// to each ordinary workflow. Runtime checks still cover dynamically computed data. +func ValidateExperiment(e Experiment) []ExperimentDiagnostic { + var ds []ExperimentDiagnostic + add := func(participant, node, port, message string) { + ds = append(ds, ExperimentDiagnostic{participant, Diagnostic{node, port, message}}) + } + if e.Version != 1 || e.Kind != "experiment" { + add("", "", "", "experiment requires version 1 and kind experiment") + } + if strings.TrimSpace(e.Name) == "" || len(e.Name) > 128 { + add("", "", "", "name must contain 1–128 characters") + } + if len(e.Participants) < 2 || len(e.Participants) > 3 { + add("", "", "", "an experiment requires 2–3 participants") + return ds + } + members := map[string]bool{} + for _, p := range e.Participants { + if !identifier.MatchString(p.ID) || members[p.ID] { + add(p.ID, "", "", "participant id must be valid and unique") + } + if len(p.Label) > 128 { + add(p.ID, "", "", "participant label exceeds 128 characters") + } + members[p.ID] = true + if p.Workflow.Experiment != nil { + add(p.ID, "", "", "participant source must not contain injected experiment context") + } + for _, d := range Validate(p.Workflow) { + ds = append(ds, ExperimentDiagnostic{p.ID, d}) + } + } + if len(e.Channels) > 64 { + add("", "", "", "experiment has more than 64 channels") + return ds + } + channels := map[string]Channel{} + for _, c := range e.Channels { + if !identifier.MatchString(c.ID) || channels[c.ID].ID != "" { + add("", "", "", "channel id must be valid and unique") + } + if !members[c.From] || !members[c.To] || c.From == c.To { + add("", "", "", "channel endpoints must be distinct experiment participants") + } + if !slices.Contains(valueTypes, c.Type) || (c.ResponseType != "" && !slices.Contains(valueTypes, c.ResponseType)) { + add("", "", "", "unsupported channel value type") + } + channels[c.ID] = c + } + for _, p := range e.Participants { + wired := map[Endpoint]bool{} + for _, edge := range p.Workflow.Edges { + if edge.Kind == "data" { + wired[edge.To] = true + } + } + for _, n := range p.Workflow.Nodes { + if n.Block != "send_message" && n.Block != "await_message" { + continue + } + endpoint := "destination" + if n.Block == "await_message" { + endpoint = "source" + } + for _, field := range []string{endpoint, "channel"} { + if wired[Endpoint{n.ID, field}] { + add(p.ID, n.ID, field, "select a declared participant and channel directly; these fields cannot be wired") + } + } + peer, _ := n.Inputs[endpoint].(string) + id, _ := n.Inputs["channel"].(string) + c, found := channels[id] + if !found { + add(p.ID, n.ID, "channel", "select an experiment channel") + continue + } + from, to := p.ID, peer + if n.Block == "await_message" { + from, to = peer, p.ID + } + forward := from == c.From && to == c.To + reverse := from == c.To && to == c.From && c.ResponseType != "" + if !forward && !reverse { + add(p.ID, n.ID, endpoint, "participant does not match the declared channel direction") + continue + } + request, _ := n.Inputs["request_id"].(string) + hasRequest := request != "" || wired[Endpoint{n.ID, "request_id"}] + if reverse && !hasRequest { + add(p.ID, n.ID, "request_id", "a response must use the request_id from the earlier Send or Await") + } + if forward && hasRequest { + add(p.ID, n.ID, "request_id", "leave request_id empty for a new message or request") + } + if n.Block == "send_message" && !wired[Endpoint{n.ID, "value"}] { + typ := c.Type + if reverse { + typ = c.ResponseType + } + if !matches(n.Inputs["value"], typ) { + add(p.ID, n.ID, "value", "channel expects "+typ) + } + } + } + } + return ds +} + +// GenerateParticipant injects the shared readiness manifest into one ordinary +// guest. Identity and listener addresses come from the admitted group at runtime. +func GenerateParticipant(e Experiment, id string) (Workflow, Program, error) { + if ds := ValidateExperiment(e); len(ds) > 0 { + return Workflow{}, Program{}, fmt.Errorf("invalid experiment: %s", ds[0].Message) + } + raw, err := json.Marshal(e) + if err != nil { + return Workflow{}, Program{}, err + } + if len(raw) > 4<<20 { + return Workflow{}, Program{}, fmt.Errorf("experiment exceeds 4 MiB") + } + hash := sha256.Sum256(raw) + ids := make([]string, len(e.Participants)) + var selected *Workflow + for i, p := range e.Participants { + ids[i] = p.ID + if p.ID == id { + w := p.Workflow + selected = &w + } + } + if selected == nil { + return Workflow{}, Program{}, fmt.Errorf("unknown experiment participant %q", id) + } + selected.Experiment = &ExperimentContext{id, hex.EncodeToString(hash[:]), ids, e.Channels} + encoded, err := json.Marshal(selected.Experiment) + if err != nil { + return Workflow{}, Program{}, err + } + program, err := generate(*selected, string(encoded)) + return *selected, program, err +} diff --git a/pkg/workflow/experiment_test.go b/pkg/workflow/experiment_test.go new file mode 100644 index 00000000..c80e646d --- /dev/null +++ b/pkg/workflow/experiment_test.go @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func sampleExperiment(t *testing.T) Experiment { + t.Helper() + raw, err := os.ReadFile("../../examples/workflows/experiments/three-node.json") + if err != nil { + t.Fatal(err) + } + var e Experiment + if err = json.Unmarshal(raw, &e); err != nil { + t.Fatal(err) + } + return e +} +func TestExperimentContextIsPartOfProgramIdentity(t *testing.T) { + e := sampleExperiment(t) + if ds := ValidateExperiment(e); len(ds) > 0 { + t.Fatal(ds) + } + wf, p, err := GenerateParticipant(e, "A") + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(wf) + hash := sha256.Sum256(raw) + if p.WorkflowSHA256 != hex.EncodeToString(hash[:]) { + t.Fatal("program identity omits injected context") + } + if wf.Experiment == nil || wf.Experiment.Participant != "A" || !strings.Contains(p.Source, "guest.JoinExperiment") || !strings.Contains(p.Source, "experiment.Send") || !strings.Contains(p.Source, "experiment.Await") { + t.Fatal("missing participant setup") + } + e.Name = "Different experiment" + _, changed, err := GenerateParticipant(e, "A") + if err != nil { + t.Fatal(err) + } + if p.WorkflowSHA256 == changed.WorkflowSHA256 { + t.Fatal("different experiment has same program identity") + } + if _, err := Generate(wf); err == nil { + t.Fatal("standalone builder accepted injected context") + } + if _, err := Generate(e.Participants[0].Workflow); err == nil { + t.Fatal("standalone builder accepted message blocks") + } +} +func TestExperimentRejectsInvalidMembershipAndChannels(t *testing.T) { + cases := map[string]func(*Experiment){ + "duplicate participant": func(e *Experiment) { e.Participants[1].ID = "A" }, + "unknown destination": func(e *Experiment) { e.Channels[0].To = "missing" }, + "self channel": func(e *Experiment) { e.Channels[0].To = "A" }, + "duplicate channel": func(e *Experiment) { e.Channels = append(e.Channels, e.Channels[0]) }, + "wrong type": func(e *Experiment) { e.Channels[0].Type = "string" }, + "wrong peer": func(e *Experiment) { e.Participants[0].Workflow.Nodes[0].Inputs["destination"] = "C" }, + "response without request": func(e *Experiment) { + e.Channels[0].From = "B" + e.Channels[0].To = "A" + e.Channels[0].ResponseType = "number" + }, + "injected context": func(e *Experiment) { e.Participants[0].Workflow.Experiment = &ExperimentContext{} }, + "dynamic destination": func(e *Experiment) { + w := &e.Participants[0].Workflow + delete(w.Nodes[0].Inputs, "destination") + w.Nodes = append(w.Nodes, Node{ID: "peer", Block: "constant", Inputs: map[string]any{"value": "B"}}) + w.Edges = append(w.Edges, edge("data", "peer", "value", "send", "destination")) + }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + e := sampleExperiment(t) + change(&e) + if len(ValidateExperiment(e)) == 0 { + t.Fatal("accepted invalid experiment") + } + }) + } +} + +func TestRequestReplyExamplePreservesCorrelationConnections(t *testing.T) { + raw, err := os.ReadFile("../../examples/workflows/experiments/request-reply.json") + if err != nil { + t.Fatal(err) + } + var e Experiment + if err := json.Unmarshal(raw, &e); err != nil { + t.Fatal(err) + } + for _, participant := range e.Participants { + _, program, err := GenerateParticipant(e, participant.ID) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(program.Source, `("request_id")`) { + t.Fatalf("%s does not consume the preceding request handle", participant.ID) + } + } + // Removing the wire must not silently turn the response into a new request. + w := &e.Participants[0].Workflow + w.Edges = w.Edges[:len(w.Edges)-1] + if len(ValidateExperiment(e)) == 0 { + t.Fatal("accepted a reply wait without its request ID") + } +} + +func TestGeneratedExperimentRetainsTopologyBeforeReadiness(t *testing.T) { + if testing.Short() { + t.Skip("generated program compilation") + } + e := sampleExperiment(t) + e.Channels[0].ResponseType = "number" + w, program, err := GenerateParticipant(e, "A") + if err != nil { + t.Fatal(err) + } + source := filepath.Join(t.TempDir(), "main.go") + if err := os.WriteFile(source, []byte(program.Source), 0600); err != nil { + t.Fatal(err) + } + run := func(stdout *bytes.Buffer, output *os.File) string { + t.Helper() + cmd := exec.Command("go", "run", source) + cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOSUMDB=off") + var stderr bytes.Buffer + cmd.Stderr = &stderr + if output != nil { + cmd.Stdout = output + } else { + cmd.Stdout = stdout + } + if err := cmd.Run(); err == nil { + t.Fatal("native execution unexpectedly joined an experiment") + } + return stderr.String() + } + var stdout bytes.Buffer + if stderr := run(&stdout, nil); !strings.Contains(stderr, "SDK calls only work inside a wasip1 guest") { + t.Fatalf("program did not reach readiness: %s", stderr) + } + if bytes.Count(stdout.Bytes(), []byte{'\n'}) != 1 { + t.Fatalf("expected exactly one complete manifest, got %q", stdout.String()) + } + var manifest struct { + Schema string `json:"schema"` + WorkflowSHA256 string `json:"workflow_sha256"` + ExperimentSHA256 string `json:"experiment_sha256"` + Participant string `json:"participant"` + Participants []string `json:"participants"` + Channels []Channel `json:"channels"` + } + if err := json.Unmarshal(stdout.Bytes(), &manifest); err != nil { + t.Fatal(err) + } + if manifest.Schema != "debuglet.experiment.v1" || manifest.WorkflowSHA256 != program.WorkflowSHA256 || + manifest.ExperimentSHA256 != w.Experiment.Fingerprint || manifest.Participant != "A" || + !reflect.DeepEqual(manifest.Participants, w.Experiment.Participants) || + !reflect.DeepEqual(manifest.Channels, e.Channels) { + t.Fatalf("manifest differs from the compiled experiment: %+v", manifest) + } + if len(stdout.Bytes()) > 65537 { + t.Fatal("manifest exceeds one bounded output record") + } + t.Run("output failure stops before readiness", func(t *testing.T) { + full, err := os.OpenFile("/dev/full", os.O_WRONLY, 0) + if err != nil { + t.Skip("/dev/full is unavailable") + } + defer full.Close() + stderr := run(nil, full) + if strings.Contains(stderr, "SDK calls only work inside a wasip1 guest") || + !strings.Contains(stderr, "no space left on device") { + t.Fatalf("failed manifest write did not stop execution: %s", stderr) + } + }) +} diff --git a/pkg/workflow/generate.go b/pkg/workflow/generate.go new file mode 100644 index 00000000..0be0eaf5 --- /dev/null +++ b/pkg/workflow/generate.go @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "net" + "net/url" + "slices" + "strconv" +) + +// Generate produces ordinary Go source for one WASI guest. Destination metadata +// contains static hints only; the normal submission policy remains authoritative. +func Generate(w Workflow) (Program, error) { + if w.Experiment != nil { + return Program{}, fmt.Errorf("build experiment context through the whole experiment") + } + return generate(w, "") +} + +func generate(w Workflow, experimentConfig string) (Program, error) { + encoded, err := json.Marshal(w) + if err != nil { + return Program{}, err + } + if len(encoded) > 1024*1024 { + return Program{}, fmt.Errorf("workflow exceeds 1 MiB") + } + hash := sha256.Sum256(encoded) + digest := hex.EncodeToString(hash[:]) + // JSON is also the boundary for callers constructing Workflow directly in Go. + if err = json.Unmarshal(encoded, &w); err != nil { + return Program{}, err + } + if ds := Validate(w); len(ds) > 0 { + return Program{}, fmt.Errorf("invalid workflow: %s", ds[0].Message) + } + if experimentConfig == "" { + for _, n := range w.Nodes { + if n.Block == "send_message" || n.Block == "await_message" { + return Program{}, fmt.Errorf("%s requires building the whole experiment", n.Block) + } + } + } + catalog := catalogFor(w) + indexes := map[string]int{} + nodes := map[string]Node{} + for i, n := range w.Nodes { + indexes[n.ID] = i + nodes[n.ID] = n + } + data := map[Endpoint]Endpoint{} + control := map[Endpoint]string{} + adj := map[string][]string{} + for _, e := range w.Edges { + if e.Kind == "data" { + data[e.To] = e.From + } else { + control[e.From] = e.To.Node + adj[e.From.Node] = append(adj[e.From.Node], e.To.Node) + } + } + custom := map[string]int{} + for i, c := range w.CustomBlocks { + custom[c.ID] = i + } + var out bytes.Buffer + out.WriteString("// Generated measurement program. Edit its workflow source to regenerate.\npackage main\nimport(\"encoding/json\";\"fmt\";\"os\";\"github.com/netsec-ethz/debuglet/pkg/workflow/guest\";") + for _, path := range customImports(w) { + fmt.Fprintf(&out, "%q;", path) + } + fmt.Fprintf(&out, ")\nvar runtime=guest.New(%q)\n", digest) + out.WriteString("func literal(s string)any{var v any;if err:=json.Unmarshal([]byte(s),&v);err!=nil{panic(err)};return v}\n") + if experimentConfig == "" { + fmt.Fprintf(&out, "func main(){if err:=run%d();err!=nil{fmt.Fprintln(os.Stderr,err);os.Exit(1)}}\n", indexes[w.Entry]) + } else { + manifest, err := json.Marshal(struct { + Schema string `json:"schema"` + WorkflowSHA256 string `json:"workflow_sha256"` + ExperimentSHA256 string `json:"experiment_sha256"` + Participant string `json:"participant"` + Participants []string `json:"participants"` + Channels []Channel `json:"channels"` + }{"debuglet.experiment.v1", digest, w.Experiment.Fingerprint, w.Experiment.Participant, w.Experiment.Participants, w.Experiment.Channels}) + if err != nil { + return Program{}, err + } + if len(manifest) > 65536 { + return Program{}, fmt.Errorf("experiment manifest exceeds 64 KiB") + } + out.WriteString("var experiment *guest.Experiment\n") + // Retain the declared graph even if readiness fails before the first block. + fmt.Fprintf(&out, "func main(){if _,err:=fmt.Fprintln(os.Stdout,%q);err!=nil{fmt.Fprintln(os.Stderr,err);os.Exit(1)};var err error;experiment,err=guest.JoinExperiment(%q);if err==nil{err=run%d()};if err!=nil{fmt.Fprintln(os.Stderr,err);os.Exit(1)}}\n", string(manifest), experimentConfig, indexes[w.Entry]) + } + for i, c := range w.CustomBlocks { + fmt.Fprintf(&out, "// Custom block %s\nfunc custom%d(inputs map[string]any)(map[string]any,error){\n%s\n}\n", c.ID, i, c.Code) + } + for i, n := range w.Nodes { + b := catalog[n.Block] + fmt.Fprintf(&out, "// Inputs for node %s (%s).\nfunc inputs%d()(map[string]any,error){\nin:=map[string]any{}\n", n.ID, n.Block, i) + assigned := []Port{} + for _, p := range b.Inputs { + if from, ok := data[Endpoint{n.ID, p.Name}]; ok { + fmt.Fprintf(&out, "{value,err:=value%d(%q);if err!=nil{return nil,fmt.Errorf(%q,err)};in[%q]=value}\n", indexes[from.Node], from.Port, "input "+p.Name+": %w", p.Name) + assigned = append(assigned, p) + continue + } + value, ok := n.Inputs[p.Name] + if !ok { + value = p.Default + if value == nil { + continue + } + } + v, _ := json.Marshal(value) + fmt.Fprintf(&out, "in[%q]=literal(%q)\n", p.Name, string(v)) + assigned = append(assigned, p) + } + fmt.Fprintf(&out, "if err:=guest.Check(in,%s);err!=nil{return nil,err};return in,nil}\n", typeMap(assigned)) + fmt.Fprintf(&out, "func value%d(port string)(any,error){\n", i) + if b.Pure { + fmt.Fprintf(&out, "in,err:=inputs%d();if err!=nil{return nil,runtime.Fail(%q,%q,err)}\nvalues,err:=guest.Pure(%q,in);if err!=nil{return nil,runtime.Fail(%q,%q,err)}\nif err=guest.Check(values,%s);err!=nil{return nil,runtime.Fail(%q,%q,err)}\nif err=runtime.Record(%q,%q,values);err!=nil{return nil,err}\n", i, n.ID, n.Block, n.Block, n.ID, n.Block, typeMap(b.Outputs), n.ID, n.Block, n.ID, n.Block) + } + fmt.Fprintf(&out, "return runtime.Output(%q,port)}\n", n.ID) + if b.Pure { + continue + } + fmt.Fprintf(&out, "// Execute node %s (%s).\nfunc run%d()error{\nif err:=runtime.Step();err!=nil{return runtime.Fail(%q,%q,err)}\nin,err:=inputs%d();if err!=nil{return runtime.Fail(%q,%q,err)}\n", n.ID, n.Block, i, n.ID, n.Block, i, n.ID, n.Block) + callNext := func(port string) { + if next, ok := control[Endpoint{n.ID, port}]; ok { + fmt.Fprintf(&out, "return run%d()\n", indexes[next]) + } else { + out.WriteString("return nil\n") + } + } + switch n.Block { + case "if": + fmt.Fprintf(&out, "if err=runtime.Record(%q,%q,map[string]any{});err!=nil{return err}\nif in[\"condition\"].(bool){\n", n.ID, n.Block) + callNext("true") + out.WriteString("}\n") + callNext("false") + case "repeat": + fmt.Fprintf(&out, "count,err:=guest.RepeatCount(in[\"count\"]);if err!=nil{return runtime.Fail(%q,%q,err)}\nfor iteration:=1;iteration<=count;iteration++{\n", n.ID, n.Block) + body := control[Endpoint{n.ID, "body"}] + bodyNodes := reachable(body, adj) + ids := make([]string, 0, len(bodyNodes)) + for id := range bodyNodes { + ids = append(ids, id) + } + slices.Sort(ids) + out.WriteString("runtime.Forget(") + for _, id := range ids { + fmt.Fprintf(&out, "%q,", id) + } + out.WriteString(")\n") + fmt.Fprintf(&out, "if err=runtime.Record(%q,%q,map[string]any{\"iteration\":float64(iteration)});err!=nil{return err}\nif err=run%d();err!=nil{return err}\n}\n", n.ID, n.Block, indexes[body]) + callNext("done") + case "wait": + fmt.Fprintf(&out, "if err=guest.Wait(in[\"duration_ms\"]);err!=nil{return runtime.Fail(%q,%q,err)}\nif err=runtime.Record(%q,%q,map[string]any{});err!=nil{return err}\n", n.ID, n.Block, n.ID, n.Block) + callNext("next") + default: + switch n.Block { + case "send_message": + out.WriteString("values,err:=experiment.Send(in)\n") + case "await_message": + out.WriteString("values,err:=experiment.Await(in)\n") + case "http": + out.WriteString("values,err:=guest.HTTP(in)\n") + case "tcp_connect": + out.WriteString("values,err:=guest.TCPConnect(in)\n") + case "ping": + out.WriteString("values,err:=guest.Ping(in)\n") + case "emit": + out.WriteString("values:=in\n") + default: + fmt.Fprintf(&out, "values,err:=guest.RunCustom(custom%d,in)\n", custom[n.Block]) + } + fmt.Fprintf(&out, "if err!=nil{return runtime.Fail(%q,%q,err)}\nif err=guest.Check(values,%s);err!=nil{return runtime.Fail(%q,%q,err)}\nif err=runtime.Record(%q,%q,values);err!=nil{return runtime.Fail(%q,%q,err)}\n", n.ID, n.Block, typeMap(b.Outputs), n.ID, n.Block, n.ID, n.Block, n.ID, n.Block) + callNext("next") + } + out.WriteString("}\n") + } + source, err := format.Source(out.Bytes()) + if err != nil { + return Program{}, fmt.Errorf("format generated program: %w", err) + } + program := Program{Source: string(source), WorkflowSHA256: digest, Destinations: []string{}} + for _, n := range w.Nodes { + if n.Block == "ping" { + program.RequireICMP = true + } + field := "target" + if n.Block == "http" { + field = "url" + } else if n.Block != "ping" && n.Block != "tcp_connect" { + continue + } + value := n.Inputs[field] + if from, ok := data[Endpoint{n.ID, field}]; ok { + source := nodes[from.Node] + if source.Block == "constant" { + value = source.Inputs["value"] + } else { + continue + } + } + target, ok := value.(string) + if !ok { + continue + } + if n.Block == "http" { + u, e := url.Parse(target) + if e != nil { + continue + } + target = u.Hostname() + } else if n.Block == "tcp_connect" { + host, _, e := net.SplitHostPort(target) + if e != nil { + continue + } + target = host + } + if target != "" && !slices.Contains(program.Destinations, target) { + program.Destinations = append(program.Destinations, target) + } + } + slices.Sort(program.Destinations) + return program, nil +} +func typeMap(ports []Port) string { + var out bytes.Buffer + out.WriteString("map[string]string{") + for _, p := range ports { + fmt.Fprintf(&out, "%s:%s,", strconv.Quote(p.Name), strconv.Quote(p.Type)) + } + out.WriteString("}") + return out.String() +} + +// Custom blocks can refer to a fixed set of available packages. Their bodies +// cannot add project files, compiler directives or arbitrary imports. +func customImports(w Workflow) []string { + allowed := map[string]string{"debuglet": "github.com/netsec-ethz/debuglet/pkg/debuglet", "time": "time", "strings": "strings", "math": "math"} + used := map[string]bool{} + for _, block := range w.CustomBlocks { + file, err := parser.ParseFile(token.NewFileSet(), "custom.go", "package main\nfunc custom(inputs map[string]any)(map[string]any,error){\n"+block.Code+"\n}", 0) + if err != nil { + continue + } + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if !ok { + return true + } + name, ok := selector.X.(*ast.Ident) + if ok && name.Obj == nil { + if path, found := allowed[name.Name]; found { + used[path] = true + } + } + return true + }) + } + paths := make([]string, 0, len(used)) + for path := range used { + paths = append(paths, path) + } + slices.Sort(paths) + return paths +} diff --git a/pkg/workflow/groups_test.go b/pkg/workflow/groups_test.go new file mode 100644 index 00000000..8d002a40 --- /dev/null +++ b/pkg/workflow/groups_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "encoding/json" + "math" + "strings" + "testing" +) + +func groupedWorkflow() Workflow { + w := sampleWorkflow() + w.Nodes = append(w.Nodes, Node{ID: "number", Block: "constant", Inputs: map[string]any{"value": float64(42)}}) + delete(w.Nodes[0].Inputs, "value") + w.Edges = append(w.Edges, edge("data", "number", "value", "result", "value")) + w.Groups = []Group{{ID: "answer", Label: "Produce answer", NodeIDs: []string{"number", "result"}, Position: &Position{X: 100, Y: 200}}} + return w +} + +func TestGroupingPreservesProgramExecutionAndPortableMetadata(t *testing.T) { + w := groupedWorkflow() + raw, err := json.Marshal(w) + if err != nil { + t.Fatal(err) + } + var restored Workflow + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.DisallowUnknownFields() + if err = decoder.Decode(&restored); err != nil { + t.Fatal(err) + } + if len(restored.Groups) != 1 || restored.Groups[0].NodeIDs[1] != "result" || restored.Groups[0].Position.Y != 200 { + t.Fatalf("lost group metadata: %+v", restored.Groups) + } + grouped, err := Generate(restored) + if err != nil { + t.Fatal(err) + } + w.Groups = nil + plain, err := Generate(w) + if err != nil { + t.Fatal(err) + } + if grouped.WorkflowSHA256 == plain.WorkflowSHA256 { + t.Fatal("portable graph identity must include its groups") + } + if strings.ReplaceAll(grouped.Source, grouped.WorkflowSHA256, plain.WorkflowSHA256) != plain.Source { + t.Fatal("grouping changed generated execution code") + } +} + +func TestGroupingRejectsAmbiguousMembership(t *testing.T) { + for name, change := range map[string]func(*Workflow){ + "missing member": func(w *Workflow) { w.Groups[0].NodeIDs[0] = "absent" }, + "duplicate member": func(w *Workflow) { w.Groups[0].NodeIDs[0] = "result" }, + "overlapping groups": func(w *Workflow) { + w.Groups = append(w.Groups, Group{ID: "other", Label: "Other", NodeIDs: []string{"number", "result"}}) + }, + "duplicate group id": func(w *Workflow) { w.Groups = append(w.Groups, w.Groups[0]) }, + "node id collision": func(w *Workflow) { w.Groups[0].ID = "result" }, + "invalid group id": func(w *Workflow) { w.Groups[0].ID = "../group" }, + "empty group": func(w *Workflow) { w.Groups[0].NodeIDs = nil }, + "single member": func(w *Workflow) { w.Groups[0].NodeIDs = []string{"result"} }, + "blank label": func(w *Workflow) { w.Groups[0].Label = " " }, + "long label": func(w *Workflow) { w.Groups[0].Label = strings.Repeat("x", 129) }, + "invalid position": func(w *Workflow) { w.Groups[0].Position.X = math.Inf(1) }, + } { + t.Run(name, func(t *testing.T) { + w := groupedWorkflow() + change(&w) + if len(Validate(w)) == 0 { + t.Fatal("accepted ambiguous grouping") + } + }) + } +} diff --git a/pkg/workflow/guest/experiment.go b/pkg/workflow/guest/experiment.go new file mode 100644 index 00000000..ca4b3f22 --- /dev/null +++ b/pkg/workflow/guest/experiment.go @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "time" + + "github.com/netsec-ethz/debuglet/pkg/debuglet" + experimentwire "github.com/netsec-ethz/debuglet/pkg/wire/experiment" +) + +const ( + experimentFrameLimit = 8 * 1024 + experimentInboxLimit = 64 * 1024 + experimentPendingLimit = 128 +) + +type experimentConfig struct { + Participant string `json:"participant"` + Fingerprint string `json:"fingerprint"` + Participants []string `json:"participants"` + Channels []experimentChannel `json:"channels"` +} + +type experimentChannel struct { + ID string `json:"id"` + From string `json:"from"` + To string `json:"to"` + Type string `json:"type"` + ResponseType string `json:"response_type,omitempty"` +} + +type experimentEndpoint struct { + Participant string `json:"participant"` + Fingerprint string `json:"fingerprint"` + Address string `json:"address"` +} + +type experimentPeer struct { + runID, executorID, address string +} + +type experimentFrame struct { + ExperimentID string `json:"experiment_id"` + Fingerprint string `json:"fingerprint"` + From string `json:"from"` + FromRunID string `json:"from_run_id"` + To string `json:"to"` + ToRunID string `json:"to_run_id"` + Channel string `json:"channel"` + RequestID string `json:"request_id"` + Sequence uint64 `json:"sequence"` + Value any `json:"value"` +} + +type experimentMessage struct { + frame experimentFrame + size int +} + +type experimentRequest struct { + channel, peer string + ready bool +} + +type experimentSequence struct { + highest, seen uint64 +} + +// Experiment exchanges bounded plaintext messages directly between admitted +// participants. Readiness metadata identifies expected peers but does not +// authenticate their sockets: use this transport for controlled experiments, +// not confidential data. Calls are sequential; there is no background receiver. +type Experiment struct { + id, participant, fingerprint string + peers map[string]experimentPeer + channels map[string]experimentChannel + sequence uint64 + received map[string]experimentSequence + pending, requests map[string]experimentRequest + inbox []experimentMessage + inboxBytes int +} + +// JoinExperiment publishes this run's listener, checks the admitted membership, +// and waits for the group's common start before any workflow action executes. +func JoinExperiment(configJSON string) (*Experiment, error) { + var config experimentConfig + if err := experimentDecode([]byte(configJSON), &config); err != nil { + return nil, fmt.Errorf("experiment configuration: %w", err) + } + session, err := newExperiment(config) + if err != nil { + return nil, err + } + address, err := debuglet.ListenAddr() + if err != nil { + return nil, fmt.Errorf("experiment requires a reachable TCP listener: %w", err) + } + metadata, err := json.Marshal(experimentEndpoint{config.Participant, config.Fingerprint, address}) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + group, err := debuglet.Ready(ctx, metadata) + if err != nil { + return nil, fmt.Errorf("experiment readiness: %w", err) + } + if err := session.admit(group, address); err != nil { + return nil, err + } + if err := debuglet.WaitStart(ctx, group); err != nil { + return nil, fmt.Errorf("experiment start: %w", err) + } + return session, nil +} + +func newExperiment(config experimentConfig) (*Experiment, error) { + if len(config.Participants) < 2 || len(config.Participants) > 128 || len(config.Channels) > 128 { + return nil, fmt.Errorf("experiment requires 2–128 participants and at most 128 channels") + } + digest, err := hex.DecodeString(config.Fingerprint) + if err != nil || len(digest) != 32 { + return nil, fmt.Errorf("experiment fingerprint must be a SHA-256 digest") + } + session := &Experiment{ + participant: config.Participant, fingerprint: config.Fingerprint, + peers: map[string]experimentPeer{}, channels: map[string]experimentChannel{}, + received: map[string]experimentSequence{}, pending: map[string]experimentRequest{}, + requests: map[string]experimentRequest{}, + } + for _, participant := range config.Participants { + if participant == "" || len(participant) > 128 { + return nil, fmt.Errorf("invalid experiment participant") + } + if _, exists := session.peers[participant]; exists { + return nil, fmt.Errorf("duplicate experiment participant %q", participant) + } + session.peers[participant] = experimentPeer{} + } + if _, exists := session.peers[config.Participant]; !exists { + return nil, fmt.Errorf("current participant is absent from experiment") + } + for _, channel := range config.Channels { + _, from := session.peers[channel.From] + _, to := session.peers[channel.To] + _, duplicate := session.channels[channel.ID] + if channel.ID == "" || len(channel.ID) > 128 || duplicate || !from || !to || + channel.From == channel.To || !experimentType(channel.Type) || + (channel.ResponseType != "" && !experimentType(channel.ResponseType)) { + return nil, fmt.Errorf("invalid experiment channel %q", channel.ID) + } + session.channels[channel.ID] = channel + } + return session, nil +} + +func (e *Experiment) admit(group experimentwire.Experiment, address string) error { + if group.ID == "" || len(group.Participants) != len(e.peers) { + return fmt.Errorf("admitted experiment membership does not match the program") + } + members, runs, addresses := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, member := range group.Participants { + var endpoint experimentEndpoint + if err := experimentDecode(member.Metadata, &endpoint); err != nil { + return fmt.Errorf("invalid experiment participant metadata: %w", err) + } + _, known := e.peers[endpoint.Participant] + host, port, err := net.SplitHostPort(endpoint.Address) + if !known || members[endpoint.Participant] || runs[member.ID] || member.ID == "" || + member.ExecutorID == "" || addresses[endpoint.Address] || + endpoint.Fingerprint != e.fingerprint || err != nil || !networkHost(host) || !networkPort(port) { + return fmt.Errorf("admitted experiment participant does not match the program") + } + if endpoint.Participant == e.participant && endpoint.Address != address { + return fmt.Errorf("admitted listener does not match this participant") + } + members[endpoint.Participant], runs[member.ID], addresses[endpoint.Address] = true, true, true + e.peers[endpoint.Participant] = experimentPeer{member.ID, member.ExecutorID, endpoint.Address} + } + e.id = group.ID + return nil +} + +// Send writes one message with no retry. Success means the frame was written, +// not that the recipient's workflow has consumed it or produced a response. +func (e *Experiment) Send(inputs map[string]any) (map[string]any, error) { + channel, peer, requestID, response, err := e.operation(inputs, true) + if err != nil { + return nil, err + } + value, exists := inputs["value"] + if !exists { + return nil, fmt.Errorf("missing value") + } + kind := channel.Type + if response { + kind = channel.ResponseType + } + if err := Check(map[string]any{"value": value}, map[string]string{"value": kind}); err != nil { + return nil, err + } + timeout, err := networkTimeout(inputs) + if err != nil { + return nil, err + } + if !response && channel.ResponseType != "" { + if len(e.pending) >= experimentPendingLimit { + return nil, fmt.Errorf("experiment exceeds 128 outstanding requests") + } + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return nil, fmt.Errorf("generate request ID: %w", err) + } + requestID = hex.EncodeToString(id) + } + e.sequence++ + frame := experimentFrame{ + ExperimentID: e.id, Fingerprint: e.fingerprint, From: e.participant, + FromRunID: e.peers[e.participant].runID, To: peer, ToRunID: e.peers[peer].runID, + Channel: channel.ID, RequestID: requestID, Sequence: e.sequence, Value: value, + } + data, err := experimentEncode(frame) + if err != nil { + return nil, err + } + out := map[string]any{"ok": false, "error": "", "request_id": requestID} + deadline := time.Now().Add(timeout) + conn, err := debuglet.DialTimeout("tcp", e.peers[peer].address, timeout) + if err == nil { + defer conn.Close() + err = conn.SetWriteDeadline(deadline) + if err == nil { + err = experimentWrite(conn, data) + } + } + if err != nil { + out["error"] = err.Error() + return out, nil + } + if response { + delete(e.requests, requestID) + } else if channel.ResponseType != "" { + e.pending[requestID] = experimentRequest{channel: channel.ID, peer: peer} + } + out["ok"] = true + return out, nil +} + +// Await consumes a matching message or waits until its deadline. Valid messages +// for other channels are buffered within the inbox limit, never discarded. +func (e *Experiment) Await(inputs map[string]any) (map[string]any, error) { + channel, peer, requestID, response, err := e.operation(inputs, false) + if err != nil { + return nil, err + } + timeout, err := networkTimeout(inputs) + if err != nil { + return nil, err + } + out := map[string]any{"ok": false, "error": "", "value": nil, "request_id": "", "source": ""} + deadline := time.Now().Add(timeout) + for { + for i, message := range e.inbox { + frame := message.frame + if frame.Channel != channel.ID || frame.From != peer || (response && frame.RequestID != requestID) { + continue + } + e.inbox = append(e.inbox[:i], e.inbox[i+1:]...) + e.inboxBytes -= message.size + if response { + delete(e.pending, requestID) + } else if channel.ResponseType != "" { + request := e.requests[frame.RequestID] + request.ready = true + e.requests[frame.RequestID] = request + } + out["ok"], out["value"], out["request_id"], out["source"] = true, frame.Value, frame.RequestID, frame.From + return out, nil + } + if time.Until(deadline) <= 0 { + out["error"] = "experiment await timed out" + return out, nil + } + conn, err := debuglet.AcceptTCPTimeout(time.Until(deadline)) + if err == nil { + err = conn.SetReadDeadline(deadline) + if err == nil { + var frame experimentFrame + var size int + frame, size, err = experimentRead(conn) + if err == nil { + err = e.enqueue(frame, size) + } + } + conn.Close() + } + if err != nil { + out["error"] = err.Error() + return out, nil + } + } +} + +func (e *Experiment) operation(inputs map[string]any, send bool) (experimentChannel, string, string, bool, error) { + var empty experimentChannel + channelID, err := networkString(inputs, "channel", "") + if err != nil { + return empty, "", "", false, err + } + channel, exists := e.channels[channelID] + if !exists || e.id == "" { + return empty, "", "", false, fmt.Errorf("experiment channel is unavailable") + } + selector := "source" + if send { + selector = "destination" + } + peer, err := networkString(inputs, selector, "") + if err != nil { + return empty, "", "", false, err + } + from, to := peer, e.participant + if send { + from, to = e.participant, peer + } + response := from == channel.To && to == channel.From && channel.ResponseType != "" + if !response && (from != channel.From || to != channel.To) { + return empty, "", "", false, fmt.Errorf("participant is not an endpoint of this channel direction") + } + requestID, err := experimentRequestID(inputs) + if err != nil { + return empty, "", "", false, err + } + if !response && requestID != "" { + return empty, "", "", false, fmt.Errorf("forward channel operations must not supply a request_id") + } + if response { + requests := e.pending + if send { + requests = e.requests + } + request, exists := requests[requestID] + if requestID == "" || !exists || request.channel != channel.ID || request.peer != peer || (send && !request.ready) { + return empty, "", "", false, fmt.Errorf("response requires an unconsumed request_id from this channel and peer") + } + } + return channel, peer, requestID, response, nil +} + +func (e *Experiment) enqueue(frame experimentFrame, size int) error { + channel, known := e.channels[frame.Channel] + peer, member := e.peers[frame.From] + if !known || !member || frame.From == e.participant || frame.ExperimentID != e.id || + frame.Fingerprint != e.fingerprint || frame.FromRunID != peer.runID || + frame.To != e.participant || frame.ToRunID != e.peers[e.participant].runID { + return fmt.Errorf("message does not belong to this experiment participant") + } + response := frame.From == channel.To && frame.To == channel.From && channel.ResponseType != "" + if !response && (frame.From != channel.From || frame.To != channel.To) { + return fmt.Errorf("message does not match the channel direction") + } + kind := channel.Type + if response { + kind = channel.ResponseType + } + if err := Check(map[string]any{"value": frame.Value}, map[string]string{"value": kind}); err != nil { + return err + } + if size < 1 || size > experimentFrameLimit || e.inboxBytes+size > experimentInboxLimit || len(e.inbox) >= 128 { + return fmt.Errorf("experiment inbox exceeds 64 KiB or 128 messages") + } + sequence, err := e.received[frame.From].accept(frame.Sequence) + if err != nil { + return err + } + if channel.ResponseType == "" { + if frame.RequestID != "" { + return fmt.Errorf("one-way channel message has a request ID") + } + } else { + id, err := hex.DecodeString(frame.RequestID) + if err != nil || len(id) != 16 { + return fmt.Errorf("invalid experiment request ID") + } + if response { + request, exists := e.pending[frame.RequestID] + if !exists || request.channel != channel.ID || request.peer != frame.From || request.ready { + return fmt.Errorf("unexpected or duplicate experiment response") + } + request.ready = true + e.pending[frame.RequestID] = request + } else { + if _, exists := e.requests[frame.RequestID]; exists || len(e.requests) >= experimentPendingLimit { + return fmt.Errorf("duplicate request or more than 128 pending incoming requests") + } + e.requests[frame.RequestID] = experimentRequest{channel: channel.ID, peer: frame.From} + } + } + e.received[frame.From] = sequence + e.inbox = append(e.inbox, experimentMessage{frame: frame, size: size}) + e.inboxBytes += size + return nil +} + +// A fixed window admits limited connection reordering without an unbounded +// replay table. Frames older than the latest 64 sequence numbers are rejected. +func (s experimentSequence) accept(sequence uint64) (experimentSequence, error) { + if sequence == 0 { + return s, fmt.Errorf("invalid experiment message sequence") + } + if sequence > s.highest { + delta := sequence - s.highest + s.seen = s.seen<= 64 || s.seen&(uint64(1)< 32 { + return "", fmt.Errorf("request_id must be a string of at most 32 characters") + } + return id, nil +} + +func experimentType(kind string) bool { + switch kind { + case "any", "string", "number", "boolean", "object", "numbers", "strings": + return true + default: + return false + } +} + +func experimentEncode(frame experimentFrame) ([]byte, error) { + data, err := json.Marshal(frame) + if err != nil { + return nil, err + } + if len(data) > experimentFrameLimit { + return nil, fmt.Errorf("experiment message exceeds the 8 KiB frame limit") + } + packet := make([]byte, 4+len(data)) + binary.BigEndian.PutUint32(packet, uint32(len(data))) + copy(packet[4:], data) + return packet, nil +} + +func experimentWrite(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if n < 0 || n > len(data) { + return fmt.Errorf("invalid message write count") + } + data = data[n:] + if err != nil { + return err + } + if n == 0 { + return io.ErrNoProgress + } + } + return nil +} + +func experimentRead(reader io.Reader) (experimentFrame, int, error) { + var frame experimentFrame + var header [4]byte + if _, err := io.ReadFull(reader, header[:]); err != nil { + return frame, 0, err + } + size := int(binary.BigEndian.Uint32(header[:])) + if size < 1 || size > experimentFrameLimit { + return frame, 0, fmt.Errorf("experiment message exceeds the 8 KiB frame limit") + } + data := make([]byte, size) + if _, err := io.ReadFull(reader, data); err != nil { + return frame, 0, err + } + err := experimentDecode(data, &frame) + return frame, size, err +} + +func experimentDecode(data []byte, out any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + return err + } + if err := decoder.Decode(new(any)); err != io.EOF { + return fmt.Errorf("expected one JSON value") + } + return nil +} diff --git a/pkg/workflow/guest/experiment_test.go b/pkg/workflow/guest/experiment_test.go new file mode 100644 index 00000000..446aba23 --- /dev/null +++ b/pkg/workflow/guest/experiment_test.go @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "io" + "net" + "reflect" + "strings" + "testing" + "time" + + experimentwire "github.com/netsec-ethz/debuglet/pkg/wire/experiment" +) + +func experimentFixture(t *testing.T, self string) *Experiment { + t.Helper() + config := experimentConfig{ + Participant: self, Fingerprint: strings.Repeat("a", 64), + Participants: []string{"a", "b", "c"}, + Channels: []experimentChannel{ + {ID: "request", From: "a", To: "b", Type: "object", ResponseType: "number"}, + {ID: "notice", From: "a", To: "c", Type: "string"}, + {ID: "other", From: "c", To: "b", Type: "boolean"}, + }, + } + session, err := newExperiment(config) + if err != nil { + t.Fatal(err) + } + group := experimentGroup(t, session) + address := "127.0.0.1:2001" + for i, participant := range []string{"a", "b", "c"} { + if participant == self { + address = groupAddress(i) + } + } + if err := session.admit(group, address); err != nil { + t.Fatal(err) + } + return session +} + +func groupAddress(index int) string { + return net.JoinHostPort("127.0.0.1", []string{"2001", "2002", "2003"}[index]) +} + +func experimentGroup(t *testing.T, session *Experiment) experimentwire.Experiment { + t.Helper() + group := experimentwire.Experiment{ID: "group"} + for i, participant := range []string{"a", "b", "c"} { + metadata, err := json.Marshal(experimentEndpoint{participant, session.fingerprint, groupAddress(i)}) + if err != nil { + t.Fatal(err) + } + group.Participants = append(group.Participants, experimentwire.ExperimentParticipant{ + ID: "run-" + participant, ExecutorID: "executor-" + participant, Metadata: metadata, + }) + } + return group +} + +func experimentIncoming(e *Experiment, from, channel, requestID string, sequence uint64, value any) experimentFrame { + return experimentFrame{ + ExperimentID: e.id, Fingerprint: e.fingerprint, + From: from, FromRunID: e.peers[from].runID, + To: e.participant, ToRunID: e.peers[e.participant].runID, + Channel: channel, RequestID: requestID, Sequence: sequence, Value: value, + } +} + +func queueExperiment(t *testing.T, e *Experiment, frame experimentFrame) { + t.Helper() + data, err := experimentEncode(frame) + if err != nil { + t.Fatal(err) + } + decoded, size, err := experimentRead(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if err := e.enqueue(decoded, size); err != nil { + t.Fatal(err) + } +} + +func TestExperimentMembershipRequiresExactAdmittedProgram(t *testing.T) { + for _, test := range []struct { + name string + edit func(*experimentwire.Experiment) + }{ + {"missing", func(g *experimentwire.Experiment) { g.Participants = g.Participants[:2] }}, + {"duplicate run", func(g *experimentwire.Experiment) { g.Participants[1].ID = g.Participants[0].ID }}, + {"empty executor", func(g *experimentwire.Experiment) { g.Participants[1].ExecutorID = "" }}, + {"duplicate participant", func(g *experimentwire.Experiment) { g.Participants[1].Metadata = g.Participants[0].Metadata }}, + {"wrong fingerprint", func(g *experimentwire.Experiment) { + g.Participants[1].Metadata = bytes.ReplaceAll(g.Participants[1].Metadata, []byte(strings.Repeat("a", 64)), []byte(strings.Repeat("b", 64))) + }}, + {"invalid endpoint", func(g *experimentwire.Experiment) { + g.Participants[1].Metadata = bytes.ReplaceAll(g.Participants[1].Metadata, []byte("127.0.0.1:2002"), []byte("127.0.0.1:0")) + }}, + {"wrong self address", func(g *experimentwire.Experiment) { + g.Participants[0].Metadata = bytes.ReplaceAll(g.Participants[0].Metadata, []byte("127.0.0.1:2001"), []byte("127.0.0.1:4000")) + }}, + } { + t.Run(test.name, func(t *testing.T) { + e := experimentFixture(t, "a") + group := experimentGroup(t, e) + test.edit(&group) + if err := e.admit(group, "127.0.0.1:2001"); err == nil { + t.Fatal("accepted mismatched membership") + } + }) + } +} + +func TestExperimentRequestResponseOwnership(t *testing.T) { + requestID := strings.Repeat("1", 32) + receiver := experimentFixture(t, "b") + request := experimentIncoming(receiver, "a", "request", requestID, 1, map[string]any{"url": "https://example.test"}) + queueExperiment(t, receiver, request) + responseInputs := map[string]any{"destination": "a", "channel": "request", "request_id": requestID} + if _, _, _, _, err := receiver.operation(responseInputs, true); err == nil { + t.Fatal("reply allowed before the request was consumed") + } + out, err := receiver.Await(map[string]any{"source": "a", "channel": "request"}) + if err != nil || out["ok"] != true || out["request_id"] != requestID || out["source"] != "a" { + t.Fatalf("request result %#v, %v", out, err) + } + if _, _, _, reverse, err := receiver.operation(responseInputs, true); err != nil || !reverse { + t.Fatalf("reply to consumed request: reverse=%v, err=%v", reverse, err) + } + responseInputs["request_id"] = strings.Repeat("2", 32) + if _, _, _, _, err := receiver.operation(responseInputs, true); err == nil { + t.Fatal("reply accepted an unrelated request ID") + } + + sender := experimentFixture(t, "a") + sender.pending[requestID] = experimentRequest{channel: "request", peer: "b"} + response := experimentIncoming(sender, "b", "request", requestID, 1, float64(200)) + queueExperiment(t, sender, response) + response.Sequence++ + if err := sender.enqueue(response, 400); err == nil { + t.Fatal("queued a duplicate response under a new sequence") + } + out, err = sender.Await(map[string]any{"source": "b", "channel": "request", "request_id": requestID}) + if err != nil || out["ok"] != true || out["value"] != float64(200) || len(sender.pending) != 0 { + t.Fatalf("response result %#v, %v, pending=%#v", out, err, sender.pending) + } + if _, err := sender.Await(map[string]any{"source": "b", "channel": "request", "request_id": requestID}); err == nil { + t.Fatal("consumed response could be awaited again") + } + if err := sender.enqueue(response, 400); err == nil { + t.Fatal("accepted a late response to a consumed request") + } +} + +func TestExperimentAwaitRetainsOtherChannels(t *testing.T) { + e := experimentFixture(t, "b") + queueExperiment(t, e, experimentIncoming(e, "c", "other", "", 1, true)) + id := strings.Repeat("1", 32) + queueExperiment(t, e, experimentIncoming(e, "a", "request", id, 1, map[string]any{"value": float64(1)})) + out, err := e.Await(map[string]any{"source": "a", "channel": "request"}) + if err != nil || out["ok"] != true || len(e.inbox) != 1 || e.inboxBytes == 0 { + t.Fatalf("matching request did not preserve unrelated message: %#v %v", out, err) + } + out, err = e.Await(map[string]any{"source": "c", "channel": "other"}) + if err != nil || out["value"] != true || len(e.inbox) != 0 || e.inboxBytes != 0 { + t.Fatalf("buffered message lost: %#v %v", out, err) + } +} + +func TestExperimentRejectsMisdirectedAndUntypedFrames(t *testing.T) { + for _, test := range []struct { + name string + edit func(*experimentFrame) + }{ + {"experiment", func(f *experimentFrame) { f.ExperimentID = "other" }}, + {"fingerprint", func(f *experimentFrame) { f.Fingerprint = strings.Repeat("b", 64) }}, + {"sender run", func(f *experimentFrame) { f.FromRunID = "other" }}, + {"recipient run", func(f *experimentFrame) { f.ToRunID = "other" }}, + {"recipient", func(f *experimentFrame) { f.To = "a" }}, + {"type", func(f *experimentFrame) { f.Value = "true" }}, + {"request ID on one-way", func(f *experimentFrame) { f.RequestID = strings.Repeat("1", 32) }}, + {"sequence zero", func(f *experimentFrame) { f.Sequence = 0 }}, + } { + t.Run(test.name, func(t *testing.T) { + e := experimentFixture(t, "b") + frame := experimentIncoming(e, "c", "other", "", 1, true) + test.edit(&frame) + if err := e.enqueue(frame, 400); err == nil { + t.Fatal("accepted invalid frame") + } + if len(e.inbox) != 0 || len(e.received) != 0 { + t.Fatal("invalid frame changed the inbox or replay window") + } + }) + } +} + +func TestExperimentInboxAndSequenceBounds(t *testing.T) { + e := experimentFixture(t, "b") + frame := experimentIncoming(e, "c", "other", "", 1, true) + for i := 0; i < experimentInboxLimit/experimentFrameLimit; i++ { + frame.Sequence = uint64(i + 1) + if err := e.enqueue(frame, experimentFrameLimit); err != nil { + t.Fatal(err) + } + } + frame.Sequence++ + if err := e.enqueue(frame, 1); err == nil { + t.Fatal("accepted message above inbox byte limit") + } + if e.inboxBytes != experimentInboxLimit { + t.Fatal("overflow changed inbox accounting") + } + var sequence experimentSequence + for _, n := range []uint64{2, 1, 65, 64} { + var err error + sequence, err = sequence.accept(n) + if err != nil { + t.Fatalf("sequence %d: %v", n, err) + } + } + for _, n := range []uint64{0, 1, 2, 64, 65} { + if _, err := sequence.accept(n); err == nil { + t.Fatalf("accepted stale or duplicate sequence %d", n) + } + } +} + +func TestExperimentFrameRoundTripAndBounds(t *testing.T) { + e := experimentFixture(t, "b") + frame := experimentIncoming(e, "c", "other", "", 1, true) + packet, err := experimentEncode(frame) + if err != nil { + t.Fatal(err) + } + got, size, err := experimentRead(bytes.NewReader(packet)) + if err != nil || size != len(packet)-4 || !reflect.DeepEqual(got, frame) { + t.Fatalf("round trip: %#v %d %v", got, size, err) + } + for _, test := range [][]byte{packet[:3], packet[:len(packet)-1], {0, 0, 0, 0}, {0, 0, 32, 1}} { + if _, _, err := experimentRead(bytes.NewReader(test)); err == nil { + t.Fatalf("accepted incomplete or oversized frame %v", test[:min(len(test), 4)]) + } + } + frame.Value = strings.Repeat("x", experimentFrameLimit) + if _, err := experimentEncode(frame); err == nil { + t.Fatal("encoded oversized message") + } + var decoded experimentEndpoint + for _, text := range []string{`{"address":"x","unexpected":true}`, `{} {}`} { + if err := experimentDecode([]byte(text), &decoded); err == nil { + t.Fatalf("accepted invalid metadata %s", text) + } + } +} + +func TestExperimentFramingOverTCP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + e := experimentFixture(t, "b") + frame := experimentIncoming(e, "c", "other", "", 1, true) + packet, err := experimentEncode(frame) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + done <- err + return + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(2 * time.Second)); err != nil { + done <- err + return + } + got, size, err := experimentRead(conn) + if err == nil { + err = e.enqueue(got, size) + } + done <- err + }() + conn, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatal(err) + } + // Split the length header as well as the payload across writes. + for _, part := range [][]byte{packet[:2], packet[2:5], packet[5:]} { + if err := experimentWrite(conn, part); err != nil { + t.Fatal(err) + } + } + if err := <-done; err != nil { + t.Fatal(err) + } + out, err := e.Await(map[string]any{"source": "c", "channel": "other"}) + if err != nil || out["ok"] != true || out["value"] != true { + t.Fatalf("TCP result %#v, %v", out, err) + } +} + +func TestExperimentFramingReadDeadline(t *testing.T) { + reader, writer := net.Pipe() + defer reader.Close() + defer writer.Close() + if err := reader.SetReadDeadline(time.Now().Add(20 * time.Millisecond)); err != nil { + t.Fatal(err) + } + wrote := make(chan error, 1) + go func() { + var header [4]byte + binary.BigEndian.PutUint32(header[:], 10) + _, err := writer.Write(header[:]) + wrote <- err + }() + if _, _, err := experimentRead(reader); err == nil { + t.Fatal("partial peer frame did not hit its read deadline") + } else if netErr, ok := err.(net.Error); !ok || !netErr.Timeout() { + t.Fatalf("expected timeout, got %v", err) + } + if err := <-wrote; err != nil { + t.Fatal(err) + } +} + +type noProgressWriter struct{} + +func (noProgressWriter) Write([]byte) (int, error) { return 0, nil } + +func TestExperimentWriteRejectsNoProgress(t *testing.T) { + if err := experimentWrite(noProgressWriter{}, []byte("message")); !errors.Is(err, io.ErrNoProgress) { + t.Fatalf("write result: %v", err) + } +} diff --git a/pkg/workflow/guest/http_request_test.go b/pkg/workflow/guest/http_request_test.go new file mode 100644 index 00000000..b06d5d12 --- /dev/null +++ b/pkg/workflow/guest/http_request_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + "reflect" + "strings" + "testing" +) + +func TestHTTPRequestPreservesBodyAndHeaders(t *testing.T) { + for _, body := range []string{"", " text\n", `{"message":"Grüezi ☀"}`} { + t.Run(body, func(t *testing.T) { + req, _, err := networkHTTPRequest(map[string]any{ + "url": "http://example.test:8080/echo?kind=json", "method": "POST", "body": body, + "headers": map[string]any{"content-type": "application/json", "X-Experiment": "sample"}, + }) + if err != nil { + t.Fatal(err) + } + var wire bytes.Buffer + if err = req.Write(&wire); err != nil { + t.Fatal(err) + } + if strings.Count(strings.ToLower(wire.String()), "\r\ncontent-length:") != 1 { + t.Fatal("request did not contain exactly one generated Content-Length") + } + parsed, err := http.ReadRequest(bufio.NewReader(&wire)) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(parsed.Body) + if err != nil || string(got) != body || parsed.ContentLength != int64(len(body)) { + t.Fatalf("body=%q length=%d error=%v", got, parsed.ContentLength, err) + } + if parsed.Method != "POST" || parsed.RequestURI != "/echo?kind=json" || parsed.Host != "example.test:8080" || len(parsed.TransferEncoding) != 0 { + t.Fatalf("unexpected request: %#v", parsed) + } + if parsed.Header.Get("Content-Type") != "application/json" || parsed.Header.Get("X-Experiment") != "sample" || parsed.Header.Get("Accept-Encoding") != "identity" || !parsed.Close { + t.Fatalf("headers = %#v, close=%v", parsed.Header, parsed.Close) + } + }) + } +} + +func TestHTTPRequestRejectsInvalidAndManagedHeaders(t *testing.T) { + cases := map[string]map[string]any{ + "body type": {"body": map[string]any{}}, + "body limit": {"body": strings.Repeat("x", networkBodyLimit+1)}, + "header type": {"headers": "x=y"}, + "header value type": {"headers": map[string]any{"X-Test": []any{"a", "b"}}}, + "header name": {"headers": map[string]any{"X Test": "x"}}, + "header injection": {"headers": map[string]any{"X-Test": "x\r\nHost: other.test"}}, + "header null": {"headers": map[string]any{"X-Test": "x\x00y"}}, + "header case collision": {"headers": map[string]any{"X-Test": "a", "x-test": "b"}}, + "header limit": {"headers": map[string]any{"X-Test": strings.Repeat("x", networkBodyLimit)}}, + } + for _, name := range []string{"Host", "Content-Length", "Transfer-Encoding", "Connection", "Proxy-Connection", "Keep-Alive", "TE", "Trailer", "Upgrade", "Expect", "Accept-Encoding"} { + cases[name] = map[string]any{"headers": map[string]any{name: "override"}} + } + for name, inputs := range cases { + t.Run(name, func(t *testing.T) { + inputs["url"] = "http://example.test/" + if request, _, err := networkHTTPRequest(inputs); err == nil || request != nil { + t.Fatalf("invalid request accepted: %v %v", request, err) + } + }) + } +} + +func TestHTTPResponseHeadersRemainStructured(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "http://example.test/", nil) + wire := "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nSet-Cookie: a=1\r\nSet-Cookie: b=2\r\nX-Measurement: sample\r\n\r\nok" + status, body, headers, err := networkReadHTTP(strings.NewReader(wire), req) + if err != nil || status != 200 || body != "ok" { + t.Fatalf("response status=%d body=%q error=%v", status, body, err) + } + if !reflect.DeepEqual(headers["set-cookie"], []any{"a=1", "b=2"}) || !reflect.DeepEqual(headers["x-measurement"], []any{"sample"}) { + t.Fatalf("headers = %#v", headers) + } + wire = "HTTP/1.1 200 OK\r\nX-Large: " + strings.Repeat("x", networkBodyLimit) + "\r\nContent-Length: 2\r\n\r\nok" + status, body, headers, err = networkReadHTTP(strings.NewReader(wire), req) + if err == nil || status != 200 || body != "" || len(headers) != 0 { + t.Fatalf("oversized headers accepted: status=%d body=%q headers=%d error=%v", status, body, len(headers), err) + } +} + +func TestHTTPHeaderCountAndBodyBoundaries(t *testing.T) { + request, _, err := networkHTTPRequest(map[string]any{"url": "http://example.test/", "body": strings.Repeat("x", networkBodyLimit)}) + if err != nil || request.ContentLength != networkBodyLimit { + t.Fatalf("body at limit was rejected: %v", err) + } + for _, count := range []int{64, 65} { + headers := map[string]any{} + var wire strings.Builder + wire.WriteString("HTTP/1.1 200 OK\r\n") + for i := 0; i < count; i++ { + name := fmt.Sprintf("X-Value-%d", i) + headers[name] = "v" + fmt.Fprintf(&wire, "%s: v\r\n", name) + } + wire.WriteString("\r\nok") + _, _, requestErr := networkHTTPRequest(map[string]any{"url": "http://example.test/", "headers": headers}) + _, _, _, responseErr := networkReadHTTP(strings.NewReader(wire.String()), request) + wantError := count > 64 + if (requestErr != nil) != wantError || (responseErr != nil) != wantError { + t.Fatalf("%d headers: request=%v response=%v", count, requestErr, responseErr) + } + } +} diff --git a/pkg/workflow/guest/network.go b/pkg/workflow/guest/network.go new file mode 100644 index 00000000..55b0e6d6 --- /dev/null +++ b/pkg/workflow/guest/network.go @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bufio" + "bytes" + "crypto/rand" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/netsec-ethz/debuglet/pkg/debuglet" +) + +const networkBodyLimit = 16 * 1024 + +// HTTP performs one request without following redirects. A received response, +// including a non-2xx status, is successful; status checks belong in the graph. +func HTTP(inputs map[string]any) (map[string]any, error) { + req, timeout, err := networkHTTPRequest(inputs) + if err != nil { + return nil, err + } + u := req.URL + out := map[string]any{"ok": false, "error": "", "status": float64(0), "body": "", "headers": map[string]any{}, "duration_ms": float64(0)} + start := time.Now() + defer networkDuration(out, start) + port, transport := u.Port(), "tcp" + if u.Scheme == "https" { + transport = "tls" + } + if port == "" { + port = "80" + if transport == "tls" { + port = "443" + } + } + conn, err := debuglet.DialTimeout(transport, net.JoinHostPort(u.Hostname(), port), timeout) + if err != nil { + out["error"] = err.Error() + return out, nil + } + defer conn.Close() + deadline := start.Add(timeout) + if err = conn.SetWriteDeadline(deadline); err == nil { + err = conn.SetReadDeadline(deadline) + } + if err == nil { + err = req.Write(conn) + } + if err == nil { + var status int + var body string + var headers map[string]any + status, body, headers, err = networkReadHTTP(conn, req) + out["status"], out["body"], out["headers"] = float64(status), body, headers + } + out["ok"] = err == nil + if err != nil { + out["error"] = err.Error() + } + return out, nil +} + +// networkHTTPRequest constructs the exact request before any network access. +func networkHTTPRequest(inputs map[string]any) (*http.Request, time.Duration, error) { + raw, err := networkString(inputs, "url", "") + if err != nil { + return nil, 0, err + } + u, err := url.Parse(raw) + if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || + !networkHost(u.Hostname()) || u.User != nil || u.Opaque != "" || strings.Contains(raw, "#") || + strings.HasSuffix(u.Host, ":") || (u.Port() != "" && !networkPort(u.Port())) { + return nil, 0, fmt.Errorf("url must be an HTTP(S) URL without credentials or a fragment") + } + method, err := networkString(inputs, "method", http.MethodGet) + if err != nil { + return nil, 0, err + } + body, ok := inputs["body"].(string) + if _, exists := inputs["body"]; exists && !ok { + return nil, 0, fmt.Errorf("body must be a string") + } + if len(body) > networkBodyLimit { + return nil, 0, fmt.Errorf("HTTP request body exceeds 16 KiB") + } + req, err := http.NewRequest(method, u.String(), strings.NewReader(body)) + if err != nil { + return nil, 0, fmt.Errorf("HTTP request: %w", err) + } + timeout, err := networkTimeout(inputs) + if err != nil { + return nil, 0, err + } + req.Close = true + req.Header.Set("Accept-Encoding", "identity") + req.Header.Set("User-Agent", "debuglet-workflow/1") + if value, exists := inputs["headers"]; exists { + headers, ok := value.(map[string]any) + if !ok || len(headers) > 64 { + return nil, 0, fmt.Errorf("headers must be an object with at most 64 fields") + } + seen := map[string]bool{} + size := 0 + for name, raw := range headers { + value, ok := raw.(string) + if !ok || !networkHeaderName(name) || !networkHeaderValue(value) { + return nil, 0, fmt.Errorf("HTTP headers require valid names and string values without control characters") + } + key := strings.ToLower(name) + switch key { + case "host", "content-length", "transfer-encoding", "connection", "proxy-connection", "keep-alive", "te", "trailer", "upgrade", "expect", "accept-encoding": + return nil, 0, fmt.Errorf("HTTP header %q is managed by the HTTP block", name) + } + if seen[key] { + return nil, 0, fmt.Errorf("HTTP header %q is specified more than once", name) + } + seen[key] = true + size += len(name) + len(value) + 4 + if size > networkBodyLimit { + return nil, 0, fmt.Errorf("HTTP request headers exceed 16 KiB") + } + req.Header.Set(name, value) + } + } + return req, timeout, nil +} + +func networkHeaderName(name string) bool { + if name == "" || len(name) > 256 { + return false + } + for _, c := range name { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune("!#$%&'*+-.^_`|~", c)) { + return false + } + } + return true +} + +func networkHeaderValue(value string) bool { + for _, c := range value { + if c < 32 && c != '\t' || c == 127 { + return false + } + } + return true +} + +// TCPConnect measures resolution and connection establishment, then closes. +func TCPConnect(inputs map[string]any) (map[string]any, error) { + target, err := networkString(inputs, "target", "") + if err != nil { + return nil, err + } + host, port, err := net.SplitHostPort(target) + if err != nil || !networkHost(host) || !networkPort(port) { + return nil, fmt.Errorf("target must be a host:port with a port from 1 to 65535") + } + timeout, err := networkTimeout(inputs) + if err != nil { + return nil, err + } + out := map[string]any{"ok": false, "error": "", "duration_ms": float64(0)} + start := time.Now() + defer networkDuration(out, start) + conn, err := debuglet.DialTimeout("tcp", target, timeout) + if err == nil { + err = conn.Close() + } + out["ok"] = err == nil + if err != nil { + out["error"] = err.Error() + } + return out, nil +} + +// Ping sends up to ten serial ICMPv4 echo requests. RTTs include only matching +// replies. Loss is a percentage; no successfully sent probes yields 100. +func Ping(inputs map[string]any) (map[string]any, error) { + target, err := networkString(inputs, "target", "") + if err != nil { + return nil, err + } + if !networkHost(target) || strings.Contains(target, ":") { + return nil, fmt.Errorf("target must be an IPv4 address or hostname") + } + count, err := networkInteger(inputs, "count", 3, 1, 10) + if err != nil { + return nil, err + } + timeout, err := networkTimeout(inputs) + if err != nil { + return nil, err + } + out := map[string]any{"ok": false, "error": "", "rtts_ms": []any{}, "sent": float64(0), "received": float64(0), "loss": float64(100)} + conn, err := debuglet.DialTimeout("ip4:icmp", target, timeout) + if err != nil { + out["error"] = err.Error() + return out, nil + } + defer conn.Close() + payload := make([]byte, 16) + if _, err := rand.Read(payload); err != nil { + return nil, fmt.Errorf("ICMP nonce: %w", err) + } + id := binary.BigEndian.Uint16(payload) + rtts := []any{} + sent := 0 + var lastErr error + for seq := 0; seq < count; seq++ { + start := time.Now() + deadline := start.Add(timeout) + err = conn.SetWriteDeadline(deadline) + if err == nil { + err = conn.SetReadDeadline(deadline) + } + if err == nil { + packet := networkEcho(id, uint16(seq), payload) + var n int + n, err = conn.Write(packet) + if err == nil && n != len(packet) { + err = io.ErrShortWrite + } + } + if err != nil { + lastErr = err + break + } + sent++ + buf := make([]byte, 1500) + for { + var n int + n, err = conn.Read(buf) + if err != nil { + lastErr = err + break + } + if networkEchoReply(buf[:n], id, uint16(seq), payload) { + rtts = append(rtts, float64(time.Since(start))/float64(time.Millisecond)) + break + } + if !time.Now().Before(deadline) { + lastErr = debuglet.ErrTimeout + break + } + } + } + out["rtts_ms"], out["sent"], out["received"] = rtts, float64(sent), float64(len(rtts)) + if sent > 0 { + out["loss"] = 100 * float64(sent-len(rtts)) / float64(sent) + } + out["ok"] = len(rtts) == count + if len(rtts) != count { + out["error"] = fmt.Sprintf("received %d of %d requested replies", len(rtts), count) + if lastErr != nil { + out["error"] = fmt.Sprintf("%s: %v", out["error"], lastErr) + } + } + return out, nil +} + +func networkReadHTTP(source io.Reader, req *http.Request) (int, string, map[string]any, error) { + // Bound status lines and headers as well as the body. + limited := &io.LimitedReader{R: source, N: 64*1024 + 1} + reader := bufio.NewReader(limited) + for n := 0; n < 8; n++ { + response, err := http.ReadResponse(reader, req) + if err != nil { + return 0, "", map[string]any{}, fmt.Errorf("read HTTP response: %w", err) + } + if response.StatusCode == http.StatusSwitchingProtocols { + return response.StatusCode, "", map[string]any{}, fmt.Errorf("HTTP protocol upgrades are unsupported") + } + if response.StatusCode < 200 { + continue + } + headers := map[string]any{} + headerSize := 0 + for name, values := range response.Header { + items := make([]any, len(values)) + for i, value := range values { + headerSize += len(name) + len(value) + 4 + items[i] = value + } + headers[strings.ToLower(name)] = items + } + if len(headers) > 64 || headerSize > networkBodyLimit || checkValue(headers, 0) != nil { + return response.StatusCode, "", map[string]any{}, fmt.Errorf("HTTP response headers exceed 16 KiB or 64 fields") + } + body, err := io.ReadAll(io.LimitReader(response.Body, networkBodyLimit+1)) + // The owner closes the underlying socket. Closing Body here could drain + // an arbitrarily long response before returning its bounded prefix. + if len(body) > networkBodyLimit { + body = body[:networkBodyLimit] + err = fmt.Errorf("HTTP response body exceeds 16 KiB") + } else if limited.N == 0 { + err = fmt.Errorf("HTTP response exceeds 64 KiB including headers") + } + return response.StatusCode, string(body), headers, err + } + return 0, "", map[string]any{}, fmt.Errorf("too many informational HTTP responses") +} + +func networkEcho(id, seq uint16, payload []byte) []byte { + packet := make([]byte, 8+len(payload)) + packet[0] = 8 + binary.BigEndian.PutUint16(packet[4:6], id) + binary.BigEndian.PutUint16(packet[6:8], seq) + copy(packet[8:], payload) + binary.BigEndian.PutUint16(packet[2:4], networkChecksum(packet)) + return packet +} + +func networkEchoReply(packet []byte, id, seq uint16, payload []byte) bool { + if len(packet) >= 20 && packet[0]>>4 == 4 { + header := int(packet[0]&15) * 4 + length := int(binary.BigEndian.Uint16(packet[2:4])) + if header < 20 || header > len(packet) || length < header || length > len(packet) || packet[9] != 1 { + return false + } + packet = packet[header:length] + } + return len(packet) == 8+len(payload) && packet[0] == 0 && packet[1] == 0 && + networkChecksum(packet) == 0 && + binary.BigEndian.Uint16(packet[4:6]) == id && + binary.BigEndian.Uint16(packet[6:8]) == seq && bytes.Equal(packet[8:], payload) +} + +func networkChecksum(b []byte) uint16 { + var sum uint32 + for len(b) >= 2 { + sum += uint32(binary.BigEndian.Uint16(b)) + b = b[2:] + } + if len(b) == 1 { + sum += uint32(b[0]) << 8 + } + for sum>>16 != 0 { + sum = (sum & 0xffff) + (sum >> 16) + } + return ^uint16(sum) +} + +func networkString(inputs map[string]any, key, fallback string) (string, error) { + value, exists := inputs[key] + if !exists { + value = fallback + } + s, ok := value.(string) + if !ok || s == "" || s != strings.TrimSpace(s) { + return "", fmt.Errorf("%s must be a non-empty string without surrounding whitespace", key) + } + return s, nil +} + +func networkInteger(inputs map[string]any, key string, fallback, minimum, maximum int) (int, error) { + value, exists := inputs[key] + if !exists { + return fallback, nil + } + var number float64 + switch v := value.(type) { + case float64: + number = v + case int: + number = float64(v) + case json.Number: + var err error + number, err = v.Float64() + if err != nil { + return 0, fmt.Errorf("%s must be an integer from %d to %d", key, minimum, maximum) + } + default: + return 0, fmt.Errorf("%s must be an integer from %d to %d", key, minimum, maximum) + } + if math.IsNaN(number) || math.IsInf(number, 0) || number < float64(minimum) || number > float64(maximum) || math.Trunc(number) != number { + return 0, fmt.Errorf("%s must be an integer from %d to %d", key, minimum, maximum) + } + return int(number), nil +} + +func networkTimeout(inputs map[string]any) (time.Duration, error) { + ms, err := networkInteger(inputs, "timeout_ms", 2000, 1, 60000) + return time.Duration(ms) * time.Millisecond, err +} + +func networkHost(host string) bool { + if host == "" || strings.ContainsAny(host, " \t\r\n/@?#%[]") { + return false + } + if net.ParseIP(host) != nil { + return true + } + if len(host) > 253 { + return false + } + for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, char := range label { + if !(char >= 'a' && char <= 'z' || char >= 'A' && char <= 'Z' || char >= '0' && char <= '9' || char == '-') { + return false + } + } + } + return true +} + +func networkPort(port string) bool { + n, err := strconv.ParseUint(port, 10, 16) + return err == nil && n != 0 +} + +func networkDuration(out map[string]any, start time.Time) { + out["duration_ms"] = float64(time.Since(start)) / float64(time.Millisecond) +} diff --git a/pkg/workflow/guest/network_test.go b/pkg/workflow/guest/network_test.go new file mode 100644 index 00000000..ee31fb6f --- /dev/null +++ b/pkg/workflow/guest/network_test.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "net/http" + "strings" + "testing" +) + +func TestNetworkInvalidInputsDoNotDial(t *testing.T) { + for name, test := range map[string]struct { + run func(map[string]any) (map[string]any, error) + inputs map[string]any + }{ + "HTTP scheme": {HTTP, map[string]any{"url": "ftp://example.test/"}}, + "HTTP credentials": {HTTP, map[string]any{"url": "https://user:pass@example.test/"}}, + "HTTP fragment": {HTTP, map[string]any{"url": "https://example.test/#fragment"}}, + "HTTP port": {HTTP, map[string]any{"url": "https://example.test:0/"}}, + "HTTP method": {HTTP, map[string]any{"url": "https://example.test/", "method": "GET\r\nX: y"}}, + "HTTP timeout": {HTTP, map[string]any{"url": "https://example.test/", "timeout_ms": float64(0)}}, + "TCP port": {TCPConnect, map[string]any{"target": "example.test:65536"}}, + "TCP address": {TCPConnect, map[string]any{"target": "https://example.test"}}, + "TCP host": {TCPConnect, map[string]any{"target": "bad host:80"}}, + "Ping IPv6": {Ping, map[string]any{"target": "::1"}}, + "Ping count": {Ping, map[string]any{"target": "example.test", "count": float64(11)}}, + "Ping fractional": {Ping, map[string]any{"target": "example.test", "count": 1.5}}, + "Ping NaN": {Ping, map[string]any{"target": "example.test", "timeout_ms": math.NaN()}}, + "Ping infinite": {Ping, map[string]any{"target": "example.test", "timeout_ms": math.Inf(1)}}, + } { + t.Run(name, func(t *testing.T) { + out, err := test.run(test.inputs) + if err == nil || out != nil { + t.Fatalf("invalid configuration returned output %v, error %v", out, err) + } + }) + } +} + +func TestNetworkHTTPResponse(t *testing.T) { + for name, test := range map[string]struct { + wire string + status int + body string + wantErr string + }{ + "redirect remains observable": { + wire: "HTTP/1.1 301 Moved Permanently\r\nContent-Length: 4\r\nLocation: https://other.test/\r\n\r\nmove", + status: 301, body: "move", + }, + "error status remains observable": { + wire: "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 4\r\n\r\ndown", + status: 503, body: "down", + }, + "chunked": { + wire: "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n", + status: 200, body: "body", + }, + "early hints": { + wire: "HTTP/1.1 103 Early Hints\r\nLink: \r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok", + status: 200, body: "ok", + }, + "truncated": { + wire: "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nno", + status: 200, body: "no", wantErr: "unexpected EOF", + }, + "upgrade": { + wire: "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: test\r\n\r\n", + status: 101, wantErr: "upgrades are unsupported", + }, + "large body": { + wire: fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n", networkBodyLimit+1) + strings.Repeat("x", networkBodyLimit+1), + status: 200, body: strings.Repeat("x", networkBodyLimit), wantErr: "exceeds 16 KiB", + }, + "oversized headers": { + wire: "HTTP/1.1 200 OK\r\nX-Large: " + strings.Repeat("x", 70*1024) + "\r\n\r\n", + wantErr: "read HTTP response", + }, + "too many hints": { + wire: strings.Repeat("HTTP/1.1 103 Early Hints\r\n\r\n", 8), + wantErr: "too many informational", + }, + } { + t.Run(name, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "http://example.test/", nil) + status, body, _, err := networkReadHTTP(strings.NewReader(test.wire), req) + if status != test.status || body != test.body { + t.Fatalf("response = (%d, %d body bytes), want (%d, %d body bytes)", status, len(body), test.status, len(test.body)) + } + if test.wantErr == "" && err != nil || test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("error = %v, want %q", err, test.wantErr) + } + }) + } +} + +func TestNetworkEchoReply(t *testing.T) { + payload := []byte("probe-nonce-1234") + request := networkEcho(123, 7, payload) + if request[0] != 8 || networkChecksum(request) != 0 { + t.Fatal("request type or checksum is invalid") + } + reply := append([]byte(nil), request...) + reply[0], reply[2], reply[3] = 0, 0, 0 + binary.BigEndian.PutUint16(reply[2:4], networkChecksum(reply)) + ipv4 := make([]byte, 24) + ipv4[0], ipv4[9] = 0x46, 1 + binary.BigEndian.PutUint16(ipv4[2:4], uint16(len(ipv4)+len(reply))) + ipv4 = append(ipv4, reply...) + badChecksum := append([]byte(nil), reply...) + badChecksum[len(badChecksum)-1] ^= 1 + for name, test := range map[string]struct { + packet []byte + id, seq uint16 + payload []byte + want bool + }{ + "plain reply": {reply, 123, 7, payload, true}, + "IPv4 options": {ipv4, 123, 7, payload, true}, + "echo request": {request, 123, 7, payload, false}, + "wrong id": {reply, 124, 7, payload, false}, + "wrong seq": {reply, 123, 8, payload, false}, + "wrong nonce": {reply, 123, 7, bytes.Repeat([]byte{0}, len(payload)), false}, + "bad checksum": {badChecksum, 123, 7, payload, false}, + "short reply": {reply[:7], 123, 7, payload, false}, + "truncated IP": {ipv4[:len(ipv4)-1], 123, 7, payload, false}, + "extra payload": {append(append([]byte(nil), reply...), 0), 123, 7, payload, false}, + } { + t.Run(name, func(t *testing.T) { + if got := networkEchoReply(test.packet, test.id, test.seq, test.payload); got != test.want { + t.Fatalf("match = %v, want %v", got, test.want) + } + }) + } +} + +func TestNetworkChecksumOddLength(t *testing.T) { + if got := networkChecksum([]byte{0x01, 0x02, 0x03}); got != 0xfbfd { + t.Fatalf("checksum = %#x, want 0xfbfd", got) + } +} diff --git a/pkg/workflow/guest/primitives_test.go b/pkg/workflow/guest/primitives_test.go new file mode 100644 index 00000000..cc98eeca --- /dev/null +++ b/pkg/workflow/guest/primitives_test.go @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "math" + "reflect" + "strings" + "testing" +) + +func TestCollectionAndTextPrimitives(t *testing.T) { + cases := []struct { + block string + in map[string]any + want any + }{ + {"stringify_json", map[string]any{"value": map[string]any{"enabled": true}}, `{"enabled":true}`}, + {"list_item", map[string]any{"values": []any{"first", nil, float64(3)}, "index": float64(1)}, nil}, + {"append", map[string]any{"values": []any{}, "value": false}, []any{false}}, + {"concat", map[string]any{"a": "probe ", "b": "complete"}, "probe complete"}, + {"contains", map[string]any{"text": "HTTP/2 200", "substring": "200"}, true}, + {"contains", map[string]any{"text": "HTTP/2 200", "substring": "503"}, false}, + {"split", map[string]any{"text": "a::b::", "separator": "::"}, []any{"a", "b", ""}}, + {"split", map[string]any{"text": "", "separator": ","}, []any{""}}, + {"join", map[string]any{"values": []any{"one", "", "three"}, "separator": ":"}, "one::three"}, + {"join", map[string]any{"values": []any{}, "separator": ","}, ""}, + } + for _, tc := range cases { + t.Run(tc.block, func(t *testing.T) { + out, err := Pure(tc.block, tc.in) + if err != nil || !reflect.DeepEqual(out["value"], tc.want) { + t.Fatalf("got %v, %v; want %v", out, err, tc.want) + } + }) + } +} + +func TestCollectionPrimitivesDoNotMutateInputs(t *testing.T) { + object := map[string]any{"a": float64(1)} + out, err := Pure("set_field", map[string]any{"object": object, "name": "a", "value": float64(2)}) + if err != nil { + t.Fatal(err) + } + if object["a"] != float64(1) || out["value"].(map[string]any)["a"] != float64(2) { + t.Fatalf("object or copy changed incorrectly: %v %v", object, out) + } + + backing := []any{"original", "untouched", "spare"} + out, err = Pure("append", map[string]any{"values": backing[:1], "value": "added"}) + if err != nil { + t.Fatal(err) + } + out["value"].([]any)[0] = "edited" + if !reflect.DeepEqual(backing, []any{"original", "untouched", "spare"}) { + t.Fatalf("append reused the input backing array: %v", backing) + } +} + +func TestListIndicesMustSelectAnExistingItem(t *testing.T) { + for _, index := range []float64{-1, 0.5, 2, math.NaN(), math.Inf(1), math.Inf(-1)} { + if _, err := Pure("list_item", map[string]any{"values": []any{"a", "b"}, "index": index}); err == nil { + t.Errorf("accepted index %v", index) + } + } + if _, err := Pure("list_item", map[string]any{"values": []any{}, "index": float64(0)}); err == nil { + t.Fatal("selected an item from an empty list") + } + for _, block := range []string{"list_item", "append"} { + if _, err := Pure(block, map[string]any{"values": "not a list", "index": float64(0), "value": nil}); err == nil { + t.Errorf("%s accepted a string as a list", block) + } + } +} + +func TestPrimitiveOutputBounds(t *testing.T) { + maxString := strings.Repeat("x", 16384) + if _, err := Pure("concat", map[string]any{"a": maxString, "b": ""}); err != nil { + t.Fatal(err) + } + if _, err := Pure("concat", map[string]any{"a": maxString, "b": "x"}); err == nil { + t.Fatal("concat exceeded the string limit") + } + if _, err := Pure("stringify_json", map[string]any{"value": maxString}); err == nil { + t.Fatal("JSON quoting exceeded the string limit") + } + if _, err := Pure("join", map[string]any{"values": []any{maxString}, "separator": ","}); err != nil { + t.Fatal(err) + } + if _, err := Pure("join", map[string]any{"values": []any{maxString, ""}, "separator": ","}); err == nil { + t.Fatal("join ignored separator bytes") + } + for _, count := range []int{1024, 1025} { + _, err := Pure("split", map[string]any{"text": strings.Repeat(",", count-1), "separator": ","}) + if (err == nil) != (count == 1024) { + t.Errorf("split into %d values: %v", count, err) + } + _, err = Pure("append", map[string]any{"values": make([]any, count-1), "value": nil}) + if (err == nil) != (count == 1024) { + t.Errorf("append to %d values: %v", count, err) + } + } + if _, err := Pure("split", map[string]any{"text": "abc", "separator": ""}); err == nil { + t.Fatal("split accepted an empty separator") + } + object := map[string]any{} + for i := 0; i < 1024; i++ { + object[string(rune(i))] = nil + } + if _, err := Pure("set_field", map[string]any{"object": object, "name": "new", "value": nil}); err == nil { + t.Fatal("set_field exceeded the object limit") + } + if len(object) != 1024 { + t.Fatal("failed set_field changed its input") + } + if _, err := Pure("set_field", map[string]any{"object": object, "name": string(rune(0)), "value": true}); err != nil { + t.Fatal("set_field rejected replacement in a full object:", err) + } +} + +func TestNumericPrimitives(t *testing.T) { + cases := []struct { + block string + in map[string]any + want float64 + }{ + {"multiply", map[string]any{"a": float64(3), "b": float64(4)}, 12}, + {"divide", map[string]any{"a": float64(3), "b": float64(4)}, 0.75}, + {"median", map[string]any{"values": []any{float64(9), float64(1), float64(5)}}, 5}, + {"median", map[string]any{"values": []any{float64(9), float64(1), float64(5), float64(3)}}, 4}, + {"median", map[string]any{"values": []any{math.MaxFloat64, math.MaxFloat64}}, math.MaxFloat64}, + {"percentile", map[string]any{"values": []any{float64(10), float64(0), float64(20)}, "percentile": float64(25)}, 5}, + {"percentile", map[string]any{"values": []any{float64(10), float64(0), float64(20)}, "percentile": float64(0)}, 0}, + {"percentile", map[string]any{"values": []any{float64(10), float64(0), float64(20)}, "percentile": float64(100)}, 20}, + {"percentile", map[string]any{"values": []any{float64(7)}, "percentile": float64(99)}, 7}, + {"percentile", map[string]any{"values": []any{-math.MaxFloat64, math.MaxFloat64}, "percentile": float64(50)}, 0}, + } + for _, tc := range cases { + t.Run(tc.block, func(t *testing.T) { + values, _ := tc.in["values"].([]any) + before := append([]any(nil), values...) + out, err := Pure(tc.block, tc.in) + if err != nil || out["value"] != tc.want { + t.Fatalf("got %v, %v; want %v", out, err, tc.want) + } + if values != nil && !reflect.DeepEqual(values, before) { + t.Fatal("statistic sorted the input in place") + } + }) + } +} + +func TestNumericPrimitiveErrors(t *testing.T) { + for _, block := range []string{"median", "percentile"} { + if _, err := Pure(block, map[string]any{"values": []any{}, "percentile": float64(50)}); err == nil { + t.Errorf("%s accepted no samples", block) + } + } + for _, p := range []float64{-1, 101, math.NaN(), math.Inf(1), math.Inf(-1)} { + if _, err := Pure("percentile", map[string]any{"values": []any{float64(1)}, "percentile": p}); err == nil { + t.Errorf("accepted percentile %v", p) + } + } + for _, tc := range []struct { + block string + a, b float64 + }{ + {"divide", 1, 0}, + {"divide", 1, math.Copysign(0, -1)}, + {"divide", math.MaxFloat64, 0.5}, + {"multiply", math.MaxFloat64, 2}, + } { + if _, err := Pure(tc.block, map[string]any{"a": tc.a, "b": tc.b}); err == nil { + t.Errorf("%s(%v, %v) did not reject an invalid result", tc.block, tc.a, tc.b) + } + } +} diff --git a/pkg/workflow/guest/runtime.go b/pkg/workflow/guest/runtime.go new file mode 100644 index 00000000..aeedd680 --- /dev/null +++ b/pkg/workflow/guest/runtime.go @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// Package guest contains operations linked into generated workflow WASI guests. +package guest + +import ( + "encoding/json" + "fmt" + "io" + "math" + "os" + "reflect" + "sort" + "strings" + "time" +) + +// Runtime holds this run's action results and emits bounded block events. +type Runtime struct { + digest string + writer io.Writer + values map[string]map[string]any + steps, events, bytes int +} + +func New(digest string) *Runtime { + return &Runtime{digest: digest, writer: os.Stdout, values: map[string]map[string]any{}} +} +func (r *Runtime) Step() error { + r.steps++ + if r.steps > 1000 { + return fmt.Errorf("workflow exceeded 1000 action steps") + } + return nil +} +func (r *Runtime) Forget(ids ...string) { + for _, id := range ids { + delete(r.values, id) + } +} +func (r *Runtime) Output(node, port string) (any, error) { + outputs, ok := r.values[node] + if !ok { + return nil, fmt.Errorf("action %s has not executed on this control path", node) + } + value, ok := outputs[port] + if !ok { + return nil, fmt.Errorf("node %s has no output %s", node, port) + } + return value, nil +} +func (r *Runtime) Record(node, block string, outputs map[string]any) error { + if outputs == nil { + outputs = map[string]any{} + } + if err := r.event(node, block, "completed", outputs, ""); err != nil { + return err + } + r.values[node] = outputs + return nil +} +func (r *Runtime) Fail(node, block string, err error) error { + if err == nil { + return nil + } + message := err.Error() + if len(message) > 1024 { + message = message[:1024] + } + if emitErr := r.event(node, block, "failed", map[string]any{}, message); emitErr != nil { + return fmt.Errorf("node %s: %w (trace: %v)", node, err, emitErr) + } + return fmt.Errorf("node %s: %w", node, err) +} +func (r *Runtime) event(node, block, status string, outputs map[string]any, message string) error { + if err := checkValue(outputs, 0); err != nil { + return err + } + event := struct { + Schema string `json:"schema"` + Workflow string `json:"workflow_sha256"` + Node string `json:"node_id"` + Block string `json:"block"` + Status string `json:"status"` + Outputs map[string]any `json:"outputs"` + Error string `json:"error,omitempty"` + }{"debuglet.workflow.v1", r.digest, node, block, status, outputs, message} + b, err := json.Marshal(event) + if err != nil { + return err + } + if len(b) > 65536 { + return fmt.Errorf("node output exceeds 64 KiB") + } + if r.events >= 4096 || r.bytes+len(b)+1 > 4*1024*1024 { + return fmt.Errorf("workflow trace exceeds its output limit") + } + b = append(b, '\n') + n, err := r.writer.Write(b) + r.events++ + r.bytes += n + if err == nil && n != len(b) { + err = io.ErrShortWrite + } + return err +} + +// Check validates JSON values at each typed boundary, including dynamic data. +func Check(values map[string]any, types map[string]string) error { + for name, kind := range types { + v, ok := values[name] + if !ok { + return fmt.Errorf("missing value %s", name) + } + if !Matches(v, kind) { + return fmt.Errorf("%s requires %s", name, kind) + } + if err := checkValue(v, 0); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + return nil +} +func Matches(v any, kind string) bool { + switch kind { + case "any": + return true + case "string": + _, ok := v.(string) + return ok + case "number": + n, ok := v.(float64) + return ok && !math.IsNaN(n) && !math.IsInf(n, 0) + case "boolean": + _, ok := v.(bool) + return ok + case "object": + _, ok := v.(map[string]any) + return ok + case "numbers", "strings": + items, ok := v.([]any) + if !ok { + return false + } + itemType := "number" + if kind == "strings" { + itemType = "string" + } + for _, x := range items { + if !Matches(x, itemType) { + return false + } + } + return true + } + return false +} +func checkValue(v any, depth int) error { + if depth > 16 { + return fmt.Errorf("value is nested more than 16 levels") + } + switch x := v.(type) { + case nil, bool: + case float64: + if math.IsNaN(x) || math.IsInf(x, 0) { + return fmt.Errorf("number must be finite") + } + case string: + if len(x) > 16384 { + return fmt.Errorf("string exceeds 16 KiB") + } + case []any: + if len(x) > 1024 { + return fmt.Errorf("list exceeds 1024 values") + } + for _, v := range x { + if err := checkValue(v, depth+1); err != nil { + return err + } + } + case map[string]any: + if len(x) > 1024 { + return fmt.Errorf("object exceeds 1024 fields") + } + for k, v := range x { + if len(k) > 256 { + return fmt.Errorf("field name exceeds 256 bytes") + } + if err := checkValue(v, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("value of type %T is not a JSON workflow value", v) + } + return nil +} + +// Normalize converts custom Go results to the same JSON types used by wires. +func Normalize(values map[string]any) (map[string]any, error) { + b, err := json.Marshal(values) + if err != nil { + return nil, err + } + if len(b) > 65536 { + return nil, fmt.Errorf("custom result exceeds 64 KiB") + } + var out map[string]any + if err = json.Unmarshal(b, &out); err != nil { + return nil, err + } + if out == nil { + out = map[string]any{} + } + return out, checkValue(out, 0) +} + +// Pure evaluates a built-in operation with already type-checked inputs. +func Pure(block string, in map[string]any) (map[string]any, error) { + var value any + switch block { + case "constant": + value = in["value"] + case "field": + v, ok := in["object"].(map[string]any)[in["name"].(string)] + if !ok { + return nil, fmt.Errorf("field %q is absent", in["name"]) + } + value = v + case "parse_json": + if err := json.Unmarshal([]byte(in["text"].(string)), &value); err != nil { + return nil, fmt.Errorf("invalid JSON: %w", err) + } + case "stringify_json": + encoded, err := json.Marshal(in["value"]) + if err != nil { + return nil, fmt.Errorf("cannot encode JSON: %w", err) + } + value = string(encoded) + case "set_field": + object := in["object"].(map[string]any) + updated := make(map[string]any, len(object)+1) + for key, item := range object { + updated[key] = item + } + updated[in["name"].(string)] = in["value"] + value = updated + case "list_item": + items, ok := in["values"].([]any) + if !ok { + return nil, fmt.Errorf("list_item requires a list") + } + index := in["index"].(float64) + if math.IsNaN(index) || index < 0 || index >= float64(len(items)) || math.Trunc(index) != index { + return nil, fmt.Errorf("list index must be a non-negative integer smaller than the list length") + } + value = items[int(index)] + case "append": + items, ok := in["values"].([]any) + if !ok { + return nil, fmt.Errorf("append requires a list") + } + updated := make([]any, len(items)+1) + copy(updated, items) + updated[len(items)] = in["value"] + value = updated + case "concat": + value = in["a"].(string) + in["b"].(string) + case "contains": + value = strings.Contains(in["text"].(string), in["substring"].(string)) + case "split": + separator := in["separator"].(string) + if separator == "" { + return nil, fmt.Errorf("split requires a non-empty separator") + } + parts := strings.SplitN(in["text"].(string), separator, 1025) + items := make([]any, len(parts)) + for i, part := range parts { + items[i] = part + } + value = items + case "join": + items := in["values"].([]any) + separator := in["separator"].(string) + parts := make([]string, len(items)) + size := 0 + for i, item := range items { + parts[i] = item.(string) + size += len(parts[i]) + if i > 0 { + size += len(separator) + } + if size > 16384 { + return nil, fmt.Errorf("string exceeds 16 KiB") + } + } + value = strings.Join(parts, separator) + case "mean", "min", "max": + items := in["values"].([]any) + if len(items) == 0 { + return nil, fmt.Errorf("%s requires at least one sample", block) + } + n := items[0].(float64) + for i, item := range items { + if i == 0 { + continue + } + x := item.(float64) + switch block { + case "mean": + n += x + case "min": + n = math.Min(n, x) + case "max": + n = math.Max(n, x) + } + } + if block == "mean" { + n /= float64(len(items)) + } + value = n + case "median", "percentile": + items := in["values"].([]any) + if len(items) == 0 { + return nil, fmt.Errorf("%s requires at least one sample", block) + } + percentile := float64(50) + if block == "percentile" { + percentile = in["percentile"].(float64) + } + if math.IsNaN(percentile) || percentile < 0 || percentile > 100 { + return nil, fmt.Errorf("percentile must be between 0 and 100") + } + samples := make([]float64, len(items)) + for i, item := range items { + samples[i] = item.(float64) + } + sort.Float64s(samples) + rank := percentile / 100 * float64(len(samples)-1) + lower, upper := int(math.Floor(rank)), int(math.Ceil(rank)) + fraction := rank - float64(lower) + value = (1-fraction)*samples[lower] + fraction*samples[upper] + case "count": + items, ok := in["values"].([]any) + if !ok { + return nil, fmt.Errorf("count requires a list") + } + value = float64(len(items)) + case "add": + value = in["a"].(float64) + in["b"].(float64) + case "subtract": + value = in["a"].(float64) - in["b"].(float64) + case "multiply": + value = in["a"].(float64) * in["b"].(float64) + case "divide": + divisor := in["b"].(float64) + if divisor == 0 { + return nil, fmt.Errorf("cannot divide by zero") + } + value = in["a"].(float64) / divisor + case "greater_than": + value = in["a"].(float64) > in["b"].(float64) + case "less_than": + value = in["a"].(float64) < in["b"].(float64) + case "equals": + value = reflect.DeepEqual(in["a"], in["b"]) + case "and": + value = in["a"].(bool) && in["b"].(bool) + case "or": + value = in["a"].(bool) || in["b"].(bool) + case "not": + value = !in["value"].(bool) + default: + return nil, fmt.Errorf("unsupported pure block %s", block) + } + out := map[string]any{"value": value} + return out, checkValue(out, 0) +} +func RepeatCount(v any) (int, error) { + n, ok := v.(float64) + if !ok || math.IsNaN(n) || n < 1 || n > 100 || math.Trunc(n) != n { + return 0, fmt.Errorf("repeat count must be an integer from 1 to 100") + } + return int(n), nil +} +func Wait(v any) error { + n, ok := v.(float64) + if !ok || math.IsNaN(n) || n < 0 || n > 60000 { + return fmt.Errorf("wait must be between 0 and 60000 ms") + } + time.Sleep(time.Duration(n * float64(time.Millisecond))) + return nil +} + +// RunCustom reports a returned error or panic at the calling block boundary. +// The executor still bounds code that does not return, including infinite loops. +func RunCustom(fn func(map[string]any) (map[string]any, error), inputs map[string]any) (out map[string]any, err error) { + defer func() { + if problem := recover(); problem != nil { + out = nil + err = fmt.Errorf("custom block panicked: %v", problem) + } + }() + out, err = fn(inputs) + if err != nil { + return nil, err + } + return Normalize(out) +} diff --git a/pkg/workflow/guest/runtime_test.go b/pkg/workflow/guest/runtime_test.go new file mode 100644 index 00000000..1a985116 --- /dev/null +++ b/pkg/workflow/guest/runtime_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package guest + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestRuntimeNeverInventsActionOutputs(t *testing.T) { + r := New(strings.Repeat("a", 64)) + r.writer = &bytes.Buffer{} + if _, err := r.Output("probe", "ok"); err == nil { + t.Fatal("unexecuted action had an output") + } + if err := r.Record("probe", "ping", map[string]any{"ok": false}); err != nil { + t.Fatal(err) + } + value, err := r.Output("probe", "ok") + if err != nil || value != false { + t.Fatalf("failed network outcome lost: %v %v", value, err) + } + r.Forget("probe") + if _, err = r.Output("probe", "ok"); err == nil { + t.Fatal("previous iteration output remained") + } +} +func TestRuntimeTraceAndLimits(t *testing.T) { + var output bytes.Buffer + r := New(strings.Repeat("a", 64)) + r.writer = &output + if err := r.Record("probe", "http", map[string]any{"ok": false, "error": "connection refused"}); err != nil { + t.Fatal(err) + } + var event map[string]any + if err := json.Unmarshal(bytes.TrimSpace(output.Bytes()), &event); err != nil { + t.Fatal(err) + } + if event["schema"] != "debuglet.workflow.v1" || event["status"] != "completed" { + t.Fatal(event) + } + if err := r.Record("large", "emit", map[string]any{"value": strings.Repeat("x", 16385)}); err == nil { + t.Fatal("unbounded string accepted") + } + for i := 0; i < 1000; i++ { + if err := r.Step(); err != nil { + t.Fatal(err) + } + } + if err := r.Step(); err == nil { + t.Fatal("unbounded action steps") + } +} +func TestTypedOperations(t *testing.T) { + if _, err := Pure("mean", map[string]any{"values": []any{}}); err == nil { + t.Fatal("empty samples became zero") + } + if err := Check(map[string]any{"condition": "true"}, map[string]string{"condition": "boolean"}); err == nil { + t.Fatal("string silently coerced to boolean") + } + out, err := Pure("mean", map[string]any{"values": []any{float64(10), float64(20)}}) + if err != nil || out["value"] != float64(15) { + t.Fatalf("mean = %v %v", out, err) + } + if _, err = Pure("field", map[string]any{"object": map[string]any{}, "name": "missing"}); err == nil { + t.Fatal("missing field invented") + } + out, err = Normalize(map[string]any{"values": []float64{10, 20}, "count": 2}) + if err != nil { + t.Fatal(err) + } + if err = Check(out, map[string]string{"values": "numbers", "count": "number"}); err != nil { + t.Fatal(err) + } +} + +func TestCustomPanicIsAnError(t *testing.T) { + _, err := RunCustom(func(map[string]any) (map[string]any, error) { panic("bad sample") }, map[string]any{}) + if err == nil || !strings.Contains(err.Error(), "bad sample") { + t.Fatalf("panic outcome: %v", err) + } +} diff --git a/pkg/workflow/model.go b/pkg/workflow/model.go new file mode 100644 index 00000000..ef44dcaa --- /dev/null +++ b/pkg/workflow/model.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +// Package workflow validates visual measurement programs and generates Go WASI guests. +package workflow + +// Workflow is the portable source shared by visual and programmatic authors. +type Workflow struct { + Version int `json:"version"` + Experiment *ExperimentContext `json:"experiment,omitempty"` + Name string `json:"name"` + Entry string `json:"entry"` + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` + CustomBlocks []CustomBlock `json:"custom_blocks,omitempty"` + Groups []Group `json:"groups,omitempty"` +} + +// Group folds ordinary nodes in the editor without changing their execution. +type Group struct { + ID string `json:"id"` + Label string `json:"label"` + NodeIDs []string `json:"node_ids"` + Position *Position `json:"position,omitempty"` +} + +type Node struct { + ID string `json:"id"` + Block string `json:"block"` + Inputs map[string]any `json:"inputs,omitempty"` + Position *Position `json:"position,omitempty"` +} +type Position struct { + X float64 `json:"x"` + Y float64 `json:"y"` +} +type Endpoint struct { + Node string `json:"node"` + Port string `json:"port"` +} +type Edge struct { + Kind string `json:"kind"` + From Endpoint `json:"from"` + To Endpoint `json:"to"` +} + +type Port struct { + Name string `json:"name"` + Type string `json:"type"` + Required bool `json:"required,omitempty"` + Default any `json:"default,omitempty"` +} +type Block struct { + ID string `json:"id"` + Label string `json:"label"` + Category string `json:"category"` + Description string `json:"description"` + Pure bool `json:"pure"` + Inputs []Port `json:"inputs"` + Outputs []Port `json:"outputs"` + ControlOutputs []string `json:"control_outputs"` +} + +// CustomBlock is a Go function body, compiled inside the same guest as its callers. +// It is an action: data connections alone never cause it to execute. +type CustomBlock struct { + ID string `json:"id"` + Label string `json:"label"` + Inputs []Port `json:"inputs"` + Outputs []Port `json:"outputs"` + Code string `json:"code"` +} +type Diagnostic struct { + Node string `json:"node,omitempty"` + Port string `json:"port,omitempty"` + Message string `json:"message"` +} +type Program struct { + Source string `json:"source"` + WorkflowSHA256 string `json:"workflow_sha256"` + Destinations []string `json:"destinations"` + RequireICMP bool `json:"require_icmp"` +} diff --git a/pkg/workflow/validate.go b/pkg/workflow/validate.go new file mode 100644 index 00000000..be291ede --- /dev/null +++ b/pkg/workflow/validate.go @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "math" + "regexp" + "slices" + "strings" +) + +var identifier = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`) +var valueTypes = []string{"string", "number", "boolean", "numbers", "strings", "object", "any"} + +// Validate checks the portable graph without compiling or executing custom code. +func Validate(w Workflow) []Diagnostic { + var ds []Diagnostic + add := func(node, port, msg string) { ds = append(ds, Diagnostic{node, port, msg}) } + if w.Version != 1 { + add("", "", "workflow version must be 1") + } + if strings.TrimSpace(w.Name) == "" || len(w.Name) > 128 { + add("", "", "name must contain 1–128 characters") + } + if len(w.Nodes) == 0 || len(w.Nodes) > 128 { + add("", "", "workflow must contain 1–128 nodes") + return ds + } + if len(w.Edges) > 512 { + add("", "", "workflow has more than 512 edges") + return ds + } + if len(w.CustomBlocks) > 32 { + add("", "", "workflow has more than 32 custom blocks") + return ds + } + if len(w.Groups) > 64 { + add("", "", "workflow has more than 64 groups") + return ds + } + catalog := map[string]Block{} + for _, b := range Catalog() { + catalog[b.ID] = b + } + for _, b := range w.CustomBlocks { + if !identifier.MatchString(b.ID) { + add("", "", "custom block has invalid id: "+b.ID) + continue + } + if _, exists := catalog[b.ID]; exists { + add("", "", "duplicate or reserved custom block id: "+b.ID) + continue + } + if strings.TrimSpace(b.Label) == "" || len(b.Label) > 128 { + add("", "", "custom block needs a label of at most 128 characters") + } + for _, ports := range [][]Port{b.Inputs, b.Outputs} { + seen := map[string]bool{} + if len(ports) > 32 { + add("", "", "custom block has more than 32 ports") + } + for _, p := range ports { + if !identifier.MatchString(p.Name) || seen[p.Name] { + add("", "", "invalid or duplicate custom port: "+p.Name) + } + seen[p.Name] = true + if !slices.Contains(valueTypes, p.Type) { + add("", "", "unsupported port type: "+p.Type) + } + if p.Default != nil { + if !matches(p.Default, p.Type) { + add("", "", "invalid default for custom port: "+p.Name) + } + if err := boundedValue(p.Default, 0); err != nil { + add("", "", "custom default "+p.Name+": "+err.Error()) + } + } + } + } + if err := validateCustom(b.Code); err != nil { + add("", "", fmt.Sprintf("custom block %s: %v", b.ID, err)) + } + catalog[b.ID] = Block{b.ID, b.Label, "Custom", "", false, b.Inputs, b.Outputs, []string{"next"}} + } + nodes := map[string]Node{} + for _, n := range w.Nodes { + if !identifier.MatchString(n.ID) { + add(n.ID, "", "node id must start with a letter and use letters, digits, hyphens or underscores (64 characters maximum)") + } + if _, ok := nodes[n.ID]; ok { + add(n.ID, "", "duplicate node id") + } + nodes[n.ID] = n + b, ok := catalog[n.Block] + if !ok { + add(n.ID, "", "unknown block: "+n.Block) + continue + } + for k, v := range n.Inputs { + p, found := port(b.Inputs, k) + if !found { + add(n.ID, k, "unknown input") + continue + } + if !matches(v, p.Type) { + add(n.ID, k, "expected "+p.Type) + } + if err := boundedValue(v, 0); err != nil { + add(n.ID, k, err.Error()) + } + } + if n.Position != nil && (math.IsNaN(n.Position.X) || math.IsInf(n.Position.X, 0) || math.IsNaN(n.Position.Y) || math.IsInf(n.Position.Y, 0)) { + add(n.ID, "", "position must be finite") + } + } + groupIDs, groupedNodes := map[string]bool{}, map[string]bool{} + for _, g := range w.Groups { + if !identifier.MatchString(g.ID) || groupIDs[g.ID] || nodes[g.ID].ID != "" { + add("", "", "group id must be valid and distinct from node and group ids") + } + groupIDs[g.ID] = true + if strings.TrimSpace(g.Label) == "" || len(g.Label) > 128 { + add("", "", "group needs a label of at most 128 bytes") + } + if len(g.NodeIDs) < 2 || len(g.NodeIDs) > 128 { + add("", "", "group must contain 2–128 nodes") + } + for _, id := range g.NodeIDs { + if _, exists := nodes[id]; !exists || groupedNodes[id] { + add(id, "", "group members must be existing nodes and belong to only one group") + } + groupedNodes[id] = true + } + if g.Position != nil && (math.IsNaN(g.Position.X) || math.IsInf(g.Position.X, 0) || math.IsNaN(g.Position.Y) || math.IsInf(g.Position.Y, 0)) { + add("", "", "group position must be finite") + } + } + entry, ok := nodes[w.Entry] + if !ok { + add("", "", "entry must name a node") + } else if catalog[entry.Block].Pure { + add(entry.ID, "", "entry must be an action or control block") + } + connected := map[Endpoint]bool{} + controls := map[Endpoint]bool{} + dataAdj, controlAdj := map[string][]string{}, map[string][]string{} + for _, e := range w.Edges { + source, sok := nodes[e.From.Node] + target, tok := nodes[e.To.Node] + if !sok || !tok { + add(e.To.Node, e.To.Port, "edge refers to a missing node") + continue + } + sb, tb := catalog[source.Block], catalog[target.Block] + switch e.Kind { + case "data": + sp, spok := port(sb.Outputs, e.From.Port) + tp, tpok := port(tb.Inputs, e.To.Port) + if !spok { + add(source.ID, e.From.Port, "unknown output") + } + if !tpok { + add(target.ID, e.To.Port, "unknown input") + } + if spok && tpok && sp.Type != tp.Type && sp.Type != "any" && tp.Type != "any" { + add(target.ID, e.To.Port, "cannot connect "+sp.Type+" to "+tp.Type) + } + if connected[e.To] { + add(target.ID, e.To.Port, "input has multiple connections") + } + if _, ok := target.Inputs[e.To.Port]; ok { + add(target.ID, e.To.Port, "connected input must not also have a literal") + } + connected[e.To] = true + dataAdj[source.ID] = append(dataAdj[source.ID], target.ID) + case "control": + if sb.Pure || !slices.Contains(sb.ControlOutputs, e.From.Port) { + add(source.ID, e.From.Port, "unknown control output") + } + if tb.Pure || e.To.Port != "in" { + add(target.ID, e.To.Port, "control edges must enter an action's in port") + } + if controls[e.From] { + add(source.ID, e.From.Port, "control output has multiple connections; use an explicit branch") + } + controls[e.From] = true + controlAdj[source.ID] = append(controlAdj[source.ID], target.ID) + default: + add(target.ID, e.To.Port, "edge kind must be data or control") + } + } + for _, n := range w.Nodes { + b, ok := catalog[n.Block] + if !ok { + continue + } + for _, p := range b.Inputs { + _, literal := n.Inputs[p.Name] + if p.Required && !literal && !connected[Endpoint{n.ID, p.Name}] { + add(n.ID, p.Name, "required input is missing") + } + } + for _, field := range []string{"count", "timeout_ms", "duration_ms"} { + v, ok := n.Inputs[field] + if !ok { + continue + } + number, isNumber := asNumber(v) + if !isNumber { + continue + } + if field == "count" && (n.Block == "repeat" || n.Block == "ping") { + max := float64(100) + if n.Block == "ping" { + max = 10 + } + if number < 1 || number > max || math.Trunc(number) != number { + add(n.ID, field, fmt.Sprintf("must be an integer from 1 to %.0f", max)) + } + } + if field == "timeout_ms" && (number < 1 || number > 60000 || math.Trunc(number) != number) { + add(n.ID, field, "must be an integer between 1 and 60000 ms") + } + if field == "duration_ms" && (number < 0 || number > 60000) { + add(n.ID, field, "must be between 0 and 60000 ms") + } + } + } + if hasCycle(dataAdj) { + add("", "", "data connections contain a cycle") + } + if hasCycle(controlAdj) { + add("", "", "control connections contain a cycle; use Repeat for bounded repetition") + } else { + for _, n := range w.Nodes { + if n.Block != "repeat" { + continue + } + var body, done string + for _, e := range w.Edges { + if e.Kind == "control" && e.From.Node == n.ID { + if e.From.Port == "body" { + body = e.To.Node + } + if e.From.Port == "done" { + done = e.To.Node + } + } + } + if body == "" { + add(n.ID, "body", "repeat needs a body connection") + continue + } + inside := reachable(body, controlAdj) + after := reachable(done, controlAdj) + if inside[w.Entry] { + add(n.ID, "body", "workflow entry cannot be inside a repeat body") + } + for id := range inside { + if after[id] { + add(n.ID, "body", "repeat body must end before the done continuation") + break + } + } + for _, e := range w.Edges { + if e.Kind == "control" && inside[e.To.Node] && !inside[e.From.Node] && !(e.From.Node == n.ID && e.From.Port == "body") { + add(n.ID, "body", "repeat body cannot have control entry from outside its body") + break + } + } + } + } + return ds +} + +func port(ports []Port, name string) (Port, bool) { + for _, p := range ports { + if p.Name == name { + return p, true + } + } + return Port{}, false +} +func asNumber(v any) (float64, bool) { + switch n := v.(type) { + case float64: + return n, !math.IsNaN(n) && !math.IsInf(n, 0) + case int: + return float64(n), true + case int64: + return float64(n), true + case json.Number: + f, e := n.Float64() + return f, e == nil && !math.IsNaN(f) && !math.IsInf(f, 0) + } + return 0, false +} +func matches(v any, kind string) bool { + switch kind { + case "any": + return true + case "string": + _, ok := v.(string) + return ok + case "number": + _, ok := asNumber(v) + return ok + case "boolean": + _, ok := v.(bool) + return ok + case "object": + _, ok := v.(map[string]any) + return ok + case "numbers", "strings": + a, ok := v.([]any) + if !ok { + return false + } + typ := "number" + if kind == "strings" { + typ = "string" + } + for _, item := range a { + if !matches(item, typ) { + return false + } + } + return true + } + return false +} +func boundedValue(v any, depth int) error { + if depth > 16 { + return fmt.Errorf("value is nested more than 16 levels") + } + switch x := v.(type) { + case string: + if len(x) > 16384 { + return fmt.Errorf("string exceeds 16 KiB") + } + case []any: + if len(x) > 1024 { + return fmt.Errorf("list exceeds 1024 values") + } + for _, v := range x { + if err := boundedValue(v, depth+1); err != nil { + return err + } + } + case map[string]any: + if len(x) > 1024 { + return fmt.Errorf("object exceeds 1024 fields") + } + for k, v := range x { + if len(k) > 256 { + return fmt.Errorf("object field name exceeds 256 bytes") + } + if err := boundedValue(v, depth+1); err != nil { + return err + } + } + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("value is not JSON: %w", err) + } + if len(b) > 65536 { + return fmt.Errorf("value exceeds 64 KiB") + } + return nil +} +func hasCycle(adj map[string][]string) bool { + state := map[string]int{} + var visit func(string) bool + visit = func(n string) bool { + if state[n] == 1 { + return true + } + if state[n] == 2 { + return false + } + state[n] = 1 + for _, child := range adj[n] { + if visit(child) { + return true + } + } + state[n] = 2 + return false + } + for n := range adj { + if visit(n) { + return true + } + } + return false +} +func reachable(start string, adj map[string][]string) map[string]bool { + seen := map[string]bool{} + var visit func(string) + visit = func(n string) { + if n == "" || seen[n] { + return + } + seen[n] = true + for _, child := range adj[n] { + visit(child) + } + } + visit(start) + return seen +} +func validateCustom(body string) error { + if len(body) == 0 || len(body) > 32768 { + return fmt.Errorf("code must contain 1–32768 bytes") + } + f, err := parser.ParseFile(token.NewFileSet(), "custom.go", "package main\nfunc custom(inputs map[string]any) (map[string]any,error) {\n"+body+"\n}", parser.ParseComments) + if err != nil { + return fmt.Errorf("invalid Go function body: %w", err) + } + if len(f.Decls) != 1 { + return fmt.Errorf("code must be a single function body") + } + fn, ok := f.Decls[0].(*ast.FuncDecl) + if !ok || fn.Name.Name != "custom" { + return fmt.Errorf("code must be a single function body") + } + for _, group := range f.Comments { + for _, c := range group.List { + if strings.Contains(c.Text, "go:") { + return fmt.Errorf("compiler directives are not supported") + } + } + } + return nil +} diff --git a/pkg/workflow/workflow_test.go b/pkg/workflow/workflow_test.go new file mode 100644 index 00000000..c8ac3f33 --- /dev/null +++ b/pkg/workflow/workflow_test.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package workflow + +import ( + "encoding/json" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func sampleWorkflow() Workflow { + return Workflow{Version: 1, Name: "Sample", Entry: "result", Nodes: []Node{{ID: "result", Block: "emit", Inputs: map[string]any{"name": "answer", "value": float64(42)}}}, Edges: []Edge{}} +} +func edge(kind, from, port, to, input string) Edge { + return Edge{kind, Endpoint{from, port}, Endpoint{to, input}} +} +func TestValidationRejectsAmbiguousAndUnsafeGraphs(t *testing.T) { + cases := map[string]func(*Workflow){ + "unknown block": func(w *Workflow) { w.Nodes[0].Block = "missing" }, + "duplicate node": func(w *Workflow) { w.Nodes = append(w.Nodes, w.Nodes[0]) }, + "unknown literal": func(w *Workflow) { w.Nodes[0].Inputs["typo"] = true }, + "fractional timeout": func(w *Workflow) { + w.Nodes = append(w.Nodes, Node{ID: "http", Block: "http", Inputs: map[string]any{"url": "https://example.test", "timeout_ms": 1.5}}) + }, + "wrong type": func(w *Workflow) { w.Nodes[0].Inputs["name"] = float64(1) }, + "missing input": func(w *Workflow) { delete(w.Nodes[0].Inputs, "name") }, + "control cycle": func(w *Workflow) { w.Edges = append(w.Edges, edge("control", "result", "next", "result", "in")) }, + "literal and wire": func(w *Workflow) { + w.Nodes = append(w.Nodes, Node{ID: "c", Block: "constant", Inputs: map[string]any{"value": true}}) + w.Edges = append(w.Edges, edge("data", "c", "value", "result", "value")) + }, + "data cycle": func(w *Workflow) { + w.Nodes = append(w.Nodes, Node{ID: "c", Block: "constant"}) + delete(w.Nodes[0].Inputs, "value") + w.Edges = append(w.Edges, edge("data", "c", "value", "result", "value"), edge("data", "result", "value", "c", "value")) + }, + "code escapes function": func(w *Workflow) { + w.CustomBlocks = []CustomBlock{{ID: "bad", Label: "Bad", Code: "return nil,nil\n}\nfunc init(){ panic(\"escape\") }\nfunc other(){"}} + }, + "large custom default": func(w *Workflow) { + w.CustomBlocks = []CustomBlock{{ID: "bad", Label: "Bad", Inputs: []Port{{Name: "text", Type: "string", Default: strings.Repeat("a", 16385)}}, Code: "return nil,nil"}} + }, + "compiler directive": func(w *Workflow) { + w.CustomBlocks = []CustomBlock{{ID: "bad", Label: "Bad", Code: "//go:linkname hidden\nreturn nil,nil"}} + }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + w := sampleWorkflow() + change(&w) + if ds := Validate(w); len(ds) == 0 { + t.Fatal("accepted invalid graph") + } + }) + } +} +func TestRepeatBodyCannotEscape(t *testing.T) { + w := Workflow{Version: 1, Name: "Repeat", Entry: "repeat", Nodes: []Node{{ID: "repeat", Block: "repeat", Inputs: map[string]any{"count": float64(2)}}, {ID: "body", Block: "wait", Inputs: map[string]any{"duration_ms": float64(0)}}, {ID: "done", Block: "emit", Inputs: map[string]any{"name": "done", "value": true}}}, Edges: []Edge{edge("control", "repeat", "body", "body", "in"), edge("control", "repeat", "done", "done", "in")}} + if ds := Validate(w); len(ds) > 0 { + t.Fatal(ds) + } + w.Edges = append(w.Edges, edge("control", "body", "next", "done", "in")) + if ds := Validate(w); len(ds) == 0 { + t.Fatal("accepted body escaping to done") + } +} +func TestExampleProgramsGenerate(t *testing.T) { + files, err := filepath.Glob("../../examples/workflows/*.json") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + if filepath.Base(file) == "catalog.json" { + continue + } + t.Run(filepath.Base(file), func(t *testing.T) { + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var w Workflow + if err = json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + if ds := Validate(w); len(ds) > 0 { + t.Fatal(ds) + } + program, err := Generate(w) + if err != nil { + t.Fatal(err) + } + if _, err = parser.ParseFile(token.NewFileSet(), "main.go", program.Source, parser.AllErrors); err != nil { + t.Fatal(err) + } + if len(program.WorkflowSHA256) != 64 { + t.Fatal("missing digest") + } + if filepath.Base(file) == "ping-branch.json" && !program.RequireICMP { + t.Fatal("Ping missing ICMP requirement") + } + }) + } +} +func TestDestinationHints(t *testing.T) { + w := Workflow{Version: 1, Name: "Targets", Entry: "http", Nodes: []Node{{ID: "http", Block: "http", Inputs: map[string]any{"url": "https://example.test:8443/a"}}, {ID: "tcp", Block: "tcp_connect", Inputs: map[string]any{"target": "127.0.0.1:1234"}}}, Edges: []Edge{edge("control", "http", "next", "tcp", "in")}} + p, err := Generate(w) + if err != nil { + t.Fatal(err) + } + if strings.Join(p.Destinations, ",") != "127.0.0.1,example.test" { + t.Fatal(p.Destinations) + } +} + +// Running generated code catches mistakes that syntax-only generation tests miss. +func TestGeneratedProgramExecutesBranchesAndRepeat(t *testing.T) { + if testing.Short() { + t.Skip("generated program compilation") + } + w := Workflow{Version: 1, Name: "Bounded branch", Entry: "custom", CustomBlocks: []CustomBlock{{ID: "shadowed", Label: "Shadowed name", Inputs: []Port{}, Outputs: []Port{{Name: "value", Type: "string"}}, Code: "strings := struct{Value string}{\"ok\"}; return map[string]any{\"value\":strings.Value},nil"}}, Nodes: []Node{ + {ID: "custom", Block: "shadowed"}, + {ID: "repeat", Block: "repeat", Inputs: map[string]any{"count": float64(3)}}, + {ID: "compare", Block: "greater_than", Inputs: map[string]any{"b": float64(1)}}, + {ID: "branch", Block: "if"}, + {ID: "yes", Block: "emit", Inputs: map[string]any{"name": "later"}}, + {ID: "no", Block: "emit", Inputs: map[string]any{"name": "first"}}, + {ID: "done", Block: "emit", Inputs: map[string]any{"name": "done", "value": true}}, + }, Edges: []Edge{edge("control", "custom", "next", "repeat", "in"), edge("control", "repeat", "body", "branch", "in"), edge("control", "repeat", "done", "done", "in"), edge("data", "repeat", "iteration", "compare", "a"), edge("data", "compare", "value", "branch", "condition"), edge("control", "branch", "true", "yes", "in"), edge("control", "branch", "false", "no", "in"), edge("data", "repeat", "iteration", "yes", "value"), edge("data", "repeat", "iteration", "no", "value")}} + p, err := Generate(w) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + source := filepath.Join(dir, "main.go") + if err = os.WriteFile(source, []byte(p.Source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "run", source) + cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOSUMDB=off") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("generated program failed: %v\n%s", err, out) + } + counts := map[string]int{} + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + var e struct { + Status string `json:"status"` + Node string `json:"node_id"` + } + if err = json.Unmarshal([]byte(line), &e); err != nil { + t.Fatal(err) + } + if e.Status != "completed" { + t.Fatal(string(line)) + } + counts[e.Node]++ + } + if counts["yes"] != 2 || counts["no"] != 1 || counts["done"] != 1 { + t.Fatal(counts) + } +} + +func TestCatalogFixtureMatches(t *testing.T) { + raw, err := os.ReadFile("../../examples/workflows/catalog.json") + if err != nil { + t.Fatal(err) + } + var fixture []Block + if err = json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(Catalog(), fixture) { + t.Fatal("catalog fixture differs from Catalog()") + } +}