diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml new file mode 100644 index 000000000..798e5fdcc --- /dev/null +++ b/.github/workflows/benchmark-cli.yml @@ -0,0 +1,40 @@ +name: Benchmark CLI build + +on: + pull_request: + paths: + - crates/cli/** + - crates/sandbox/** + - .github/workflows/benchmark-cli.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux-cli: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-unknown-linux-musl + - uses: mlugg/setup-zig@v2 + - uses: taiki-e/install-action@v2 + with: + tool: cargo-zigbuild@0.23.0 + - uses: Swatinem/rust-cache@v2 + with: + key: benchmark-linux-cli + - name: Verify detached Linux kernel fence + run: cargo test -p devo-sandbox --test headless_tty -- --nocapture + - name: Build native Linux CLI for container smoke runs + run: cargo zigbuild --target x86_64-unknown-linux-musl -p devo-cli + - name: Strip debug information from the benchmark artifact + run: strip target/x86_64-unknown-linux-musl/debug/devo + - uses: actions/upload-artifact@v4 + with: + name: devo-benchmark-linux-${{ github.event.pull_request.head.sha || github.sha }} + path: target/x86_64-unknown-linux-musl/debug/devo + if-no-files-found: error + retention-days: 7 diff --git a/crates/cli/src/prompt_command.rs b/crates/cli/src/prompt_command.rs index cdfc48f93..8f3c5782f 100644 --- a/crates/cli/src/prompt_command.rs +++ b/crates/cli/src/prompt_command.rs @@ -64,7 +64,7 @@ pub(crate) async fn run_prompt( model_override, &resolved_provider.default_model, Some(home_dir.as_path()), - ); + )?; let selected_model = turn_config.model.slug.clone(); let mut session_state = SessionState::new( @@ -113,7 +113,7 @@ pub(crate) async fn run_prompt( let registry = Arc::new(handlers::build_registry_from_plan(&tool_plan)); let runtime = Arc::new(ToolRuntime::new_with_context( std::sync::Arc::clone(®istry), - devo_core::tools::PermissionChecker::always_allow(), + crate::prompt_host::permission_checker(), devo_core::tools::ToolRuntimeContext { session_id: session_state.id.as_str().into(), turn_id: None, @@ -268,7 +268,7 @@ fn prompt_turn_config( requested_model: Option<&str>, default_model: &str, home_dir: Option<&Path>, -) -> TurnConfig { +) -> Result { let catalog_model = |model_slug: &str| { model_catalog .get(model_slug) @@ -286,10 +286,16 @@ fn prompt_turn_config( let selected_model = requested_model .or(effective.model.as_deref()) .or(Some(default_model)); - if let Some(selection) = selected_model - .and_then(|model| effective.resolve_model(Some(model)).ok()) - .or_else(|| effective.resolve_model(None).ok()) - { + let selection = if let Some(requested) = requested_model { + Some(effective.resolve_model(Some(requested)).map_err(|error| { + anyhow::anyhow!("cannot select requested model `{requested}`: {error}") + })?) + } else { + selected_model + .and_then(|model| effective.resolve_model(Some(model)).ok()) + .or_else(|| effective.resolve_model(None).ok()) + }; + if let Some(selection) = selection { let model_reference = format!("{}/{}", selection.provider_id, selection.model_id); let mut model_config = effective .providers @@ -340,14 +346,14 @@ fn prompt_turn_config( ); turn_config.model_binding_id = Some(model_reference); turn_config.variant = variant_id; - return turn_config; + return Ok(turn_config); } let selected_model = requested_model.unwrap_or(default_model); - TurnConfig::new( + Ok(TurnConfig::new( catalog_model(selected_model), provider_config.reasoning_effort.clone(), - ) + )) } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] diff --git a/crates/cli/src/prompt_command/prompt_routing_tests.rs b/crates/cli/src/prompt_command/prompt_routing_tests.rs index 258439a20..09ce9fe90 100644 --- a/crates/cli/src/prompt_command/prompt_routing_tests.rs +++ b/crates/cli/src/prompt_command/prompt_routing_tests.rs @@ -148,7 +148,8 @@ fn prompt_turn_config_routes_requested_connection_model_to_provider_route() { Some("anthropic-provider/alt/fast"), "default-provider/main", None, - ); + ) + .expect("configured model route"); assert_eq!(turn_config.model.slug, "anthropic-provider/alt"); assert_eq!(turn_config.request_model, "alt"); @@ -176,3 +177,25 @@ async fn routed_prompt_provider_forwards_configured_route() { assert_eq!(router.calls(), vec![(route, "vendor/alt".to_string())]); } + +#[test] +fn invalid_explicit_prompt_model_does_not_fall_back_to_another_model() { + let app_config = AppConfig::default(); + let catalog = PresetModelCatalog::new(Vec::new()); + for requested in ["gpt-5.6-luna", "missing-provider/missing-model"] { + let error = prompt_turn_config( + &app_config, + &catalog, + Some(requested), + "openai/gpt-5.5", + None, + ) + .expect_err("invalid explicit selection must fail"); + assert!( + error + .to_string() + .starts_with(&format!("cannot select requested model `{requested}`:")), + "{error}" + ); + } +} diff --git a/crates/cli/src/prompt_host.rs b/crates/cli/src/prompt_host.rs index 028f940c7..b24f8880a 100644 --- a/crates/cli/src/prompt_host.rs +++ b/crates/cli/src/prompt_host.rs @@ -1,9 +1,22 @@ use std::sync::Arc; -use devo_core::tools::{ToolCall, ToolContent, ToolRuntime}; +use devo_core::tools::{PermissionChecker, PermissionGrant, ToolCall, ToolContent, ToolRuntime}; use devo_kernel::HostRequestHandler; use serde_json::{Map, Value, json}; +/// Preserve the headless command's static-allow policy while delivering requested grants. +pub(crate) fn permission_checker() -> PermissionChecker { + PermissionChecker::new(|request| { + Box::pin(async move { + Ok(PermissionGrant { + bypass_sandbox: request.sandbox_permissions.bypasses_sandbox(), + already_approved: false, + sandbox_permission_overlay: request.sandbox_permissions.overlay(), + }) + }) + }) +} + pub(crate) fn host_request_handler(runtime: Arc) -> HostRequestHandler { Arc::new(move |request_id, data| { let runtime = Arc::clone(&runtime); @@ -21,6 +34,12 @@ async fn dispatch_host_request(runtime: &ToolRuntime, request_id: String, data: }; let (tool_name, input) = match action { + "bash" | "bash.start" => { + let mut input = data.as_object().cloned().unwrap_or_default(); + input.remove("type"); + input.remove("action"); + ("bash", Value::Object(input)) + } "web.search" => ( "web_search", json!({ "query": data.get("query").cloned().unwrap_or(Value::Null) }), @@ -91,11 +110,129 @@ fn error_reply(error: impl Into) -> Value { mod tests { use std::sync::Arc; - use devo_core::tools::{ToolRegistry, ToolRuntime}; + use devo_core::tools::{ + ToolCallError, ToolContext, ToolHandler, ToolProgressSender, ToolRegistry, + ToolRegistryBuilder, ToolResult, ToolResultContent, ToolRuntime, ToolSpec, + }; use pretty_assertions::assert_eq; use serde_json::json; - use super::{content_as_json, dispatch_host_request}; + use super::{content_as_json, dispatch_host_request, permission_checker}; + + #[tokio::test] + async fn headless_policy_delivers_requested_grants_as_static_allows() { + use devo_core::tools::{ + AdditionalSandboxPermissions, NetworkPermission, PermissionGrant, + SandboxPermissionRequest, ToolPermissionRequest, + }; + let overlay = AdditionalSandboxPermissions { + network: NetworkPermission::Enabled, + read_paths: Vec::new(), + write_paths: vec![std::env::temp_dir()], + }; + for (permissions, expected) in [ + ( + SandboxPermissionRequest::Default, + PermissionGrant::default(), + ), + ( + SandboxPermissionRequest::FullEscalation, + PermissionGrant { + bypass_sandbox: true, + already_approved: false, + sandbox_permission_overlay: None, + }, + ), + ( + SandboxPermissionRequest::AdditionalPermissions(overlay.clone()), + PermissionGrant { + bypass_sandbox: false, + already_approved: false, + sandbox_permission_overlay: Some(overlay), + }, + ), + ] { + let request = ToolPermissionRequest { + tool_call_id: "headless-shell".to_string(), + tool_name: "bash".to_string(), + input: json!({"command":"echo probe"}), + cwd: std::env::temp_dir(), + session_id: "ses_headless_shell".into(), + turn_id: None, + resource: devo_safety::ResourceKind::ShellExec, + action_summary: "headless shell probe".to_string(), + justification: None, + path: None, + host: None, + target: Some("echo probe".to_string()), + command_prefix: None, + command_argv: None, + command_pattern: None, + sandbox_permissions: permissions, + }; + assert_eq!(permission_checker().check(request).await, Ok(expected)); + } + } + + struct ShellInputProbe { + spec: ToolSpec, + } + + #[async_trait::async_trait] + impl ToolHandler for ShellInputProbe { + fn spec(&self) -> &ToolSpec { + &self.spec + } + + async fn handle( + &self, + _ctx: ToolContext, + input: serde_json::Value, + _progress: Option, + ) -> Result { + Ok(ToolResult::success(ToolResultContent::Json(input), "probe")) + } + } + + #[tokio::test] + async fn prompt_host_forwards_shell_requests_to_the_hidden_runtime_handler() { + let mut builder = ToolRegistryBuilder::new(); + let mut spec = devo_core::tools::handlers::ShellCommandHandler::new() + .spec() + .clone(); + spec.name = "bash".to_string(); + builder.push_spec(spec.clone()); + builder.register_handler("bash", Arc::new(ShellInputProbe { spec })); + builder.restrict_model_exposure_to(&["ipython"]); + let runtime = ToolRuntime::new_without_permissions(Arc::new(builder.build())); + for action in ["bash", "bash.start"] { + assert_eq!( + dispatch_host_request( + &runtime, + "shell-probe".to_string(), + json!({ + "type": action, + "command": "echo headless", + "sandbox_permissions": "require_escalated", + "justification": "May I run the headless probe?", + "timeout_ms": 1000, + "prefix_rule": ["echo"], + }) + ) + .await, + json!({ + "status": "ok", + "result": { + "command": "echo headless", + "sandbox_permissions": "require_escalated", + "justification": "May I run the headless probe?", + "timeout_ms": 1000, + "prefix_rule": ["echo"], + } + }) + ); + } + } #[test] fn mixed_tool_content_preserves_text_and_metadata_for_python() { diff --git a/crates/sandbox/src/profiles.rs b/crates/sandbox/src/profiles.rs index 4c9a335c4..6397c4366 100644 --- a/crates/sandbox/src/profiles.rs +++ b/crates/sandbox/src/profiles.rs @@ -336,6 +336,17 @@ impl ProfileName { if !p.exists() { continue; } + // nono opens device rules read-only when applying Landlock. Linux + // exposes /dev/tty even without a controlling terminal, where open + // returns ENXIO and aborts the entire headless child sandbox. Omit + // only that unavailable optional device; retain every other rule. + #[cfg(target_os = "linux")] + if *dev == "/dev/tty" + && std::fs::File::open(p) + .is_err_and(|error| error.raw_os_error() == Some(libc::ENXIO)) + { + continue; + } if profile.name == "rlm-kernel" && *dev == "/dev/fd" && cfg!(target_os = "linux") { // /dev/fd points to /proc/self/fd. CapabilitySet canonicalizes // it in the server *before* spawning the kernel, which would diff --git a/crates/sandbox/tests/headless_tty.rs b/crates/sandbox/tests/headless_tty.rs new file mode 100644 index 000000000..f40e7c3a9 --- /dev/null +++ b/crates/sandbox/tests/headless_tty.rs @@ -0,0 +1,69 @@ +//! A Linux sandbox must start without a controlling terminal and keep its fence. +#![cfg(all(target_os = "linux", feature = "enforce"))] + +use std::os::unix::process::CommandExt; +use std::process::Command; + +use pretty_assertions::assert_eq; + +#[test] +fn kernel_sandbox_starts_without_controlling_terminal() { + let support = nono::Sandbox::support_info(); + if !support.is_supported { + eprintln!("skipping: Landlock unavailable: {}", support.details); + return; + } + let output = unsafe { + Command::new(std::env::current_exe().expect("test executable")) + .args([ + "--ignored", + "--exact", + "--nocapture", + "headless_kernel_child", + ]) + .pre_exec(|| { + if libc::setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }) + .output() + } + .expect("spawn detached sandbox probe"); + assert!( + output.status.success(), + "headless fence failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + +#[test] +#[ignore = "invoked in a detached process by kernel_sandbox_starts_without_controlling_terminal"] +fn headless_kernel_child() { + assert_eq!( + std::fs::File::open("/dev/tty") + .expect_err("probe must have no controlling terminal") + .raw_os_error(), + Some(libc::ENXIO), + ); + let plan = devo_sandbox::resolve_enforcement_plan(Some("rlm-kernel"), &std::env::temp_dir()) + .expect("resolve kernel fence") + .expect("active fence"); + devo_sandbox::apply_resolved_enforcement_in_child(Some(&plan)) + .expect("apply fence without a terminal"); + assert!(std::fs::read("/proc/meminfo").is_ok()); + assert_eq!( + std::fs::read("/proc/self/environ") + .expect_err("process credentials must stay fenced") + .kind(), + std::io::ErrorKind::PermissionDenied, + ); + let scratch = std::env::temp_dir().join(format!("devo-headless-{}", std::process::id())); + std::fs::write(&scratch, b"fenced scratch").expect("scratch write remains allowed"); + assert_eq!( + std::fs::read(&scratch).expect("scratch read"), + b"fenced scratch" + ); + std::fs::remove_file(scratch).expect("scratch cleanup"); +} diff --git a/docs/rlm-shell-escalation.md b/docs/rlm-shell-escalation.md index 395577fae..509715aa1 100644 --- a/docs/rlm-shell-escalation.md +++ b/docs/rlm-shell-escalation.md @@ -49,3 +49,23 @@ The initial narrow-grant command nested PowerShell with incorrect quoting. It failed at execution; using native PowerShell syntax passed. The host's existing PowerShell encoding prelude emits constrained-language warnings for sandboxed commands, but the corrected probe still completed all file operations with exit 0. + +## Headless execution + +`devo prompt` accepts the same `rlm.shell()` helper. Its host bridge projects +`bash` and `bash.start` requests into the existing hidden shell runtime handler, +retaining permission metadata, command prefixes, and timeouts. Its existing +static-allow permission policy applies explicitly requested full or narrow grants +without marking them as human approvals. The headless command does not offer +interactive approval prompts; use the interactive client when a human approval +decision is required. + +Headless model selection uses `provider/model[/variant]` references, for example +`devo --model openai-codex/gpt-5.6-luna prompt "your task"`. An invalid explicit +reference returns an error rather than silently running the configured default. + +On Linux, `/dev/tty` exists even when a process has no controlling terminal. +The sandbox omits that optional device grant when opening it returns `ENXIO`, +so detached headless kernels can start while retaining filesystem and network +enforcement. The Linux regression probe verifies startup, scratch access, and +continued denial of `/proc/self/environ` after applying the kernel fence.