Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ SANDBOX_RUN_CPU_TIME=10000
SANDBOX_RUN_TIMEOUT=15000
SANDBOX_OUTPUT_MAX_SIZE=65536

# Remote stateful code bridge (Code API deployment)
# CODEAPI_SANDBOX_BACKEND=remote-bridge
# CODEAPI_EXECUTION_PROFILE=stateful
# CODEAPI_RUNTIME_SESSION_MODE=affinity
# CODEAPI_BRIDGE_WORKER_ID=my-vm
# CODEAPI_BRIDGE_TOKEN=replace-with-a-strong-random-secret

# Service Configuration
PYTHON_CONCURRENCY=5
OTHER_CONCURRENCY=15
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
data/
node_modules
packages/*/dist/
.env
.git
.npmrc
Expand Down
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ Code Interpreter (internally `codeapi`, the prefix used by its env vars, images,
- **Package Delivery** - Bakes Python, Node, and Bun into the default microVM
block-root image; a package-init PVC mode remains available for direct NsJail
development
- **Remote Code Bridge** - Lets an operator-owned VM connect outbound and serve
as a fenced, stateful sandbox through the `@librechat/code` worker

## Architecture

Expand Down Expand Up @@ -65,6 +67,18 @@ Two modes are supported:
- **NsJail mode** (`kvmEnabled: false`): Direct NsJail sandboxing with Linux namespaces and cgroups
- **MicroVM mode** (`kvmEnabled: true`): libkrun microVM with its own kernel, NsJail runs inside the guest

## Remote stateful environments

The `remote-bridge` backend keeps the Code API as the policy and queue boundary
while moving execution to a sandbox on an operator-selected VM. The worker only
makes outbound authenticated requests, so the VM does not need a public ingress
port. Assignments carry a deadline, a single-active-worker lock, a monotonically
increasing generation, and a one-time lease token to fence stale workers.

See [Remote Code Bridge](docs/remote-bridge/README.md) for deployment and threat
model details. The worker protocol and CLI live in the provider-neutral
[`@librechat/code`](packages/code/README.md) package.

## Security disclaimer

This service exists to run arbitrary, untrusted code — treat every
Expand Down
5 changes: 4 additions & 1 deletion api/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
RUN make -j$(nproc)

COPY api/src/spec-guard.c /tmp/spec-guard.c
COPY docker/rootfs-setup.c /tmp/rootfs-setup.c
RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \
&& chmod 0111 /usr/local/bin/spec-guard
&& gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \
&& chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup

# ============================================================================
# Stage 1b: Build language runtime packages (only consumed by sandbox-runner-baked)
Expand Down Expand Up @@ -212,6 +214,7 @@ RUN dnf install -y --setopt=install_weak_deps=False \
&& dnf clean all

COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher
COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup

COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh
Expand Down
9 changes: 7 additions & 2 deletions api/src/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -171,28 +171,33 @@ fi
chmod 777 "$SMOKE_DIR"
fi
SMOKE_LOG=$(mktemp)
SMOKE_STDERR=$(mktemp)

NSJAIL_CGROUP_ARGS=()
if [ "$SANDBOX_USE_CGROUPV2" = "true" ]; then
NSJAIL_CGROUP_ARGS=(--use_cgroupv2)
fi

if timeout 10 /usr/sbin/nsjail --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \
if timeout 10 "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \
"${NSJAIL_CGROUP_ARGS[@]}" --log "$SMOKE_LOG" \
--user "65534:${SMOKE_OUTSIDE_UID}:1" --group "65534:${SMOKE_OUTSIDE_GID}:1" \
-s /usr/bin:/bin -s /usr/lib:/lib -s /usr/lib64:/lib64 \
-B "$SMOKE_DIR:/mnt/data" \
-- /bin/sh -c 'printf "%s\n" sandbox_ok > /mnt/data/smoke.txt && test "$(cat /mnt/data/smoke.txt)" = sandbox_ok' > /dev/null 2>&1; then
-- /bin/sh -c 'printf "%s\n" sandbox_ok > /mnt/data/smoke.txt && test "$(cat /mnt/data/smoke.txt)" = sandbox_ok' > /dev/null 2>"$SMOKE_STDERR"; then
echo "NsJail smoke test passed"
else
echo "FATAL: NsJail smoke test failed — sandbox cannot start"
echo "NsJail log output:"
cat "$SMOKE_LOG" 2>/dev/null || true
echo "NsJail stderr:"
cat "$SMOKE_STDERR" 2>/dev/null || true
rm -f "$SMOKE_LOG"
rm -f "$SMOKE_STDERR"
rm -rf "$SMOKE_DIR"
exit 1
fi
rm -f "$SMOKE_LOG"
rm -f "$SMOKE_STDERR"
rm -rf "$SMOKE_DIR"

echo "Starting sandbox API server..."
Expand Down
6 changes: 5 additions & 1 deletion docker/Dockerfile.worker-sandbox
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
RUN make -j$(nproc)

COPY api/src/spec-guard.c /tmp/spec-guard.c
COPY docker/rootfs-setup.c /tmp/rootfs-setup.c
RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \
&& chmod 0111 /usr/local/bin/spec-guard
&& gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \
&& chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup

# ============================================================================
# Stage 1b: Build language runtime packages for the baked KVM root disk
Expand Down Expand Up @@ -83,6 +85,7 @@ WORKDIR /app
COPY service/package.json service/bun.lock ./
RUN bun install --frozen-lockfile
COPY service/src ./src
COPY packages/code/src /packages/code/src
COPY shared /shared
COPY service/tsconfig.json ./
RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*'
Expand Down Expand Up @@ -231,6 +234,7 @@ ENV PATH="/root/.bun/bin:${PATH}"

# --- Launcher (runs on host, boots microVM) ---
COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher
COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup

# --- Launcher entrypoint (DNS resolution + socat relay before VM boot) ---
COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
Expand Down
92 changes: 92 additions & 0 deletions docker/rootfs-setup.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <unistd.h>

static int bind_mount(const char *source, const char *target, int read_only) {
if (mount(source, target, NULL, MS_BIND | MS_REC, NULL) != 0) {
fprintf(stderr, "bind %s -> %s failed: %s\n", source, target, strerror(errno));
return -1;
}

if (read_only &&
mount(NULL, target, NULL, MS_BIND | MS_REMOUNT | MS_RDONLY, NULL) != 0) {
fprintf(stderr, "read-only remount of %s failed: %s\n", target, strerror(errno));
return -1;
}

return 0;
}

static int bind_rootfs_path(const char *rootfs, const char *path) {
char source[PATH_MAX];
int written = snprintf(source, sizeof(source), "%s%s", rootfs, path);
if (written < 0 || (size_t)written >= sizeof(source)) {
fprintf(stderr, "rootfs path is too long: %s%s\n", rootfs, path);
return -1;
}

return bind_mount(source, path, 1);
}

int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "usage: sandbox-rootfs-setup ROOTFS [COMMAND ...]\n");
return 2;
}

const char *rootfs = argv[1];
if (rootfs[0] != '/') {
fprintf(stderr, "rootfs must be an absolute path\n");
return 2;
}

if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0) {
fprintf(stderr, "making the mount namespace private failed: %s\n", strerror(errno));
return 1;
}

if ((mkdir("/sandbox_api", 0755) != 0 && errno != EEXIST) ||
(mkdir("/pkgs", 0755) != 0 && errno != EEXIST)) {
fprintf(stderr, "creating rootfs mount targets failed: %s\n", strerror(errno));
return 1;
}

/*
* Keep this process statically linked: the final /usr mount replaces
* the Fedora launcher's dynamic userspace with the Debian sandbox rootfs.
* A shell cannot safely perform this sequence because its next command may
* try to load a host binary against guest libraries (or vice versa).
*/
const char *paths[] = {"/sandbox_api", "/pkgs"};
for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
if (bind_rootfs_path(rootfs, paths[i]) != 0) {
return 1;
}
}

if (access("/host-packages", F_OK) == 0 &&
bind_mount("/host-packages", "/pkgs", 0) != 0) {
fprintf(stderr, "warning: sandbox will run without host packages\n");
}

/* Bind all guest userspace last, then immediately enter it. */
if (bind_rootfs_path(rootfs, "/usr") != 0) {
return 1;
}

setenv("PATH", "/root/.bun/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", 1);
setenv("LD_LIBRARY_PATH", "/usr/lib/aarch64-linux-gnu:/usr/lib/x86_64-linux-gnu", 1);

setenv("NSJAIL_PATH", "/usr/sbin/nsjail", 1);

char *default_argv[] = {"/sandbox_api/entrypoint.sh", NULL};
char **command_argv = argc > 2 ? &argv[2] : default_argv;
execv(command_argv[0], command_argv);
fprintf(stderr, "starting sandbox entrypoint failed: %s\n", strerror(errno));
return 1;
}
33 changes: 1 addition & 32 deletions docker/start-direct-sandbox.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,35 +44,4 @@ else
fi

export SANDBOX_ROOTFS="$ROOTFS"

exec unshare --mount bash -c '
ROOTFS="${SANDBOX_ROOTFS:-/sandbox-rootfs}"

mount -o bind,ro "$ROOTFS/usr/sbin" /usr/sbin || { echo "FATAL: cannot bind /usr/sbin"; exit 1; }
mount -o bind,ro "$ROOTFS/usr/lib" /usr/lib || { echo "FATAL: cannot bind /usr/lib"; exit 1; }

if [ -d "$ROOTFS/usr/lib64" ] && ! [ -L "$ROOTFS/usr/lib64" ]; then
mount -o bind,ro "$ROOTFS/usr/lib64" /usr/lib64 2>/dev/null || \
echo "[sandbox] WARNING: could not bind /usr/lib64 - sandboxed binaries may fail to exec"
fi

mount -o bind,ro "$ROOTFS/usr/local" /usr/local || { echo "FATAL: cannot bind /usr/local"; exit 1; }
mount -o bind,ro "$ROOTFS/sandbox_api" /sandbox_api || { echo "FATAL: cannot bind /sandbox_api"; exit 1; }
mount -o bind,ro "$ROOTFS/pkgs" /pkgs || { echo "FATAL: cannot bind /pkgs"; exit 1; }

if [ -d /host-packages ]; then
mount --bind /host-packages /pkgs 2>/dev/null || \
echo "WARNING: could not bind /host-packages - sandbox will run without packages"
fi

mount -o bind,ro "$ROOTFS/usr/bin" /usr/bin || { echo "FATAL: cannot bind /usr/bin"; exit 1; }

multiarch_libdir=$(find /usr/lib -maxdepth 1 -type d -name "*-linux-gnu" -print -quit)
if [ -n "$multiarch_libdir" ]; then
export LD_LIBRARY_PATH="$multiarch_libdir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
fi

export PATH="/root/.bun/bin:$PATH"

exec /sandbox_api/entrypoint.sh
'
exec unshare --mount /sandbox-rootfs-setup "$ROOTFS"
Comment thread
danny-avila marked this conversation as resolved.
87 changes: 87 additions & 0 deletions docs/remote-bridge/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Remote Code Bridge

Remote Code Bridge makes an operator-owned VM a stateful Code API execution
environment without exposing that VM to inbound internet traffic.

```text
LibreChat -> Code API -> Redis assignment
^ |
| outbound v
@librechat/code -> local sandbox
```

Code API remains the public authentication, policy, manifest, timeout, and
result-normalization boundary. The bridge worker has a separate operator
credential and never accepts end-user bearer tokens directly.

## Code API configuration

Run this as an isolated stateful Code API deployment:

```dotenv
CODEAPI_SANDBOX_BACKEND=remote-bridge
CODEAPI_EXECUTION_PROFILE=stateful
CODEAPI_RUNTIME_SESSION_MODE=affinity
CODEAPI_BRIDGE_WORKER_ID=my-vm
CODEAPI_BRIDGE_TOKEN=<strong-random-secret>
```

Use `strict` instead of `affinity` if every request must include a runtime
session hint. In hardened mode, startup requires the bridge token to be at least
32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a
remote execution cannot retain an open Code API process across tool callbacks.

Start the CLI beside a sandbox using the same worker ID and secret; see
[`@librechat/code`](../../packages/code/README.md).
Stateful deployments must also set `LIBRECHAT_CODE_STATEFUL_WORKSPACE=true`
and route the CLI's `{runtimeSessionId}` endpoint template to an isolated,
persistent local runner per session. A single sandbox endpoint is stateless and
is rejected for runtime-session assignments.

## LibreChat configuration

Expose the Code API deployment as an environment under the Agents endpoint:

```yaml
endpoints:
agents:
statefulCodeSessions:
environments:
- id: my-vm
name: My VM
type: attached
baseURL: https://code.example.com/v1
default: true
```

Agents may select this environment with `code_environment_id: my-vm`.
LibreChat derives a stable per-conversation runtime session ID, so commands in
later turns reuse the same workspace. Attached environments deliberately skip
background prewarming: the single worker lease is reserved for explicit user
execution.

## Lifecycle and fencing

- Registration is ephemeral in Redis and must be refreshed by the worker.
- Code API permits one active assignment per configured worker.
- Each assignment has an absolute deadline, generation, and random lease token.
- Settlements with the wrong worker, generation, token, or expired deadline are
rejected.
- Request cancellation is polled by the worker and aborts the local sandbox
request.
- The sandbox receives the stable runtime session ID separately from the lease;
workspace state belongs to that session, not to a transient assignment.

## Security boundaries

The bridge removes inbound VM exposure; it does not replace sandbox isolation.
For internet-facing LibreChat deployments, use the hardened microVM/NsJail
stack, default-deny sandbox egress, signed execution manifests, least-privilege
host credentials, resource limits, and host/network monitoring. Bind the local
sandbox endpoint to loopback or a private container network. Rotate a leaked
bridge token immediately; the initial protocol intentionally uses a static
operator secret and supports one configured worker per Code API deployment.

The next control-plane layer can add short-lived pairing credentials and a
multi-worker directory without changing the execution protocol or moving code
tools into the Agents SDK.
7 changes: 5 additions & 2 deletions launcher/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
RUN make -j$(nproc)

COPY api/src/spec-guard.c /tmp/spec-guard.c
COPY docker/rootfs-setup.c /tmp/rootfs-setup.c
RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \
&& chmod 0111 /usr/local/bin/spec-guard
&& gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \
&& chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup

FROM oven/bun:1.3.14-debian AS sandbox-build

Expand Down Expand Up @@ -126,6 +128,7 @@ RUN dnf install -y --setopt=install_weak_deps=False \
&& dnf clean all

COPY --from=launcher-builder /launcher/target/release/sandbox-launcher /usr/local/bin/launcher
COPY --from=nsjail-builder /usr/local/bin/sandbox-rootfs-setup /sandbox-rootfs-setup

COPY --from=sandbox-build / /sandbox-rootfs/

Expand All @@ -136,6 +139,6 @@ RUN mkdir -p /host-packages
COPY launcher/entrypoint.sh /usr/local/bin/launcher-entrypoint.sh
COPY docker/start-direct-sandbox.sh /usr/local/bin/start-direct-sandbox.sh
COPY docker/sandbox-entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /sandbox-rootfs-setup /usr/local/bin/launcher-entrypoint.sh /usr/local/bin/start-direct-sandbox.sh /usr/local/bin/entrypoint.sh

ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
15 changes: 15 additions & 0 deletions packages/code/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FROM node:24-alpine AS build
WORKDIR /app
COPY package.json package-lock.json tsconfig.json ./
RUN npm ci
COPY src ./src
RUN npm run build

FROM node:24-alpine
ENV NODE_ENV=production
RUN addgroup -S librechat-code && adduser -S librechat-code -G librechat-code
WORKDIR /app
COPY --from=build /app/package.json ./package.json
COPY --from=build /app/dist ./dist
USER librechat-code
ENTRYPOINT ["node", "dist/cli.js"]
Loading