From 81ab7ca977b4ab4368834f8c27996ab6b1473bf2 Mon Sep 17 00:00:00 2001 From: maksym Date: Tue, 14 Jul 2026 18:00:59 +0200 Subject: [PATCH] feat(api): enhance API configuration and add new scripts - Updated `apiUrl` in Nuxt config to fallback on `process.env.API_URL` for better local development support. - Modified `predev` script in both admin and app package.json to include a wait-for-swagger script for improved build reliability. - Introduced `docker-up.sh` script to handle Docker compose errors more gracefully during local development. - Added `wait-for-swagger.mjs` script to ensure the API is fully started before proceeding with the build process. - Enhanced API schemas and types to support new features, including bulk import from archives and file deletion capabilities. --- admin/nuxt.config.ts | 7 +- admin/package.json | 2 +- api/docker-compose.yml | 5 ++ api/package.json | 5 +- app/nuxt.config.ts | 7 +- app/package.json | 2 +- .../api/data/repositories/api/schemas.gen.ts | 35 ++++++++- .../api/data/repositories/api/sdk.gen.ts | 39 +++++++++- .../api/data/repositories/api/types.gen.ts | 58 ++++++++++++++- scripts/docker-up.sh | 72 +++++++++++++++++++ scripts/wait-for-swagger.mjs | 54 ++++++++++++++ 11 files changed, 274 insertions(+), 12 deletions(-) create mode 100755 scripts/docker-up.sh create mode 100755 scripts/wait-for-swagger.mjs diff --git a/admin/nuxt.config.ts b/admin/nuxt.config.ts index d9ea865..ea2b531 100644 --- a/admin/nuxt.config.ts +++ b/admin/nuxt.config.ts @@ -24,7 +24,12 @@ export default defineNuxtConfig({ runtimeConfig: { public: { ranchVersion: rootRanchVersion(), - apiUrl: '', + // Prod (k8s) sets NUXT_PUBLIC_API_URL, which Nuxt's runtimeConfig env + // override picks up automatically. Local `ranch dev` only sets plain + // API_URL (see admin/.env) — without this fallback, apiUrl stays '' + // and the axios client resolves requests against its own origin + // instead of the api. + apiUrl: process.env.API_URL || '', }, }, app: { diff --git a/admin/package.json b/admin/package.json index 740326c..b78bc01 100644 --- a/admin/package.json +++ b/admin/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "predev": "openapi-ts", + "predev": "node ../scripts/wait-for-swagger.mjs && openapi-ts", "dev": "nuxt dev --port 3001", "prebuild": "openapi-ts", "build": "nuxt build", diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 6ee2559..2e5aa46 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -58,6 +58,11 @@ services: # 17900 → 7900 auth-proxy /live + /websockify + /vnc/* (open this in a browser) browserless: image: ghcr.io/cleanslice/browser-pool:latest + # Only published for linux/amd64 (see k8s/browser-pool-image/README.md) — + # without this, `docker compose up` on Apple Silicon fails with + # "no matching manifest for linux/arm64/v8". Runs fine under Docker + # Desktop's Rosetta/QEMU emulation. + platform: linux/amd64 container_name: ranch-browserless ports: - '13000:3000' diff --git a/api/package.json b/api/package.json index b107b75..e734128 100644 --- a/api/package.json +++ b/api/package.json @@ -15,18 +15,19 @@ "start:prod": "node dist/main", "generate:swagger": "node dist/generate-swagger.js", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", + "pretest": "npm run generate", "test": "jest --passWithNoTests", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./test/jest-e2e.json", - "generate": "npx prisma-import --force", + "generate": "npx prisma-import --force && DATABASE_URL=${DATABASE_URL:-postgresql://dummy:dummy@localhost:5432/dummy} npx prisma generate", "premigrate": "npx prisma-import --force", "migrate": "dotenv -e .env.dev -- npx prisma migrate dev --name auto && dotenv -e .env.dev -- npx prisma generate", "migrate:reset": "dotenv -e .env.dev -- npx prisma migrate reset --force", "migrate:prod": "dotenv -e .env.prod -- npx prisma migrate deploy", "studio": "dotenv -e .env.dev -- npx prisma studio", - "docker": "docker compose up -d", + "docker": "bash ../scripts/docker-up.sh", "freeport": "lsof -ti:3333 | xargs kill -9 2>/dev/null || true", "predev": "npm run docker && npm run migrate && npm run freeport" }, diff --git a/app/nuxt.config.ts b/app/nuxt.config.ts index 207f104..e7680a1 100644 --- a/app/nuxt.config.ts +++ b/app/nuxt.config.ts @@ -23,7 +23,12 @@ export default defineNuxtConfig({ runtimeConfig: { public: { ranchVersion: rootRanchVersion(), - apiUrl: '', + // Prod (k8s) sets NUXT_PUBLIC_API_URL, which Nuxt's runtimeConfig env + // override picks up automatically. Local `ranch dev` only sets plain + // API_URL (see app/.env) — without this fallback, apiUrl stays '' + // and the axios client resolves requests against its own origin + // instead of the api. + apiUrl: process.env.API_URL || '', }, }, app: { diff --git a/app/package.json b/app/package.json index 56403f4..aa91d0e 100644 --- a/app/package.json +++ b/app/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "predev": "openapi-ts", + "predev": "node ../scripts/wait-for-swagger.mjs && openapi-ts", "dev": "nuxt dev --port 3000", "prebuild": "openapi-ts", "build": "nuxt build", diff --git a/app/slices/setup/api/data/repositories/api/schemas.gen.ts b/app/slices/setup/api/data/repositories/api/schemas.gen.ts index 1e56cb1..b860737 100644 --- a/app/slices/setup/api/data/repositories/api/schemas.gen.ts +++ b/app/slices/setup/api/data/repositories/api/schemas.gen.ts @@ -327,7 +327,7 @@ export const UserRoleTypesSchema = { type: "string", enum: ["Owner", "Admin", "User", "Agent"], description: - "Server filters Owner/Admin out regardless of what is passed; embed keys cannot grant platform-admin to a visitor.", + "Server filters Owner/Admin out unless the presenting API key carries the embed:mint-admin scope; plain embed keys cannot grant platform-admin to a visitor.", } as const; export const EmbedTokenDtoSchema = { @@ -346,7 +346,7 @@ export const EmbedTokenDtoSchema = { roles: { type: "array", description: - "Server filters Owner/Admin out regardless of what is passed; embed keys cannot grant platform-admin to a visitor.", + "Server filters Owner/Admin out unless the presenting API key carries the embed:mint-admin scope; plain embed keys cannot grant platform-admin to a visitor.", items: { $ref: "#/components/schemas/UserRoleTypes", }, @@ -362,7 +362,7 @@ export const EmbedTokenDtoSchema = { export const ApiKeyScopeTypesSchema = { type: "string", - enum: ["embed:mint", "admin"], + enum: ["embed:mint", "embed:mint-admin", "admin"], } as const; export const CreateApiKeyDtoSchema = { @@ -602,6 +602,23 @@ export const AddFromSitemapResultDtoSchema = { required: ["added", "discovered"], } as const; +export const AddFromArchiveResultDtoSchema = { + type: "object", + properties: { + detected: { + type: "number", + example: 288, + description: + "Number of ingestable files detected in the archive. Import runs in the background; refresh the sources list to watch them appear.", + }, + started: { + type: "boolean", + example: true, + }, + }, + required: ["detected", "started"], +} as const; + export const AgentPodStatusDtoSchema = { type: "object", properties: { @@ -973,6 +990,18 @@ export const SaveFileDtoSchema = { required: ["content"], } as const; +export const DeleteFilesDtoSchema = { + type: "object", + properties: { + deleted: { + type: "number", + example: 3, + description: "Number of S3 objects deleted by this request.", + }, + }, + required: ["deleted"], +} as const; + export const BridleTextPartDtoSchema = { type: "object", properties: { diff --git a/app/slices/setup/api/data/repositories/api/sdk.gen.ts b/app/slices/setup/api/data/repositories/api/sdk.gen.ts index 9aee10c..dedbb3f 100644 --- a/app/slices/setup/api/data/repositories/api/sdk.gen.ts +++ b/app/slices/setup/api/data/repositories/api/sdk.gen.ts @@ -59,6 +59,8 @@ import type { AddKnowledgeSourceData, AddKnowledgeSourcesFromSitemapData, AddKnowledgeSourcesFromSitemapResponse, + AddKnowledgeSourcesFromArchiveData, + AddKnowledgeSourcesFromArchiveResponse, DeleteKnowledgeSourceData, DeleteKnowledgeSourceResponse, AgentControllerFindAllData, @@ -84,6 +86,8 @@ import type { AgentControllerStartData, RestartByTemplateData, FileControllerListData, + FileControllerDeleteData, + FileControllerDeleteResponse, FileControllerReadData, FileControllerReadResponse, FileControllerSaveData, @@ -863,7 +867,7 @@ export class AuthService { } /** - * Mint a short-lived browser embed JWT for the bridle widget. Auth: API key with embed:mint scope. Owner/Admin roles are stripped from the result regardless of input. + * Mint a short-lived browser embed JWT for the bridle widget. Auth: API key with embed:mint scope. Owner/Admin roles are stripped from the result unless the key also carries embed:mint-admin — then they are kept and the TTL is capped at 7d. */ public static authControllerEmbedToken( options: Options, @@ -1170,6 +1174,23 @@ export class KnowledgeSourcesService { }); } + /** + * Bulk-import sources from a zip archive + * Accepts a .zip, extracts every ingestable file (pdf, docx, xlsx, txt, html, ...), and creates one file-type source per entry. Upload runs in the background and streams each entry to S3; the response returns immediately with the detected file count. Indexing into LightRAG happens through the normal reindex flow. + */ + public static addKnowledgeSourcesFromArchive< + ThrowOnError extends boolean = false, + >(options: Options) { + return (options.client ?? _heyApiClient).post< + AddKnowledgeSourcesFromArchiveResponse, + unknown, + ThrowOnError + >({ + url: "/knowledges/{knowledgeId}/sources/from-archive", + ...options, + }); + } + /** * Delete source */ @@ -1538,6 +1559,22 @@ export class FilesService { }); } + /** + * Delete a file, or a whole folder (e.g. a skill dir) when `recursive=true`. Template-managed skills are recreated on the next restart unless detached from the template first. + */ + public static fileControllerDelete( + options: Options, + ) { + return (options.client ?? _heyApiClient).delete< + FileControllerDeleteResponse, + unknown, + ThrowOnError + >({ + url: "/agents/{agentId}/files/content", + ...options, + }); + } + /** * Read a chunk of a file. Omit `offset`/`limit` to read the first 256 KB. Use the returned `nextOffset` to continue. */ diff --git a/app/slices/setup/api/data/repositories/api/types.gen.ts b/app/slices/setup/api/data/repositories/api/types.gen.ts index ce12e5c..12b0b7a 100644 --- a/app/slices/setup/api/data/repositories/api/types.gen.ts +++ b/app/slices/setup/api/data/repositories/api/types.gen.ts @@ -132,7 +132,7 @@ export type RegisterDto = { }; /** - * Server filters Owner/Admin out regardless of what is passed; embed keys cannot grant platform-admin to a visitor. + * Server filters Owner/Admin out unless the presenting API key carries the embed:mint-admin scope; plain embed keys cannot grant platform-admin to a visitor. */ export enum UserRoleTypes { OWNER = "Owner", @@ -148,7 +148,7 @@ export type EmbedTokenDto = { sub: string; email?: string; /** - * Server filters Owner/Admin out regardless of what is passed; embed keys cannot grant platform-admin to a visitor. + * Server filters Owner/Admin out unless the presenting API key carries the embed:mint-admin scope; plain embed keys cannot grant platform-admin to a visitor. */ roles?: Array; /** @@ -159,6 +159,7 @@ export type EmbedTokenDto = { export enum ApiKeyScopeTypes { "EMBED:MINT" = "embed:mint", + "EMBED:MINT_ADMIN" = "embed:mint-admin", ADMIN = "admin", } @@ -240,6 +241,14 @@ export type AddFromSitemapResultDto = { discovered: number; }; +export type AddFromArchiveResultDto = { + /** + * Number of ingestable files detected in the archive. Import runs in the background; refresh the sources list to watch them appear. + */ + detected: number; + started: boolean; +}; + export type AgentPodStatusDto = { agentId: string; podName: string; @@ -416,6 +425,13 @@ export type SaveFileDto = { content: string; }; +export type DeleteFilesDto = { + /** + * Number of S3 objects deleted by this request. + */ + deleted: number; +}; + export type BridleTextPartDto = { type: "text" | "image" | "file"; text: string; @@ -1751,6 +1767,22 @@ export type AddKnowledgeSourcesFromSitemapResponses = { export type AddKnowledgeSourcesFromSitemapResponse = AddKnowledgeSourcesFromSitemapResponses[keyof AddKnowledgeSourcesFromSitemapResponses]; +export type AddKnowledgeSourcesFromArchiveData = { + body?: never; + path: { + knowledgeId: string; + }; + query?: never; + url: "/knowledges/{knowledgeId}/sources/from-archive"; +}; + +export type AddKnowledgeSourcesFromArchiveResponses = { + 201: AddFromArchiveResultDto; +}; + +export type AddKnowledgeSourcesFromArchiveResponse = + AddKnowledgeSourcesFromArchiveResponses[keyof AddKnowledgeSourcesFromArchiveResponses]; + export type DeleteKnowledgeSourceData = { body?: never; path: { @@ -2017,6 +2049,28 @@ export type FileControllerListResponses = { 200: unknown; }; +export type FileControllerDeleteData = { + body?: never; + path: { + agentId: string; + }; + query: { + path: string; + /** + * When true, `path` is treated as a folder and every file under it is deleted. + */ + recursive?: boolean; + }; + url: "/agents/{agentId}/files/content"; +}; + +export type FileControllerDeleteResponses = { + 200: DeleteFilesDto; +}; + +export type FileControllerDeleteResponse = + FileControllerDeleteResponses[keyof FileControllerDeleteResponses]; + export type FileControllerReadData = { body?: never; path: { diff --git a/scripts/docker-up.sh b/scripts/docker-up.sh new file mode 100755 index 0000000..b43737b --- /dev/null +++ b/scripts/docker-up.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Wraps `docker compose up -d` for api's `predev`. Turbo interleaves +# admin:dev/app:dev/api:dev logs line-by-line, so a failure here is easy to +# miss or misdiagnose. Capture the output and match on the two failure modes +# we've actually hit locally, instead of guessing at a single cause. +set -o pipefail + +cd "$(dirname "$0")/../api" || exit 1 + +output="$(docker compose up -d 2>&1)" +code=$? +echo "$output" >&2 + +if [ "$code" -ne 0 ]; then + if echo "$output" | grep -qiE "unauthorized|denied|403 Forbidden"; then + cat >&2 <<'EOF' + +✖ `docker compose up -d` failed — registry auth error above. + + Most likely cause: ghcr.io/cleanslice/browser-pool is a private image and + Docker isn't logged into ghcr.io (or the account lacks package access). + + Fix: + 1. Create a GitHub PAT with the `read:packages` scope: + https://github.com/settings/tokens + 2. docker login ghcr.io -u + (paste the PAT as the password) + 3. If you still get 403 after logging in, the token is valid but your + account lacks read access to the package itself — ask whoever + administers ghcr.io/cleanslice/browser-pool to grant you access + (package Settings → Manage Actions access / Collaborators). Repo + access does not automatically grant package access on GHCR. + 4. Re-run `ranch dev`. +EOF + elif echo "$output" | grep -qi "no matching manifest"; then + cat >&2 <<'EOF' + +✖ `docker compose up -d` failed — no matching image manifest for your platform. + + ghcr.io/cleanslice/browser-pool is only published for linux/amd64. This + should already be handled by `platform: linux/amd64` on the browserless + service in api/docker-compose.yml — if you're still hitting this, check + that line wasn't reverted. +EOF + elif echo "$output" | grep -qi "declared as external"; then + cat >&2 <<'EOF' + +✖ `docker compose up -d` failed — the external `k3d-ranch` docker network doesn't exist. + + browserless/minio attach to it so agent runtime pods (inside k3d) can + reach them. It's created by k3d when the local cluster exists. + + Fix: + 1. brew install k3d (if not installed) + 2. k3d cluster create ranch --api-port 6550 --wait + 3. Re-run `ranch dev` (it also does this automatically when k3d is + installed — see cli/src/utils/k3d.ts). +EOF + else + cat >&2 <<'EOF' + +✖ `docker compose up -d` failed — see the docker-compose output above for the actual error. +EOF + fi + + cat >&2 <<'EOF' + + Until this succeeds, api can't finish starting, so it never writes + api/swagger-spec.json — admin/app will sit waiting on it until they time out. +EOF + exit 1 +fi diff --git a/scripts/wait-for-swagger.mjs b/scripts/wait-for-swagger.mjs new file mode 100755 index 0000000..50238ca --- /dev/null +++ b/scripts/wait-for-swagger.mjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// admin/app's `predev` runs `openapi-ts` against `../api/swagger-spec.json`. +// That file is gitignored and only written once the api Nest process +// actually finishes booting (docker pull + prisma migrate + Nest bootstrap +// in main.ts) — turbo runs admin#dev/app#dev/api#dev in parallel with no +// ordering, so without this wait `openapi-ts` fails instantly with ENOENT +// on every fresh `ranch dev`, which then aborts the whole turbo pipeline +// (killing api's still-running docker/migrate step via SIGHUP). +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const target = resolve(process.cwd(), '../api/swagger-spec.json'); +const timeoutMs = 5 * 60 * 1000; +const intervalMs = 1000; +const reminderMs = 30 * 1000; +const start = Date.now(); +let lastReminder = start; + +function poll() { + if (existsSync(target)) { + process.exit(0); + } + + const elapsed = Date.now() - start; + + if (elapsed > timeoutMs) { + console.error( + `\n✖ Timed out after ${timeoutMs / 1000}s waiting for ${target}.\n\n` + + ' api never finished starting, so it never wrote swagger-spec.json. Check the "api:dev" lines\n' + + ' above for the actual failure — the most common one locally is:\n\n' + + ' docker compose up -d ... unauthorized\n\n' + + ' which means Docker isn\'t logged into ghcr.io for the private browser-pool image. Fix with:\n\n' + + ' docker login ghcr.io -u (PAT with read:packages as the password)\n\n' + + ' then re-run `ranch dev`. If api is instead stuck on a migration or a slow pull, just wait —\n' + + ' rerunning `ranch dev` once api is healthy will pick up the file immediately.\n', + ); + process.exit(1); + } + + if (Date.now() - lastReminder > reminderMs) { + lastReminder = Date.now(); + console.log( + `⏳ Still waiting for ${target} (${Math.round(elapsed / 1000)}s) — see "api:dev" logs above if this doesn't resolve soon.`, + ); + } + + setTimeout(poll, intervalMs); +} + +if (!existsSync(target)) { + console.log(`⏳ Waiting for ${target} (api is still starting)...`); +} + +poll();