diff --git a/backend/__tests__/service/verify.wave1.test.js b/backend/__tests__/service/verify.wave1.test.js new file mode 100644 index 0000000000..819ed7271b --- /dev/null +++ b/backend/__tests__/service/verify.wave1.test.js @@ -0,0 +1,216 @@ +/** Wave 1: real HTTP mount table, real auth, Mongo and PostgreSQL. + * Only the external OAuth provider is stubbed. No model/middleware/service mocks. + * These scenarios deliberately run only in the Tier 1 service job. + */ +const request = require('supertest'); +const jwt = require('jsonwebtoken'); +const axios = require('axios'); +const mongoose = require('mongoose'); +const { randomUUID } = require('crypto'); +const { setupMongoDb, closeMongoDb, setupPgDb, closePgDb } = require('../utils/testUtils'); +const User = require('../../models/User'); +const Pod = require('../../models/Pod'); +const RoomGrant = require('../../models/RoomGrant'); +const ApprovalAction = require('../../models/ApprovalAction'); +const OAuthLoginState = require('../../models/OAuthLoginState'); +const AgentEvent = require('../../models/AgentEvent'); +const File = require('../../models/File'); +const { AgentInstallation } = require('../../models/AgentRegistry'); +const AgentIdentityService = require('../../services/agentIdentityService'); +const { hash } = require('../../utils/secret'); + +const tier1 = process.env.INTEGRATION_TEST === 'true' ? describe : describe.skip; +tier1('commonly verify wave 1 — production HTTP routes', () => { + let app; + let pg; + let owner; + let other; + let agent; + let installation; + let pod; + let ownerToken; + let otherToken; + const agentToken = 'cm_agent_verify-wave1-seat'; + const human = (token) => ({ Authorization: `Bearer ${token}` }); + const seat = () => human(agentToken); + + beforeAll(async () => { + // Importing the factory must not connect Mongo at module scope. + expect(mongoose.connection.readyState).toBe(0); + const { createApp } = require('../../server'); + expect(() => createApp({ startServices: true })).toThrow('Service startup is singleton-only'); + expect(mongoose.connection.readyState).toBe(0); + await setupMongoDb(); + pg = await setupPgDb(); + // A test app registers no boot job on the shared connection (the + // registry bootstrap, the grant-owner backfill): those are the singleton's. + const openListeners = mongoose.connection.listenerCount('open'); + app = createApp(); + expect(mongoose.connection.listenerCount('open')).toBe(openListeners); + owner = await User.create({ username: 'verify-owner', email: 'verify-owner@test.com', password: 'password123' }); + other = await User.create({ username: 'verify-other', email: 'verify-other@test.com', password: 'password123' }); + ownerToken = jwt.sign({ id: String(owner._id) }, process.env.JWT_SECRET); + otherToken = jwt.sign({ id: String(other._id) }, process.env.JWT_SECRET); + agent = await User.create({ + username: AgentIdentityService.buildAgentUsername('openclaw', 'verify-wave1'), + email: 'verify-seat@test.com', password: 'password123', isBot: true, botType: 'agent', + botMetadata: { agentName: 'openclaw', instanceId: 'verify-wave1' }, + agentRuntimeTokens: [{ tokenHash: hash(agentToken), lastUsedAt: new Date() }], + }); + pod = await Pod.create({ name: 'Verify wave 1', type: 'team', publicRead: false, joinPolicy: 'invite-only', createdBy: owner._id, members: [owner._id, other._id, agent._id] }); + installation = await AgentInstallation.create({ + agentName: 'openclaw', instanceId: 'verify-wave1', podId: pod._id, + installedBy: owner._id, version: '1.0.0', status: 'active', + config: { runtime: { runtimeType: 'native' }, tools: ['computer_request_takeover'] }, + }); + for (const user of [owner, other, agent]) { + await pg.query('INSERT INTO users (_id, username, is_bot) VALUES ($1,$2,$3)', [String(user._id), user.username, user.isBot]); + } + await pg.query('INSERT INTO pods (id,name,type,created_by) VALUES ($1,$2,$3,$4)', [String(pod._id), pod.name, pod.type, String(owner._id)]); + await pg.query('INSERT INTO messages (pod_id,user_id,content) VALUES ($1,$2,$3)', [String(pod._id), String(owner._id), 'wave1 durable chat witness']); + }); + + afterEach(() => jest.restoreAllMocks()); + afterAll(async () => { + if (app) await new Promise((resolve) => app.locals.socketServer.close(resolve)); + const { pool } = require('../../config/db-pg'); + if (pool) await pool.end(); + await closePgDb(); + await closeMongoDb(); + }); + + test('attenuation refuses wider tools and persists a strict subset', async () => { + const grantId = randomUUID(); + await RoomGrant.create({ grantId, connectionId: 'verify-connection', installationId: String(installation._id), + target: { kind: 'pod', id: String(pod._id) }, audience: [String(agent._id)], + tools: ['read', 'search'], writeMode: 'read', brokerId: 'verify-broker', expiresAt: new Date(Date.now() + 3600000) }); + const url = `/api/grants/${grantId}/attenuate`; + const widened = await request(app).post(url).set(seat()).send({ tools: ['write'] }); + expect({ status: widened.status, error: widened.body.error }).toEqual({ status: 400, error: 'grant_not_attenuated' }); + expect(await RoomGrant.countDocuments({ parentGrantId: grantId })).toBe(0); + const narrowed = await request(app).post(url).set(seat()).send({ tools: ['read'] }).expect(201); + const stored = await RoomGrant.findOne({ grantId: narrowed.body.grantId }).lean(); + expect(stored.tools).toEqual(['read']); + expect(stored.parentGrantId).toBe(grantId); + expect(stored.audience).toEqual([String(agent._id)]); + }); + + test('approval resolution belongs to its owner, even when another human is a pod member', async () => { + const action = await ApprovalAction.create({ podId: pod._id, ownerUserId: owner._id, + agentName: 'openclaw', instanceId: 'verify-wave1', actionType: 'create_pod', + params: { name: 'Must not execute' }, summary: 'Verify owner decision', expiresAt: new Date(Date.now() + 60000) }); + const url = `/api/approvals/${action._id}/resolve`; + await request(app).post(url).set(human(otherToken)).send({ decision: 'approved' }).expect(403); + expect((await ApprovalAction.findById(action._id)).status).toBe('flagged'); + await request(app).post(url).set(human(ownerToken)).send({ decision: 'declined' }).expect(200); + const stored = await ApprovalAction.findById(action._id); + expect(stored.decision).toBe('declined'); + expect(String(stored.resolvedBy)).toBe(String(owner._id)); + expect(await Pod.countDocuments({ name: 'Must not execute' })).toBe(0); + }); + + test('OAuth callback claims state once; exchange code creates one usable human session', async () => { + process.env.GITHUB_OAUTH_CLIENT_ID = 'verify-client'; + process.env.GITHUB_OAUTH_CLIENT_SECRET = 'verify-secret'; + owner.authProviders = [{ provider: 'github', providerId: '514', email: owner.email }]; + await owner.save(); + const post = jest.spyOn(axios, 'post').mockResolvedValue({ data: { access_token: 'upstream-test-only' } }); + jest.spyOn(axios, 'get').mockImplementation(async (url) => { + if (url === 'https://api.github.com/user') return { data: { id: 514, login: owner.username } }; + if (url === 'https://api.github.com/user/emails') return { data: [{ email: owner.email, verified: true, primary: true }] }; + throw new Error('Unexpected upstream URL'); + }); + // /start mints the attempt and the browser cookie that binds its callback and exchange. + const start = await request(app).get('/api/auth/oauth/github/start').expect(302); + const browserCookie = start.headers['set-cookie'].map((cookie) => cookie.split(';')[0]).join('; '); + const startState = new URL(start.headers.location).searchParams.get('state'); + const state = await OAuthLoginState.findOne({ provider: 'github', state: startState }); + expect(state).toBeTruthy(); + const callback = `/api/auth/oauth/github/callback?state=${state.state}&code=provider-code`; + const inBrowser = (req) => req.set('Cookie', browserCookie); + const first = await inBrowser(request(app).get(callback)).expect(302); + const code = new URL(first.headers.location).searchParams.get('code'); + expect(code).toBeTruthy(); + expect(first.headers.location).not.toContain('token='); + const replay = await inBrowser(request(app).get(callback)).expect(302); + expect(replay.headers.location).toContain('state_invalid'); + expect(post).toHaveBeenCalledTimes(1); + const exchanged = await inBrowser(request(app).post('/api/auth/oauth/exchange')).send({ code }).expect(200); + expect(jwt.verify(exchanged.body.token, process.env.JWT_SECRET).id).toBe(String(owner._id)); + const session = await request(app).get('/api/auth/user').set(human(exchanged.body.token)).expect(200); + expect(String(session.body._id)).toBe(String(owner._id)); + const landing = await request(app).get('/api/pods').set(human(exchanged.body.token)).expect(200); + expect(landing.body.map((entry) => String(entry._id))).toContain(String(pod._id)); + await inBrowser(request(app).post('/api/auth/oauth/exchange')).send({ code }).expect(400); + expect((await OAuthLoginState.findById(state._id)).exchangedAt).toBeTruthy(); + delete process.env.GITHUB_OAUTH_CLIENT_ID; + delete process.env.GITHUB_OAUTH_CLIENT_SECRET; + }); + + test('agent upload is bound to the URL pod and stores actual bytes', async () => { + const uploaded = await request(app).post(`/api/agents/runtime/pods/${pod._id}/uploads`).set(seat()) + .field('podId', String(other._id)).attach('file', Buffer.from('wave1 upload bytes'), 'verify.txt').expect(200); + const stored = await File.findOne({ fileName: uploaded.body.fileName }).lean(); + expect(String(stored.podId)).toBe(String(pod._id)); + expect(String(stored.uploadedBy)).toBe(String(agent._id)); + const { getObjectStore } = require('../../services/objectStore'); + const bytes = await getObjectStore().get(uploaded.body.fileName); + expect(bytes).toBeTruthy(); + const chunks = []; + for await (const chunk of bytes.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe('wave1 upload bytes'); + const refused = await Pod.create({ name: 'Other room', type: 'team', createdBy: other._id, members: [other._id] }); + await request(app).post(`/api/agents/runtime/pods/${refused._id}/uploads`).set(seat()) + .attach('file', Buffer.from('must not persist'), 'refused.txt').expect(403); + expect(await File.countDocuments({ originalName: 'refused.txt' })).toBe(0); + }); + + test('computer press gates owner and computer capability before attempting a browser action', async () => { + const keys = ['AGENT_COMPUTER_ENABLED', 'AGENT_COMPUTER_STEEL_API_URL_TEMPLATE', 'AGENT_COMPUTER_STEEL_CDP_URL_TEMPLATE', 'AGENT_COMPUTER_WAKER_URL', 'AGENT_COMPUTER_WAKER_TOKEN']; + const prior = Object.fromEntries(keys.map((key) => [key, process.env[key]])); + try { + Object.assign(process.env, { AGENT_COMPUTER_ENABLED: 'true', AGENT_COMPUTER_STEEL_API_URL_TEMPLATE: 'http://127.0.0.1:1', AGENT_COMPUTER_STEEL_CDP_URL_TEMPLATE: 'http://127.0.0.1:1', AGENT_COMPUTER_WAKER_URL: 'http://127.0.0.1:1', AGENT_COMPUTER_WAKER_TOKEN: 'verify-only' }); + const url = `/api/agent-computer/${installation._id}/press`; + const otherPress = await request(app).post(url).set(human(otherToken)).send({ key: 'Enter', frameRevision: 'stale' }).expect(403); + expect(otherPress.body.code).toBe('not_owner'); + await AgentInstallation.updateOne({ _id: installation._id }, { $set: { 'config.tools': [] } }); + const ownerPress = await request(app).post(url).set(human(ownerToken)).send({ key: 'Enter', frameRevision: 'stale' }).expect(403); + expect(ownerPress.body.code).toBe('computer_not_granted'); + } finally { for (const key of keys) { if (prior[key] === undefined) delete process.env[key]; else process.env[key] = prior[key]; } } + }); + + test('CAP context and message reads return durable PG content and exclude another pod', async () => { + const foreign = await Pod.create({ name: 'Foreign content room', type: 'team', publicRead: false, joinPolicy: 'invite-only', createdBy: other._id, members: [other._id] }); + await pg.query('INSERT INTO pods (id,name,type,created_by) VALUES ($1,$2,$3,$4)', [String(foreign._id), foreign.name, foreign.type, String(other._id)]); + await pg.query('INSERT INTO messages (pod_id,user_id,content) VALUES ($1,$2,$3)', [String(foreign._id), String(other._id), 'foreign content must stay private']); + for (const suffix of ['context', 'messages']) { + const result = await request(app).get(`/api/agents/runtime/pods/${pod._id}/${suffix}`).set(seat()).expect(200); + expect(JSON.stringify(result.body)).toContain('wave1 durable chat witness'); + const refused = await request(app).get(`/api/agents/runtime/pods/${foreign._id}/${suffix}`).set(seat()).expect(403); + expect(JSON.stringify(refused.body)).not.toContain('foreign content must stay private'); + } + }); + + test('CAP renew changes only the matching delivery nonce owned by this seat', async () => { + const old = new Date(Date.now() - 60000); + const event = await AgentEvent.create({ agentName: 'openclaw', instanceId: 'verify-wave1', podId: pod._id, + type: 'verify.wave1', status: 'delivered', deliveryNonce: 'current-nonce', deliveredAt: old }); + const foreign = await AgentEvent.create({ agentName: 'openclaw', instanceId: 'foreign-seat', podId: pod._id, + type: 'verify.wave1', status: 'delivered', deliveryNonce: 'foreign-nonce', deliveredAt: old }); + const url = '/api/agents/runtime/events/renew'; + const wrong = await request(app).post(url).set(seat()).send({ events: [{ eventId: String(event._id), deliveryId: 'old-nonce' }] }).expect(200); + expect(wrong.body.renewed).toEqual([]); + expect((await AgentEvent.findById(event._id)).deliveredAt.getTime()).toBe(old.getTime()); + const valid = await request(app).post(url).set(seat()).send({ events: [ + { eventId: String(event._id), deliveryId: 'current-nonce' }, + { eventId: String(foreign._id), deliveryId: 'foreign-nonce' }, + ] }).expect(200); + expect(valid.body.renewed).toEqual([String(event._id)]); + const stored = await AgentEvent.findById(event._id); + expect(stored.deliveredAt.getTime()).toBeGreaterThan(old.getTime()); + expect(stored.status).toBe('delivered'); + const untouched = await AgentEvent.findById(foreign._id); + expect(untouched.deliveredAt.getTime()).toBe(old.getTime()); + expect(untouched.status).toBe('delivered'); + }); +}); diff --git a/backend/server.ts b/backend/server.ts index 5d6d7b9461..b8b3245ff0 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -77,8 +77,9 @@ const AgentMentionService = require('./services/agentMentionService'); const { createPgBoot } = require('./services/pgBootService'); const { setPgMountProbe, routerIsMounted } = require('./services/pgBootService'); -// Global flag to track PostgreSQL availability -let pgAvailable = false; + +// Load file-based configuration before deciding which PG routes to require. +dotenv.config(); if (process.env.PG_HOST) { pgMessageRoutes = require('./routes/pg-messages'); @@ -86,9 +87,11 @@ if (process.env.PG_HOST) { _PGPod = require('./models/pg/Pod'); } -// Load environment variables -dotenv.config(); - +/** Construct the production mount table without connecting databases or starting workers. + * Callers own database setup and close app.locals.socketServer after their HTTP run. + */ +const buildApp = ({ startServices = false } = {}) => { + let pgAvailable = false; // Initialize express app const app = express(); // Trust the reverse proxy (Cloudflare → ingress → backend). Without this, @@ -143,7 +146,7 @@ const { bindSocketIO: bindAgentTypingSocketIO } = require('./services/agentTypin const { bindSocketIO: bindTaskEventSocketIO } = require('./services/taskEventService'); // Socket.io Redis adapter initialization is async in K8s mode -(async () => { +if (startServices) (async () => { try { await socketConfig.init(io); agentWebSocketService.init(io); @@ -155,7 +158,6 @@ const { bindSocketIO: bindTaskEventSocketIO } = require('./services/taskEventSer } })(); -const PORT = process.env.PORT || 5000; // Middleware. A disallowed origin is refused with 403 (middleware/corsPolicy). app.use(cors(corsOptions(allowedOrigins))); @@ -309,7 +311,7 @@ if (require('./config/devRoutes').devRoutesEnabled()) { } // Connect to MongoDB (for posts and user data) -connectDB(); +if (startServices) connectDB(); // Bootstrap agent registry after MongoDB connects const mongoose = require('mongoose'); @@ -322,7 +324,8 @@ const { backfillGrantOwners } = require('./services/grantOwnerBinding'); // refuse unbound grants. This runs at boot without holding requests, so a // revoke that lands while it is still scanning can be refused until it // finishes; the scan is idempotent and a restart re-runs it for the rest. -mongoose.connection.once('open', () => { +// A boot job, so only the service singleton registers it. +if (startServices) mongoose.connection.once('open', () => { if (process.env.NODE_ENV === 'test') return; backfillGrantOwners() .then((result: { scanned: number; bound: number; unbound: number }) => console.log( @@ -331,8 +334,8 @@ mongoose.connection.once('open', () => { .catch((error: any) => console.error('[grant-owner-backfill] Error:', error.message)); }); -mongoose.connection.once('open', () => { - if (process.env.NODE_ENV !== 'test') { +if (startServices) mongoose.connection.once('open', () => { + if (startServices && process.env.NODE_ENV !== 'test') { (async () => { try { await ensureLocalDevLogin(); @@ -379,10 +382,12 @@ mongoose.connection.once('open', () => { const schedulerService = require('./services/schedulerService'); const discordGatewayService = require('./services/discordGatewayService'); const { startBackgroundServices, shouldRunBackgroundServices } = require('./services/backgroundWorkerStart'); -startBackgroundServices(process.env, { - scheduler: schedulerService, - discordGateway: discordGatewayService, -}); +if (startServices) { + startBackgroundServices(process.env, { + scheduler: schedulerService, + discordGateway: discordGatewayService, + }); +} // PostgreSQL status is mounted unconditionally, and NOT as a placeholder: // checkStatus reads the pool and the schema itself, so it answers @@ -394,6 +399,10 @@ startBackgroundServices(process.env, { // by the frontend after a GET said available:true (SocketContext.tsx:45-49), // i.e. only when the pool is up. app.use('/api/pg/status', pgStatusRoutes); +if (!startServices && process.env.PG_HOST) { + app.use('/api/pg/messages', pgMessageRoutes); + pgAvailable = true; +} // Connect to PostgreSQL if configured (for chat functionality). // @@ -406,7 +415,7 @@ app.use('/api/pg/status', pgStatusRoutes); // restart. PG itself was healthy (331ms from that same pod). The retry with // backoff, the late mount and the state the health route reads all live in // services/pgBootService.ts (TASK-168). -if (process.env.PG_HOST) { +if (startServices && process.env.PG_HOST) { // Readiness asks the route table, not this block's own bookkeeping (TASK-168): // it checks that this very router is on the app, so a pod that mounts PG late // starts passing its readiness probe without being restarted. @@ -432,7 +441,7 @@ if (process.env.PG_HOST) { // Kick off the daily 30-day message retention cron. Kept out // of schedulerService.ts on purpose so other tracks can edit // that file without stomping on this cron. - if (shouldRunBackgroundServices(process.env)) { + if (startServices && shouldRunBackgroundServices(process.env)) { try { const { initPgRetention } = require('./services/pgRetentionService'); initPgRetention(); @@ -463,7 +472,7 @@ if (process.env.PG_HOST) { // an unhandled rejection at boot. console.error('Error starting the PostgreSQL boot retry:', err?.message || err); }); -} else { +} else if (!process.env.PG_HOST) { pgAvailable = false; console.log( 'PostgreSQL connection not configured. Chat functionality will use MongoDB.', @@ -784,14 +793,36 @@ io.on('connection', (socket: any) => { socket.emit('welcome', { message: 'Connected to chat server successfully' }); }); -// Start the server only when executed directly -if (require.main === module) { - require('./services/hostedShutdownService').installHostedShutdown(server); - server.listen(PORT, () => console.log(`Server running on port ${PORT}`)); -} + app.locals.httpServer = server; + app.locals.socketServer = io; + app.locals.authorizeSocketPodAccess = authorizeSocketPodAccess; + return app; +}; + +// Preserve the legacy app/server exports lazily. Merely importing the factory +// must not connect Mongo, start PostgreSQL retries or launch background workers. +let defaultApp: any; +const getDefaultApp = () => { + if (!defaultApp) defaultApp = buildApp({ startServices: true }); + return defaultApp; +}; +// Service startup owns process-wide pools, readiness and workers. Only the lazy +// singleton may start it; independent HTTP test apps use already-prepared stores. +const createApp = ({ startServices = false } = {}) => { + if (startServices) throw new Error('Service startup is singleton-only; use the app/server exports'); + return buildApp(); +}; module.exports = { - app, - server, - authorizeSocketPodAccess, + createApp, + get app() { return getDefaultApp(); }, + get server() { return getDefaultApp().locals.httpServer; }, + get authorizeSocketPodAccess() { return getDefaultApp().locals.authorizeSocketPodAccess; }, }; + +if (require.main === module) { + const app = getDefaultApp(); + const port = process.env.PORT || 5000; + require('./services/hostedShutdownService').installHostedShutdown(app.locals.httpServer); + app.locals.httpServer.listen(port, () => console.log(`Server running on port ${port}`)); +}