From a8e1fb2b6dc58365ae0c455371f49ea4cc538cb7 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Tue, 22 Sep 2026 14:16:20 +0000 Subject: [PATCH 1/2] test(e2e): cover harness happy path --- e2eTest/constants.ts | 1 + e2eTest/project/harness.test.ts | 159 ++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 e2eTest/project/harness.test.ts diff --git a/e2eTest/constants.ts b/e2eTest/constants.ts index e96363086..9eb966a72 100644 --- a/e2eTest/constants.ts +++ b/e2eTest/constants.ts @@ -1,5 +1,6 @@ export const E2E_PREFIX = "e2e"; export const TAGS = { + HARNESS: "harness", RUNTIME: "runtime", } as const; diff --git a/e2eTest/project/harness.test.ts b/e2eTest/project/harness.test.ts new file mode 100644 index 000000000..c80271ba0 --- /dev/null +++ b/e2eTest/project/harness.test.ts @@ -0,0 +1,159 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import z from "zod"; +import { E2E_PREFIX, TAGS } from "../constants"; +import { CliRunner, parseResult } from "../helpers/run"; + +type HarnessTestCase = { + name: string; + addFlags: string[]; + invokeFlags: string[]; + expectedText?: string; +}; + +const TIMEOUT_MS = { + PROJECT_CREATE: 3 * 60 * 1000, + PROJECT_DEPLOY: 10 * 60 * 1000, + PROJECT_ADD: 3 * 60 * 1000, + PROJECT_REMOVE: 60 * 1000, + PROJECT_INVOKE: 3 * 60 * 1000, +}; + +const CUSTOM_PROMPT_RESPONSE = "HARNESS_PROMPT_VERIFIED"; +const HARNESS_TEST_CASES: HarnessTestCase[] = [ + { + name: "managed_memory", + addFlags: [], + invokeFlags: ["--prompt", "Reply with a short greeting."], + }, + { + name: "disabled_memory", + addFlags: ["--memory", JSON.stringify({ mode: "disabled" })], + invokeFlags: ["--prompt", "Reply with a short greeting."], + }, + { + name: "semantic_memory", + addFlags: ["--memory", JSON.stringify({ mode: "managed", strategies: ["SEMANTIC"] })], + invokeFlags: ["--prompt", "Reply with a short greeting."], + }, + { + name: "custom_prompt", + addFlags: [ + "--system-prompt", + `Reply with exactly ${CUSTOM_PROMPT_RESPONSE} and no other text.`, + ], + invokeFlags: ["--prompt", "Respond now."], + expectedText: CUSTOM_PROMPT_RESPONSE, + }, +]; + +const ProjectCreatedSchema = z.object({ + project: z.object({ path: z.string() }), +}); +const OperationSchema = z.object({ operation: z.string() }); +const DeployResponseSchema = z.object({ message: z.string() }); +const TranscriptItemSchema = z + .object({ + kind: z.string(), + text: z.string().optional(), + }) + .passthrough(); +const HarnessInvokeResponseSchema = z.object({ + sessionId: z.string().min(33).max(100), + transcript: z.array(TranscriptItemSchema).min(2), +}); + +describe("add, deploy, and invoke harnesses", { sequential: true, tags: [TAGS.HARNESS] }, () => { + const cli = new CliRunner(); + const projectName = `${E2E_PREFIX}${Date.now().toString(36)}`; + let projectDir: string; + + beforeAll(async () => { + const projectRoot = await mkdtemp(join(tmpdir(), "agentcore-e2e-")); + const created = parseResult( + ProjectCreatedSchema, + await cli.run( + ["project", "create", "--name", projectName, "--template", "empty", "--skip-git", "--json"], + projectRoot, + ), + ); + projectDir = created.project.path; + }, TIMEOUT_MS.PROJECT_CREATE); + + test.each(HARNESS_TEST_CASES)( + "$name can be added to a project", + { timeout: TIMEOUT_MS.PROJECT_ADD }, + async (harness) => { + const added = parseResult( + OperationSchema, + await cli.run( + ["project", "add", "harness", "--name", harness.name, "--json", ...harness.addFlags], + projectDir, + ), + ); + expect(added.operation).toBe("add"); + }, + ); + + test("deploys all harnesses", { timeout: TIMEOUT_MS.PROJECT_DEPLOY }, async () => { + const deployment = parseResult( + DeployResponseSchema, + await cli.run(["project", "deploy", "--yes", "--json"], projectDir), + ); + expect(deployment.message).toContain("Deployed project"); + }); + + test.each(HARNESS_TEST_CASES)( + "$name can be invoked after deployed", + { concurrent: true, timeout: TIMEOUT_MS.PROJECT_INVOKE }, + async (harness) => { + const response = parseResult( + HarnessInvokeResponseSchema, + await cli.run( + [ + "project", + "invoke", + "harness", + "--name", + harness.name, + "--json", + ...harness.invokeFlags, + ], + projectDir, + ), + ); + const responseText = response.transcript + .filter((item) => item.kind === "text") + .flatMap((item) => item.text ?? []) + .join(""); + + expect(responseText.trim()).not.toBe(""); + if (harness.expectedText) expect(responseText).toContain(harness.expectedText); + }, + ); + + test.each(HARNESS_TEST_CASES)( + "$name can be removed from the project", + { timeout: TIMEOUT_MS.PROJECT_REMOVE }, + async (harness) => { + const removed = parseResult( + OperationSchema, + await cli.run( + ["project", "remove", "harness", "--name", harness.name, "--json"], + projectDir, + ), + ); + expect(removed.operation).toBe("remove"); + }, + ); + + test("deploys the empty project", { timeout: TIMEOUT_MS.PROJECT_DEPLOY }, async () => { + const deployment = parseResult( + DeployResponseSchema, + await cli.run(["project", "deploy", "--yes", "--json"], projectDir), + ); + expect(deployment.message).toContain("Removed project"); + }); +}); From d214d7ad7a8f98f63d2a005b4112c1fb052999af Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Tue, 22 Sep 2026 17:58:27 +0000 Subject: [PATCH 2/2] test(e2e): address harness review comments --- e2eTest/project/harness.test.ts | 36 +++++++++++-------------------- e2eTest/project/templates.test.ts | 10 +-------- e2eTest/timeouts.ts | 8 +++++++ 3 files changed, 21 insertions(+), 33 deletions(-) create mode 100644 e2eTest/timeouts.ts diff --git a/e2eTest/project/harness.test.ts b/e2eTest/project/harness.test.ts index c80271ba0..bc3be7bc4 100644 --- a/e2eTest/project/harness.test.ts +++ b/e2eTest/project/harness.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import z from "zod"; import { E2E_PREFIX, TAGS } from "../constants"; import { CliRunner, parseResult } from "../helpers/run"; +import { TIMEOUT_MS } from "../timeouts"; type HarnessTestCase = { name: string; @@ -13,14 +14,6 @@ type HarnessTestCase = { expectedText?: string; }; -const TIMEOUT_MS = { - PROJECT_CREATE: 3 * 60 * 1000, - PROJECT_DEPLOY: 10 * 60 * 1000, - PROJECT_ADD: 3 * 60 * 1000, - PROJECT_REMOVE: 60 * 1000, - PROJECT_INVOKE: 3 * 60 * 1000, -}; - const CUSTOM_PROMPT_RESPONSE = "HARNESS_PROMPT_VERIFIED"; const HARNESS_TEST_CASES: HarnessTestCase[] = [ { @@ -134,26 +127,21 @@ describe("add, deploy, and invoke harnesses", { sequential: true, tags: [TAGS.HA }, ); - test.each(HARNESS_TEST_CASES)( - "$name can be removed from the project", - { timeout: TIMEOUT_MS.PROJECT_REMOVE }, - async (harness) => { + test( + "removes all harnesses and deploys the empty project", + { timeout: TIMEOUT_MS.PROJECT_REMOVE + TIMEOUT_MS.PROJECT_DEPLOY }, + async () => { const removed = parseResult( OperationSchema, - await cli.run( - ["project", "remove", "harness", "--name", harness.name, "--json"], - projectDir, - ), + await cli.run(["project", "remove", "all", "--yes", "--json"], projectDir), ); expect(removed.operation).toBe("remove"); + + const deployment = parseResult( + DeployResponseSchema, + await cli.run(["project", "deploy", "--yes", "--json"], projectDir), + ); + expect(deployment.message).toContain("Removed project"); }, ); - - test("deploys the empty project", { timeout: TIMEOUT_MS.PROJECT_DEPLOY }, async () => { - const deployment = parseResult( - DeployResponseSchema, - await cli.run(["project", "deploy", "--yes", "--json"], projectDir), - ); - expect(deployment.message).toContain("Removed project"); - }); }); diff --git a/e2eTest/project/templates.test.ts b/e2eTest/project/templates.test.ts index d67e75170..2f177ed9c 100644 --- a/e2eTest/project/templates.test.ts +++ b/e2eTest/project/templates.test.ts @@ -6,6 +6,7 @@ import z from "zod"; import { E2E_PREFIX, TAGS } from "../constants"; import { CliRunner, parseResult, type RunResult } from "../helpers/run"; import { retry } from "../helpers/retry"; +import { TIMEOUT_MS } from "../timeouts"; type RuntimeTemplateTestCase = { name: string; @@ -15,15 +16,6 @@ type RuntimeTemplateTestCase = { invokeFlags?: string[]; }; -export const TIMEOUT_MS = { - PROJECT_CREATE: 3 * 60 * 1000, - PROJECT_DEV: 3 * 60 * 1000, - PROJECT_DEPLOY: 10 * 60 * 1000, - PROJECT_ADD: 3 * 60 * 1000, - PROJECT_REMOVE: 60 * 1000, - PROJECT_INVOKE: 3 * 60 * 1000, -}; - const RUNTIME_TEMPLATES: RuntimeTemplateTestCase[] = [ { name: "agent_python_minimal", diff --git a/e2eTest/timeouts.ts b/e2eTest/timeouts.ts new file mode 100644 index 000000000..d2549b8eb --- /dev/null +++ b/e2eTest/timeouts.ts @@ -0,0 +1,8 @@ +export const TIMEOUT_MS = { + PROJECT_CREATE: 3 * 60 * 1000, + PROJECT_DEV: 3 * 60 * 1000, + PROJECT_DEPLOY: 10 * 60 * 1000, + PROJECT_ADD: 3 * 60 * 1000, + PROJECT_REMOVE: 60 * 1000, + PROJECT_INVOKE: 3 * 60 * 1000, +};