diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 26cb71a1f..17f1b926b 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -744,6 +744,7 @@ export class EvalClient implements CoreEvalClient { evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), dataSourceConfig, kmsKeyArn: input.kmsKeyArn, + outputConfig: input.outputConfig, }), ); } @@ -1092,6 +1093,7 @@ export class EvalClient implements CoreEvalClient { dataSourceConfig, insights: input.insightIds.map((insightId) => ({ insightId })), clusteringConfig: input.clusteringConfig, + outputConfig: input.outputConfig, evaluationExecutionRoleArn: input.evaluationExecutionRoleArn, enableOnCreate: input.enableOnCreate ?? true, }); @@ -1162,6 +1164,11 @@ export class EvalClient implements CoreEvalClient { dataSourceConfig, insights, clusteringConfig, + // Not merged from `current` the way rule/insights are: GET echoes back the + // service-managed default group under /aws/bedrock-agentcore/evaluations/, + // which the write APIs reject, so re-sending it would break every update + // that leaves --output-config off. Matches `online-eval update`. + outputConfig: update.outputConfig, evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index a8b6afd48..6d4d537e9 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -152,6 +152,47 @@ describe("eval batch-insights run", () => { }); }); + test("forwards --output-config so insight results land in the caller's log group", async () => { + const { core } = await run([ + "eval", + "batch-insights", + "run", + "--name", + "insights_run", + "--agent", + "agent-1", + "--output-config", + '{"cloudWatchConfig":{"logGroupName":"/company/agent-insights","metricsNamespace":"Company/AgentInsights","resultDestination":"DEDICATED_LOG_GROUP"}}', + "--json", + ]); + + expect(core.eval.calls[0]?.args[0]).toMatchObject({ + outputConfig: { + cloudWatchConfig: { + logGroupName: "/company/agent-insights", + metricsNamespace: "Company/AgentInsights", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }, + }); + }); + + test("rejects malformed --output-config JSON", async () => { + await expect( + run([ + "eval", + "batch-insights", + "run", + "--name", + "insights_run", + "--agent", + "agent-1", + "--output-config", + "{not json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + }); + test("rejects explicit analysis configuration with an online evaluation source", async () => { await expect( run([ diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx index 775e43d58..0cb34414b 100644 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ b/src/handlers/eval/batch-insights/run/index.tsx @@ -5,6 +5,7 @@ import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; +import { BatchOutputConfig } from "../../batch-evaluation/outputConfig"; import { SessionSource } from "../../sessionSource"; const CONFIGURATION = "Configuration:"; @@ -36,6 +37,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => z.array(z.string()).optional(), { group: ANALYSIS }, ), + ...BatchOutputConfig.flags, ], handle: async (ctx, flags) => { const resolver = new SourceResolver({ stdin: io.stdin }); @@ -57,6 +59,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => evaluatorIds: flags["evaluators"], source, kmsKeyArn: flags["kms-key-arn"], + outputConfig: await BatchOutputConfig.resolve(flags["output-config"], resolver), }, coreOptsFromCtx(ctx), ); diff --git a/src/handlers/eval/online-insight/create/index.tsx b/src/handlers/eval/online-insight/create/index.tsx index f3cbf4f3e..e18e48033 100644 --- a/src/handlers/eval/online-insight/create/index.tsx +++ b/src/handlers/eval/online-insight/create/index.tsx @@ -6,6 +6,7 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { OnlineEvalOutputConfigFlag } from "../../online-eval/outputConfig"; const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; const ARN_PREFIX = "arn:"; @@ -63,6 +64,7 @@ export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) => "a description of the config's monitoring purpose", z.string().optional(), ), + ...OnlineEvalOutputConfigFlag.flags, ], handle: async (ctx, flags) => { for (const id of flags["insight"]) { @@ -91,6 +93,7 @@ export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) => ), insightIds: flags["insight"], clusteringConfig: frequencies ? { frequencies } : undefined, + outputConfig: await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source), evaluationExecutionRoleArn: flags["role-arn"], enableOnCreate: flags["enable-on-create"] === undefined diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx index 6c798ccea..180d42a74 100644 --- a/src/handlers/eval/online-insight/online-insight.test.tsx +++ b/src/handlers/eval/online-insight/online-insight.test.tsx @@ -380,6 +380,42 @@ describe("flag validation", () => { ).rejects.toThrow(/Invalid JSON for option '--data-source-config'/); }); + test("create rejects malformed --output-config JSON", async () => { + await expect( + run([ + "eval", + "online-insight", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--role-arn", + FIXTURE_ROLE_ARN, + "--insight", + FIXTURE_INSIGHT_ID, + "--sampling-rate", + "10", + "--output-config", + "{not json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + }); + + test("update rejects malformed --output-config JSON", async () => { + await expect( + run([ + "eval", + "online-insight", + "update", + "--id", + MISSING_CONFIG_ID, + "--output-config", + "{not json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + }); + // --json forces the headless path so the required-flag error surfaces; without // it a bare invocation opens the TUI under the empty-invocation middleware. test.each(["get", "pause", "resume", "delete"])("%s requires --id", async (command) => { diff --git a/src/handlers/eval/online-insight/update/index.tsx b/src/handlers/eval/online-insight/update/index.tsx index ddd55f6d7..988e59621 100644 --- a/src/handlers/eval/online-insight/update/index.tsx +++ b/src/handlers/eval/online-insight/update/index.tsx @@ -6,6 +6,7 @@ import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { OnlineEvalOutputConfigFlag } from "../../online-eval/outputConfig"; const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; const ARN_PREFIX = "arn:"; @@ -58,6 +59,7 @@ export const createUpdateOnlineInsightHandler = (core: Core, io: AppIO) => z.string().optional(), ), flag("role-arn", "replace the IAM role the online insight assumes", z.string().optional()), + ...OnlineEvalOutputConfigFlag.flags, ], handle: async (ctx, flags) => { if (flags["endpoint"] && flags["clear-endpoint"] === "true") @@ -98,6 +100,7 @@ export const createUpdateOnlineInsightHandler = (core: Core, io: AppIO) => await source.resolveText("data-source-config", flags["data-source-config"]), ), evaluationExecutionRoleArn: flags["role-arn"], + outputConfig: await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source), }, coreOptsFromCtx(ctx), ); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 0b362d526..04b3e696a 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -181,6 +181,7 @@ export type CreateOnlineInsightInput = { clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] }; evaluationExecutionRoleArn: string; enableOnCreate?: boolean; + outputConfig?: OnlineEvalOutputConfig; } & ( | { agent: string; endpoint?: string; dataSourceConfig?: undefined } | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } @@ -197,6 +198,7 @@ export type UpdateOnlineInsightInput = { clearEndpoint?: boolean; dataSourceConfig?: DataSourceConfig; evaluationExecutionRoleArn?: string; + outputConfig?: OnlineEvalOutputConfig; }; // Online insight configs are the same OnlineEvaluationConfig resource with insights @@ -302,6 +304,7 @@ export type StartBatchInsightsInput = { evaluatorIds?: string[]; source: SessionSourceValue; kmsKeyArn?: string; + outputConfig?: OutputConfig; }; // InvokeDatasetInput is the runtime-level shape for replaying a dataset: invoke each