Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/core/eval.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ export class EvalClient implements CoreEvalClient {
evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })),
dataSourceConfig,
kmsKeyArn: input.kmsKeyArn,
outputConfig: input.outputConfig,
}),
);
}
Expand Down Expand Up @@ -1092,6 +1093,7 @@ export class EvalClient implements CoreEvalClient {
dataSourceConfig,
insights: input.insightIds.map((insightId) => ({ insightId })),
clusteringConfig: input.clusteringConfig,
outputConfig: input.outputConfig,
evaluationExecutionRoleArn: input.evaluationExecutionRoleArn,
enableOnCreate: input.enableOnCreate ?? true,
});
Expand Down Expand Up @@ -1162,6 +1164,11 @@ export class EvalClient implements CoreEvalClient {
dataSourceConfig,
insights,
clusteringConfig,
// Not merged from `current` the way rule/insights are: GET echoes back the
// service-managed default group under /aws/bedrock-agentcore/evaluations/,
// which the write APIs reject, so re-sending it would break every update
// that leaves --output-config off. Matches `online-eval update`.
outputConfig: update.outputConfig,
evaluationExecutionRoleArn: update.evaluationExecutionRoleArn,
}),
);
Expand Down
41 changes: 41 additions & 0 deletions src/handlers/eval/batch-insights/batch-insights.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,47 @@ describe("eval batch-insights run", () => {
});
});

test("forwards --output-config so insight results land in the caller's log group", async () => {
const { core } = await run([
"eval",
"batch-insights",
"run",
"--name",
"insights_run",
"--agent",
"agent-1",
"--output-config",
'{"cloudWatchConfig":{"logGroupName":"/company/agent-insights","metricsNamespace":"Company/AgentInsights","resultDestination":"DEDICATED_LOG_GROUP"}}',
"--json",
]);

expect(core.eval.calls[0]?.args[0]).toMatchObject({
outputConfig: {
cloudWatchConfig: {
logGroupName: "/company/agent-insights",
metricsNamespace: "Company/AgentInsights",
resultDestination: "DEDICATED_LOG_GROUP",
},
},
});
});

test("rejects malformed --output-config JSON", async () => {
await expect(
run([
"eval",
"batch-insights",
"run",
"--name",
"insights_run",
"--agent",
"agent-1",
"--output-config",
"{not json",
]),
).rejects.toThrow(/Invalid JSON for option '--output-config'/);
});

test("rejects explicit analysis configuration with an online evaluation source", async () => {
await expect(
run([
Expand Down
3 changes: 3 additions & 0 deletions src/handlers/eval/batch-insights/run/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { createHandler, flag } from "../../../../router";
import { JsonRendererKey } from "../../../../tui";
import type { Core } from "../../../types";
import { coreOptsFromCtx } from "../../../utils";
import { BatchOutputConfig } from "../../batch-evaluation/outputConfig";
import { SessionSource } from "../../sessionSource";

const CONFIGURATION = "Configuration:";
Expand Down Expand Up @@ -36,6 +37,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
z.array(z.string()).optional(),
{ group: ANALYSIS },
),
...BatchOutputConfig.flags,
],
handle: async (ctx, flags) => {
const resolver = new SourceResolver({ stdin: io.stdin });
Expand All @@ -57,6 +59,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
evaluatorIds: flags["evaluators"],
source,
kmsKeyArn: flags["kms-key-arn"],
outputConfig: await BatchOutputConfig.resolve(flags["output-config"], resolver),
},
coreOptsFromCtx(ctx),
);
Expand Down
3 changes: 3 additions & 0 deletions src/handlers/eval/online-insight/create/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { JsonRendererKey } from "../../../../tui";
import { SourceResolver, type AppIO } from "../../../../io";
import type { Core } from "../../../types";
import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils";
import { OnlineEvalOutputConfigFlag } from "../../online-eval/outputConfig";

const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight.";
const ARN_PREFIX = "arn:";
Expand Down Expand Up @@ -63,6 +64,7 @@ export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) =>
"a description of the config's monitoring purpose",
z.string().optional(),
),
...OnlineEvalOutputConfigFlag.flags,
],
handle: async (ctx, flags) => {
for (const id of flags["insight"]) {
Expand Down Expand Up @@ -91,6 +93,7 @@ export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) =>
),
insightIds: flags["insight"],
clusteringConfig: frequencies ? { frequencies } : undefined,
outputConfig: await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source),
evaluationExecutionRoleArn: flags["role-arn"],
enableOnCreate:
flags["enable-on-create"] === undefined
Expand Down
36 changes: 36 additions & 0 deletions src/handlers/eval/online-insight/online-insight.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,42 @@ describe("flag validation", () => {
).rejects.toThrow(/Invalid JSON for option '--data-source-config'/);
});

test("create rejects malformed --output-config JSON", async () => {
await expect(
run([
"eval",
"online-insight",
"create",
"--name",
CONFIG_NAME,
"--agent",
FIXTURE_AGENT_ID,
"--role-arn",
FIXTURE_ROLE_ARN,
"--insight",
FIXTURE_INSIGHT_ID,
"--sampling-rate",
"10",
"--output-config",
"{not json",
]),
).rejects.toThrow(/Invalid JSON for option '--output-config'/);
});

test("update rejects malformed --output-config JSON", async () => {
await expect(
run([
"eval",
"online-insight",
"update",
"--id",
MISSING_CONFIG_ID,
"--output-config",
"{not json",
]),
).rejects.toThrow(/Invalid JSON for option '--output-config'/);
});

// --json forces the headless path so the required-flag error surfaces; without
// it a bare invocation opens the TUI under the empty-invocation middleware.
test.each(["get", "pause", "resume", "delete"])("%s requires --id", async (command) => {
Expand Down
3 changes: 3 additions & 0 deletions src/handlers/eval/online-insight/update/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { JsonRendererKey } from "../../../../tui";
import { SourceResolver, type AppIO } from "../../../../io";
import type { Core } from "../../../types";
import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils";
import { OnlineEvalOutputConfigFlag } from "../../online-eval/outputConfig";

const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight.";
const ARN_PREFIX = "arn:";
Expand Down Expand Up @@ -58,6 +59,7 @@ export const createUpdateOnlineInsightHandler = (core: Core, io: AppIO) =>
z.string().optional(),
),
flag("role-arn", "replace the IAM role the online insight assumes", z.string().optional()),
...OnlineEvalOutputConfigFlag.flags,
],
handle: async (ctx, flags) => {
if (flags["endpoint"] && flags["clear-endpoint"] === "true")
Expand Down Expand Up @@ -98,6 +100,7 @@ export const createUpdateOnlineInsightHandler = (core: Core, io: AppIO) =>
await source.resolveText("data-source-config", flags["data-source-config"]),
),
evaluationExecutionRoleArn: flags["role-arn"],
outputConfig: await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], source),
},
coreOptsFromCtx(ctx),
);
Expand Down
3 changes: 3 additions & 0 deletions src/handlers/eval/types.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ export type CreateOnlineInsightInput = {
clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] };
evaluationExecutionRoleArn: string;
enableOnCreate?: boolean;
outputConfig?: OnlineEvalOutputConfig;
} & (
| { agent: string; endpoint?: string; dataSourceConfig?: undefined }
| { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig }
Expand All @@ -197,6 +198,7 @@ export type UpdateOnlineInsightInput = {
clearEndpoint?: boolean;
dataSourceConfig?: DataSourceConfig;
evaluationExecutionRoleArn?: string;
outputConfig?: OnlineEvalOutputConfig;
};

// Online insight configs are the same OnlineEvaluationConfig resource with insights
Expand Down Expand Up @@ -302,6 +304,7 @@ export type StartBatchInsightsInput = {
evaluatorIds?: string[];
source: SessionSourceValue;
kmsKeyArn?: string;
outputConfig?: OutputConfig;
};

// InvokeDatasetInput is the runtime-level shape for replaying a dataset: invoke each
Expand Down
Loading