diff --git a/packages/cli/src/commands/media-treatment-analysis.ts b/packages/cli/src/commands/media-treatment-analysis.ts new file mode 100644 index 0000000000..bde303fa22 --- /dev/null +++ b/packages/cli/src/commands/media-treatment-analysis.ts @@ -0,0 +1,20 @@ +import { + analyzeMediaGrade, + type MediaTreatmentAnalysis, +} from "@hyperframes/core/media-grade-analyzer"; +import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js"; + +interface CliMediaTreatmentAnalysis extends Omit { + suggestedPatch: { adjust: MediaTreatmentAnalysis["adjust"] }; +} + +export function analyzeMediaTreatment(mediaPath: string): CliMediaTreatmentAnalysis { + const ffmpegPath = findFFmpeg(); + const ffprobePath = findFFprobe(); + if (!ffmpegPath || !ffprobePath) { + throw new Error(`FFmpeg and ffprobe are required. ${getFFmpegInstallHint()}`); + } + const analysis = analyzeMediaGrade(mediaPath, { ffmpegPath, ffprobePath }); + const { adjust, ...evidence } = analysis; + return { ...evidence, suggestedPatch: { adjust } }; +} diff --git a/packages/cli/src/commands/media-treatment.test.ts b/packages/cli/src/commands/media-treatment.test.ts index e302b32892..d525a6c64f 100644 --- a/packages/cli/src/commands/media-treatment.test.ts +++ b/packages/cli/src/commands/media-treatment.test.ts @@ -1,6 +1,6 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { runCommand } from "citty"; import { describe, expect, it, vi } from "vitest"; import { @@ -12,6 +12,7 @@ import { getMediaTreatmentCapabilityDetail, getMediaTreatmentCapabilityOverview, mediaTreatmentCommand, + resolveMediaTreatmentSource, } from "./media-treatment.js"; import { CliRuntimeError } from "../utils/commandResult.js"; @@ -22,6 +23,9 @@ describe("applyMediaTreatmentToHtml", () => { const overview = getMediaTreatmentCapabilityOverview(); expect(overview.families.find(({ id }) => id === "correction")).not.toHaveProperty("items"); + expect(overview.families.find(({ id }) => id === "grading")).toMatchObject({ + label: "Color Grading", + }); expect(overview.families.find(({ id }) => id === "art")).not.toHaveProperty("items"); expect(overview.families.find(({ id }) => id === "overlays")).toMatchObject({ owner: "registry", @@ -76,6 +80,40 @@ describe("applyMediaTreatmentToHtml", () => { family: "finishing", control: expect.objectContaining({ key: "vignette" }), }); + expect(getMediaTreatmentCapabilityDetail("wheels")).toMatchObject({ + contract: expect.objectContaining({ zones: ["shadows", "midtones", "highlights"] }), + }); + expect(getMediaTreatmentCapabilityDetail("curves")).toMatchObject({ + contract: expect.objectContaining({ channels: ["master", "red", "green", "blue"] }), + }); + const hueCurves = getMediaTreatmentCapabilityDetail("hue-curves"); + const serializedHueCurves = JSON.stringify(hueCurves); + expect(serializedHueCurves).toContain('"maxPoints":16'); + expect(serializedHueCurves).toContain('"key":"hueVsHue"'); + expect(serializedHueCurves).toContain('"key":"hueVsSaturation"'); + expect(serializedHueCurves).toContain('"key":"hueVsLuma"'); + expect(getMediaTreatmentCapabilityDetail("secondary")).toMatchObject({ + contract: expect.objectContaining({ + max: 4, + saturation: expect.objectContaining({ relation: "min < max" }), + luma: expect.objectContaining({ relation: "min < max" }), + }), + }); + expect(getMediaTreatmentCapabilityDetail("grading")).toMatchObject({ + order: [ + "adjust", + "wheels", + "curves", + "hueCurves", + "secondaries", + "lut", + "details", + "effects", + ], + }); + expect(getMediaTreatmentCapabilityDetail("scopes")).toMatchObject({ + command: expect.stringContaining("--analyze"), + }); }); it("rejects unknown capability lookups", () => { @@ -113,6 +151,118 @@ describe("applyMediaTreatmentToHtml", () => { expect(result.html).toContain("data-color-grading="); }); + it("normalizes and persists advanced grading on real media", () => { + const result = applyMediaTreatmentToHtml(VIDEO, { + selector: "#hero", + grading: { + wheels: { shadows: { hue: 205, amount: 0.08, level: 0.02 } }, + curves: { + master: [ + [0, 0], + [0.5, 0.55], + [1, 1], + ], + }, + hueCurves: { + hueVsSaturation: [ + [180, 0], + [210, 0.15], + [240, 0], + ], + }, + secondaries: [ + { + key: { + hue: { center: 215, range: 25, softness: 10 }, + saturation: { min: 0.2, max: 1, softness: 0.08 }, + luma: { min: 0.1, max: 0.9, softness: 0.08 }, + }, + correction: { saturation: 0.15, luma: 0.03 }, + }, + ], + }, + }); + + expect(result.changed).toBe(true); + expect(result.value).toContain('"wheels"'); + expect(result.value).toContain('"curves"'); + expect(result.value).toContain('"hueCurves"'); + expect(result.value).toContain('"secondaries"'); + }); + + it("persists a disabled secondary without treating it as an active grade", () => { + const result = applyMediaTreatmentToHtml(VIDEO, { + selector: "#hero", + grading: { + secondaries: [ + { + enabled: false, + key: { hue: { center: 215, range: 25 } }, + correction: { saturation: 0.15 }, + }, + ], + }, + }); + + expect(result.changed).toBe(true); + expect(result.value).toContain('"enabled":false'); + expect(result.value).toContain('"secondaries"'); + }); + + it("resolves nested composition media through the shared project-root contract", () => { + const project = mkdtempSync(join(tmpdir(), "hf-media-treatment-assets-")); + const escapedAsset = join(project, "..", `${basename(project)}-escape.mp4`); + mkdirSync(join(project, "capture"), { recursive: true }); + mkdirSync(join(project, "assets"), { recursive: true }); + writeFileSync(join(project, "capture", "talking-head.mp4"), ""); + writeFileSync(join(project, "assets", "photo.webp"), ""); + writeFileSync(join(project, "assets", "My Clip.mp4"), ""); + writeFileSync(escapedAsset, ""); + + try { + expect( + resolveMediaTreatmentSource( + project, + "compositions/scene.html", + "../capture/talking-head.mp4?v=1#frame", + ), + ).toBe(join(project, "capture/talking-head.mp4")); + expect( + resolveMediaTreatmentSource(project, "compositions/scene.html", "assets/photo.webp"), + ).toBe(join(project, "assets/photo.webp")); + expect( + resolveMediaTreatmentSource( + project, + "compositions/scene.html", + "/assets/My%20Clip.mp4?v=1", + ), + ).toBe(join(project, "assets/My Clip.mp4")); + expect(() => + resolveMediaTreatmentSource( + project, + "compositions/scene.html", + "https://example.com/a.mp4", + ), + ).toThrow(/local project asset/); + expect(() => resolveMediaTreatmentSource(project, "compositions/scene.html", "#")).toThrow( + /local project asset/, + ); + expect(() => + resolveMediaTreatmentSource(project, "compositions/scene.html", "missing.mp4"), + ).toThrow(/Media file not found/); + expect(() => + resolveMediaTreatmentSource( + project, + "compositions/scene.html", + `../../${basename(escapedAsset)}`, + ), + ).toThrow(/Media file not found/); + } finally { + rmSync(project, { recursive: true, force: true }); + rmSync(escapedAsset, { force: true }); + } + }); + it("merges a validated patch and reports the stored before and after payloads", () => { const initial = applyMediaTreatmentToHtml(VIDEO, { selector: "#hero", diff --git a/packages/cli/src/commands/media-treatment.ts b/packages/cli/src/commands/media-treatment.ts index 6ce2ecfb19..bdeba5889e 100644 --- a/packages/cli/src/commands/media-treatment.ts +++ b/packages/cli/src/commands/media-treatment.ts @@ -3,7 +3,7 @@ import { relative, resolve } from "node:path"; import { HF_COLOR_GRADING_ATTR, getHfColorGradingCapabilities, - isHfColorGradingActive, + hasHfColorGradingAuthoredValues, isPathInside, normalizeHfColorGrading, serializeHfColorGrading, @@ -12,6 +12,12 @@ import { isColorGradingVariableRef, validateColorGradingContract, } from "@hyperframes/parsers/color-grading-contract"; +import { + cleanAssetUrl, + isRemoteOrInlineUrl, + resolveExistingLocalAsset, +} from "@hyperframes/parsers/asset-resolution"; +import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths"; import { patchElementInHtml } from "@hyperframes/studio-server/source-mutation"; import { defineCommand } from "citty"; import { parseHTML } from "linkedom"; @@ -22,6 +28,7 @@ import { normalizeErrorMessage } from "../utils/errorMessage.js"; import { readOptionalString } from "../utils/pathArgs.js"; import { resolveProject } from "../utils/project.js"; import { withMeta } from "../utils/updateCheck.js"; +import { analyzeMediaTreatment } from "./media-treatment-analysis.js"; export function getMediaTreatmentCapabilityOverview() { const capabilities = getHfColorGradingCapabilities(); @@ -37,6 +44,11 @@ export function getMediaTreatmentCapabilityOverview() { colorSpace: capabilities.colorSpace, families: [ family("correction", "Adjust", "Fix exposure, tonal balance, color casts, and saturation."), + family( + "grading", + "Color Grading", + "Shape tonal color with wheels, RGB and hue curves, and selective HSL correction.", + ), family( "presets", "Presets", @@ -157,11 +169,96 @@ export function getMediaTreatmentCapabilityDetail(id: string): unknown { if (palette) return { ...palette, apply: { palette: palette.colors } }; const details = { + grading: { + id, + description: + "Shape media after primary correction with wheels, curves, HSL secondaries, and an optional user-owned LUT.", + order: [ + "adjust", + "wheels", + "curves", + "hueCurves", + "secondaries", + "lut", + "details", + "effects", + ], + discover: ["wheels", "curves", "hue-curves", "secondary", "scopes", "lut"], + }, correction: { id, description: "Fix exposure, tonal balance, color casts, and saturation.", controls: capabilities.adjustments, }, + wheels: { + id, + description: "Shadow, midtone, and highlight color wheels.", + contract: capabilities.wheels, + apply: { + wheels: { + shadows: { hue: 205, amount: 0.08, level: 0 }, + highlights: { hue: 35, amount: 0.06, level: 0 }, + }, + }, + }, + curves: { + id, + description: "Master and per-channel curves using normalized [input, output] points.", + contract: capabilities.curves, + apply: { + curves: { + master: [ + [0, 0], + [0.25, 0.2], + [0.75, 0.82], + [1, 1], + ], + }, + }, + }, + "hue-curves": { + id, + description: "Hue-selective curves using [hueDegrees, delta] points.", + contract: capabilities.hueCurves, + apply: { + hueCurves: { + hueVsSaturation: [ + [180, 0], + [210, 0.15], + [240, 0], + ], + }, + }, + }, + secondary: { + id, + description: "Up to four ordered HSL selections with bounded color correction.", + contract: capabilities.secondaries, + apply: { + secondaries: [ + { + key: { + hue: { center: 215, range: 25, softness: 10 }, + saturation: { min: 0.2, max: 1, softness: 0.08 }, + luma: { min: 0.1, max: 0.9, softness: 0.08 }, + }, + correction: { saturation: 0.15, luma: 0.03 }, + }, + ], + }, + }, + scopes: { + id, + description: + "Deterministic source measurements for agent decisions; visual scopes remain a Studio display.", + command: + "hyperframes media-treatment --file compositions/scene.html --selector '#hero' --analyze --json", + output: [ + "source color metadata and HDR/LOG warnings", + "luma percentile and clipping evidence", + "bounded suggested primary correction", + ], + }, presets: { id, description: "Tested starting points for color and stylized media effects.", @@ -222,6 +319,10 @@ export const examples: Example[] = [ "Preview the exact mutation without writing", `hyperframes media-treatment --file compositions/scene.html --selector 'video' --grading '{"preset":"warm-daylight"}' --apply --dry-run --json`, ], + [ + "Measure one local media source before choosing a correction", + `hyperframes media-treatment --selector '#hero' --analyze --json`, + ], ["Remove a treatment", `hyperframes media-treatment --selector '#hero' --clear`], ]; @@ -298,7 +399,7 @@ function serializeGradingPatch(before: unknown, patch: unknown): string | null { } const normalized = normalizeHfColorGrading(grading); if (!normalized) throw new Error("--grading must be valid HyperFrames color-grading JSON"); - return isHfColorGradingActive(normalized) ? serializeHfColorGrading(normalized) : null; + return hasHfColorGradingAuthoredValues(normalized) ? serializeHfColorGrading(normalized) : null; } function queryIncludingTemplates(root: Document | Element, selector: string): Element[] { @@ -375,6 +476,38 @@ function parseSelectorIndex(raw: string | undefined): number | undefined { return value; } +function mediaSourceForElement(element: Element): string { + const src = + element.getAttribute("src") ?? + (element.tagName.toLowerCase() === "video" + ? element.querySelector("source")?.getAttribute("src") + : null); + if (!src) throw new Error("Selected media has no analyzable src"); + return src; +} + +export function resolveMediaTreatmentSource( + projectDir: string, + compositionFile: string, + source: string, +): string { + const sourceUrl = source.trim(); + if (!sourceUrl) throw new Error("Selected media has no analyzable local src"); + if (isRemoteOrInlineUrl(sourceUrl)) { + throw new Error( + "Media analysis requires a local project asset; freeze remote media with media-use first", + ); + } + const cleanSource = cleanAssetUrl(sourceUrl); + if (!cleanSource) throw new Error("Selected media has no analyzable local src"); + const projectRelative = cleanSource.startsWith("/") + ? cleanSource + : rewriteAssetPath(compositionFile, cleanSource); + const asset = resolveExistingLocalAsset(projectDir, projectRelative); + if (!asset) throw new Error(`Media file not found: ${source}`); + return asset.resolved; +} + function parseGrading(raw: string | undefined, apply: boolean, clear: boolean): unknown { if (clear) { if (raw !== undefined || apply) { @@ -400,6 +533,157 @@ function mutationVerb(action: "apply" | "clear", changed: boolean, dryRun: boole return action === "apply" ? "Applied" : "Cleared"; } +interface MediaTreatmentCommandArgs { + capabilities?: boolean; + capability?: string; + all?: boolean; + project?: string; + file?: string; + selector?: string; + "selector-index"?: string; + grading?: string; + apply?: boolean; + analyze?: boolean; + clear?: boolean; + "dry-run"?: boolean; + json?: boolean; +} + +function runCapabilityQuery(args: MediaTreatmentCommandArgs): void { + const capability = readOptionalString(args.capability); + const hasMutationOption = [ + readOptionalString(args.selector), + readOptionalString(args.grading), + args.clear, + args["dry-run"], + args.apply, + args.analyze, + ].some(Boolean); + if (hasMutationOption) throw new Error("--capabilities cannot be combined with mutation options"); + if (args.all === true && capability) + throw new Error("Use either --all or --capability, not both"); + + let capabilities: unknown = getMediaTreatmentCapabilityOverview(); + if (args.all === true) capabilities = getHfColorGradingCapabilities(); + else if (capability) capabilities = getMediaTreatmentCapabilityDetail(capability); + console.log(JSON.stringify(withMeta({ ok: true, capabilities }), null, 2)); +} + +function resolveMutationFile(args: MediaTreatmentCommandArgs) { + const project = resolveProject(readOptionalString(args.project)); + const fileArg = readOptionalString(args.file) ?? "index.html"; + const filePath = resolve(project.dir, fileArg); + if (!isPathInside(filePath, project.dir) || !filePath.toLowerCase().endsWith(".html")) { + throw new Error("--file must be an HTML file inside the project"); + } + if (!existsSync(filePath)) throw new Error(`Composition file not found: ${fileArg}`); + return { project, filePath }; +} + +function analyzeTarget(args: MediaTreatmentCommandArgs) { + const { project, filePath } = resolveMutationFile(args); + const selector = readOptionalString(args.selector); + if (!selector) throw new Error("--selector is required"); + if ( + readOptionalString(args.grading) || + args.clear === true || + args.apply === true || + args["dry-run"] === true + ) { + throw new Error("--analyze cannot be combined with mutation options"); + } + const selectorIndex = parseSelectorIndex(readOptionalString(args["selector-index"])); + const { element, tag } = selectMediaElement( + readFileSync(filePath, "utf8"), + selector, + selectorIndex, + ); + const source = mediaSourceForElement(element); + const compositionFile = relative(project.dir, filePath).split("\\").join("/"); + const mediaPath = resolveMediaTreatmentSource(project.dir, compositionFile, source); + return { + ok: true, + action: "analyze", + file: compositionFile || "index.html", + selector, + selectorIndex: selectorIndex ?? 0, + tag, + media: source, + ...analyzeMediaTreatment(mediaPath), + }; +} + +function prepareMutation(args: MediaTreatmentCommandArgs) { + const { project, filePath } = resolveMutationFile(args); + const selector = readOptionalString(args.selector); + if (!selector) throw new Error("--selector is required"); + const clear = args.clear === true; + const apply = args.apply === true; + const selectorIndex = parseSelectorIndex(readOptionalString(args["selector-index"])); + const result = applyMediaTreatmentToHtml(readFileSync(filePath, "utf8"), { + selector, + selectorIndex, + grading: parseGrading(readOptionalString(args.grading), apply, clear), + clear, + }); + const dryRun = args["dry-run"] === true; + if (result.changed && !dryRun) writeFileSync(filePath, result.html); + + const action: "clear" | "apply" = result.value === null ? "clear" : "apply"; + return { + action, + result, + selector, + payload: { + ok: true, + action, + file: relative(project.dir, filePath) || "index.html", + selector, + selectorIndex: selectorIndex ?? 0, + tag: result.tag, + changed: result.changed, + dryRun, + before: result.before, + after: result.after, + }, + }; +} + +function isCapabilityQuery(args: MediaTreatmentCommandArgs): boolean { + return ( + args.capabilities === true || Boolean(readOptionalString(args.capability)) || args.all === true + ); +} + +function printAnalysis(args: MediaTreatmentCommandArgs): void { + const result = analyzeTarget(args); + if (args.json === true) { + console.log(JSON.stringify(withMeta(result), null, 2)); + return; + } + console.log(`${c.success("◇")} Analyzed ${c.accent(result.selector)}`); + for (const diagnosis of result.diagnosis) console.log(` ${diagnosis}`); + for (const warning of result.warnings) console.log(` ${c.warn(warning)}`); + console.log(` suggested patch: ${JSON.stringify(result.suggestedPatch)}`); +} + +function printMutation(args: MediaTreatmentCommandArgs): void { + const { action, result, selector, payload } = prepareMutation(args); + if (args.json === true) { + console.log(JSON.stringify(withMeta(payload), null, 2)); + return; + } + const verb = mutationVerb(action, result.changed, payload.dryRun); + console.log(`${c.success("◇")} ${verb} media treatment on ${c.accent(selector)}`); +} + +function printFailure(error: unknown, json: boolean): void { + const message = normalizeErrorMessage(error); + if (json) console.log(JSON.stringify(withMeta({ ok: false, error: message }))); + else console.error(`${c.error("✗")} ${message}`); + failCommand(); +} + export const mediaTreatmentCommand = defineCommand({ meta: { name: "media-treatment", @@ -439,6 +723,11 @@ export const mediaTreatmentCommand = defineCommand({ description: "Apply the validated grading patch (explicit agent form)", default: false, }, + analyze: { + type: "boolean", + description: "Measure selected local media and suggest a bounded primary correction", + default: false, + }, clear: { type: "boolean", description: "Remove color grading from the target", default: false }, "dry-run": { type: "boolean", @@ -448,90 +737,15 @@ export const mediaTreatmentCommand = defineCommand({ json: { type: "boolean", description: "Output an agent-friendly JSON result", default: false }, }, run({ args }) { - const runCapabilityQuery = () => { - const capability = readOptionalString(args.capability); - const hasMutationOption = [ - readOptionalString(args.selector), - readOptionalString(args.grading), - args.clear, - args["dry-run"], - args.apply, - ].some(Boolean); - if (hasMutationOption) { - throw new Error("--capabilities cannot be combined with mutation options"); - } - if (args.all === true && capability) { - throw new Error("Use either --all or --capability, not both"); - } - let capabilities: unknown = getMediaTreatmentCapabilityOverview(); - if (args.all === true) capabilities = getHfColorGradingCapabilities(); - else if (capability) capabilities = getMediaTreatmentCapabilityDetail(capability); - console.log(JSON.stringify(withMeta({ ok: true, capabilities }), null, 2)); - }; - - const resolveMutationFile = () => { - const project = resolveProject(readOptionalString(args.project)); - const fileArg = readOptionalString(args.file) ?? "index.html"; - const filePath = resolve(project.dir, fileArg); - if (!isPathInside(filePath, project.dir) || !filePath.toLowerCase().endsWith(".html")) { - throw new Error("--file must be an HTML file inside the project"); - } - if (!existsSync(filePath)) throw new Error(`Composition file not found: ${fileArg}`); - return { project, filePath }; - }; - - const prepareMutation = () => { - const { project, filePath } = resolveMutationFile(); - const selector = readOptionalString(args.selector); - if (!selector) throw new Error("--selector is required"); - const clear = args.clear === true; - const apply = args.apply === true; - const selectorIndex = parseSelectorIndex(readOptionalString(args["selector-index"])); - const result = applyMediaTreatmentToHtml(readFileSync(filePath, "utf8"), { - selector, - selectorIndex, - grading: parseGrading(readOptionalString(args.grading), apply, clear), - clear, - }); - const dryRun = args["dry-run"] === true; - if (result.changed && !dryRun) writeFileSync(filePath, result.html); - - const action: "clear" | "apply" = result.value === null ? "clear" : "apply"; - return { - action, - result, - selector, - payload: { - ok: true, - action, - file: relative(project.dir, filePath) || "index.html", - selector, - selectorIndex: selectorIndex ?? 0, - tag: result.tag, - changed: result.changed, - dryRun, - before: result.before, - after: result.after, - }, - }; - }; - try { - if (args.capabilities === true || readOptionalString(args.capability) || args.all === true) { - return runCapabilityQuery(); - } - const { action, result, selector, payload } = prepareMutation(); - if (args.json === true) { - console.log(JSON.stringify(withMeta(payload), null, 2)); - } else { - const verb = mutationVerb(action, result.changed, payload.dryRun); - console.log(`${c.success("◇")} ${verb} media treatment on ${c.accent(selector)}`); + if (isCapabilityQuery(args)) return runCapabilityQuery(args); + if (args.analyze === true) { + printAnalysis(args); + return; } + printMutation(args); } catch (error) { - const message = normalizeErrorMessage(error); - if (args.json === true) console.log(JSON.stringify(withMeta({ ok: false, error: message }))); - else console.error(`${c.error("✗")} ${message}`); - failCommand(); + printFailure(error, args.json === true); } }, }); diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index b73e8844a6..f3505daa6a 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -86,6 +86,12 @@ "types": "./dist/colorLuts.d.ts", "environments": ["browser", "bun", "node"] }, + "./media-grade-analyzer": { + "source": "./src/mediaGradeAnalyzer.ts", + "runtime": "./dist/mediaGradeAnalyzer.js", + "types": "./dist/mediaGradeAnalyzer.d.ts", + "environments": ["bun", "node"] + }, "./storyboard": { "source": "./src/storyboard/index.ts", "runtime": "./dist/storyboard/index.js", diff --git a/packages/core/package.json b/packages/core/package.json index e0d3f428fe..9df53a5c00 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -100,6 +100,12 @@ "import": "./src/colorLuts.ts", "types": "./src/colorLuts.ts" }, + "./media-grade-analyzer": { + "bun": "./src/mediaGradeAnalyzer.ts", + "node": "./dist/mediaGradeAnalyzer.js", + "import": "./src/mediaGradeAnalyzer.ts", + "types": "./src/mediaGradeAnalyzer.ts" + }, "./storyboard": { "bun": "./src/storyboard/index.ts", "node": "./dist/storyboard/index.js", @@ -334,6 +340,10 @@ "import": "./dist/colorLuts.js", "types": "./dist/colorLuts.d.ts" }, + "./media-grade-analyzer": { + "import": "./dist/mediaGradeAnalyzer.js", + "types": "./dist/mediaGradeAnalyzer.d.ts" + }, "./storyboard": { "import": "./dist/storyboard/index.js", "types": "./dist/storyboard/index.d.ts" diff --git a/packages/core/src/mediaGradeAnalyzer.ts b/packages/core/src/mediaGradeAnalyzer.ts new file mode 100644 index 0000000000..7ea693435a --- /dev/null +++ b/packages/core/src/mediaGradeAnalyzer.ts @@ -0,0 +1,356 @@ +import { execFileSync } from "node:child_process"; +import { basename, extname } from "node:path"; + +const IMAGE_EXTENSIONS = new Set([ + ".jpg", + ".jpeg", + ".png", + ".webp", + ".gif", + ".bmp", + ".tif", + ".tiff", +]); +const SAMPLE_FRAMES = 5; +const DEFAULT_TIMEOUT_MS = 15_000; + +export interface GradeSignalFrame { + [key: string]: number | undefined; + ptsTime?: number; + YMIN?: number; + YLOW?: number; + YAVG?: number; + YHIGH?: number; + YMAX?: number; + UAVG?: number; + VAVG?: number; + SATAVG?: number; +} + +export interface GradeMediaProbe { + duration: number | null; + colorSpace: string; + transfer: string; + primaries: string; + pixelFormat: string; +} + +export interface MediaGradeAdjust { + exposure: number; + contrast: number; + whites: number; + blacks: number; + temperature: number; + tint: number; +} + +export interface MediaTreatmentMeasurements { + frames: number; + yMin: number; + yLow: number; + yAvg: number; + yHigh: number; + yMax: number; + uAvg: number; + vAvg: number; + satAvg: number; + shadowClipRisk: number; + highlightClipRisk: number; +} + +export interface MediaTreatmentAnalysis { + adjust: MediaGradeAdjust; + measured: MediaTreatmentMeasurements; + source: { + colorSpace: string; + transfer: string; + primaries: string; + pixelFormat: string; + hdr: boolean; + log: "unknown"; + }; + diagnosis: string[]; + warnings: string[]; +} + +type AdjustKey = keyof MediaGradeAdjust; +type NumericStats = Readonly>; + +const ADJUST_LIMITS: Record = { + exposure: { min: -2, max: 2 }, + contrast: { min: -1, max: 1 }, + whites: { min: -1, max: 1 }, + blacks: { min: -1, max: 1 }, + temperature: { min: -1, max: 1 }, + tint: { min: -1, max: 1 }, +}; + +function clamp(value: number, key: AdjustKey): number { + const limit = ADJUST_LIMITS[key]; + if (!Number.isFinite(value)) return 0; + return Math.min(limit.max, Math.max(limit.min, value)); +} + +function round(value: number): number { + const rounded = Math.round(value * 1000) / 1000; + return Object.is(rounded, -0) ? 0 : rounded; +} + +function average(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function asRecord(value: unknown): Record { + return isRecord(value) ? value : {}; +} + +function probeMedia(mediaPath: string, ffprobePath: string): GradeMediaProbe { + try { + const raw = execFileSync( + ffprobePath, + [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration", + "-of", + "json", + mediaPath, + ], + { encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] }, + ); + const parsed = asRecord(JSON.parse(raw)); + const streams = Array.isArray(parsed.streams) ? parsed.streams : []; + const stream = asRecord(streams[0]); + const format = asRecord(parsed.format); + const duration = Number(stream.duration ?? format.duration); + const text = (key: string) => { + const value = stream[key]; + return typeof value === "string" && value ? value : "unknown"; + }; + return { + duration: Number.isFinite(duration) && duration > 0 ? duration : null, + colorSpace: text("color_space"), + transfer: text("color_transfer"), + primaries: text("color_primaries"), + pixelFormat: text("pix_fmt"), + }; + } catch { + return { + duration: null, + colorSpace: "unknown", + transfer: "unknown", + primaries: "unknown", + pixelFormat: "unknown", + }; + } +} + +export function parseMediaTreatmentSignalStats(raw: string): GradeSignalFrame[] { + const frames: GradeSignalFrame[] = []; + let current: GradeSignalFrame | null = null; + for (const line of raw.split(/\r?\n/)) { + const frame = line.match(/^frame:\d+.*pts_time:([+-]?(?:\d+(?:\.\d+)?|\.\d+))/); + if (frame?.[1]) { + if (current) frames.push(current); + current = { ptsTime: Number(frame[1]) }; + continue; + } + const stat = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/); + if (!stat?.[1] || !stat[2]) continue; + current ??= {}; + current[stat[1]] = Number(stat[2]); + } + if (current) frames.push(current); + return frames.filter( + (frame) => + Number.isFinite(frame.YMIN) && + Number.isFinite(frame.YLOW) && + Number.isFinite(frame.YAVG) && + Number.isFinite(frame.YHIGH) && + Number.isFinite(frame.YMAX) && + Number.isFinite(frame.UAVG) && + Number.isFinite(frame.VAVG), + ); +} + +function summarizeFrames(frames: readonly GradeSignalFrame[]): NumericStats { + if (frames.length === 0) throw new Error("FFmpeg returned no analyzable video frames"); + const values = (key: string) => frames.map((frame) => Number(frame[key])); + return { + frames: frames.length, + yMin: Math.min(...values("YMIN")), + yLow: average(values("YLOW")), + yAvg: average(values("YAVG")), + yHigh: average(values("YHIGH")), + yMax: Math.max(...values("YMAX")), + uAvg: average(values("UAVG")), + vAvg: average(values("VAVG")), + satAvg: average(frames.map((frame) => frame.SATAVG ?? 0)), + shadowClipRisk: average(frames.map((frame) => (Number(frame.YLOW) <= 16 ? 1 : 0))), + highlightClipRisk: average(frames.map((frame) => (Number(frame.YHIGH) >= 235 ? 1 : 0))), + }; +} + +function suggestedExposure(normalizedAverage: number, yLow: number, yHigh: number): number { + if (normalizedAverage < 0.28 && yHigh / 255 < 0.65) { + return clamp((0.32 - normalizedAverage) * 1.2, "exposure"); + } + if (normalizedAverage > 0.72 && yLow / 255 > 0.3) { + return clamp((0.68 - normalizedAverage) * 1.2, "exposure"); + } + return 0; +} + +function suggestedTemperature(uAverage: number, vAverage: number): number { + const chromaWarmth = (vAverage - 128 + (128 - uAverage)) / 128; + return Math.abs(chromaWarmth) >= 0.08 ? clamp(-chromaWarmth * 0.25, "temperature") : 0; +} + +function suggestedTint(uAverage: number, vAverage: number): number { + const cast = uAverage + vAverage - 256; + return Math.abs(cast) >= 10 ? clamp(-cast / 512, "tint") : 0; +} + +export function statsToAdjust( + stats: NumericStats, +): Pick { + const yMin = Number(stats.yMin); + const yLow = Number(stats.yLow ?? stats.yMin); + const yMax = Number(stats.yMax); + const yHigh = Number(stats.yHigh ?? stats.yMax); + const yAvg = Number(stats.yAvg); + const uAvg = Number(stats.uAvg); + const vAvg = Number(stats.vAvg); + const shadowClipRisk = Number(stats.shadowClipRisk ?? (yLow <= 16 ? 1 : 0)); + const highlightClipRisk = Number(stats.highlightClipRisk ?? (yHigh >= 235 ? 1 : 0)); + const percentileSpread = (yHigh - yLow) / 255; + const normalizedAverage = yAvg / 255; + const contrast = percentileSpread < 0.35 ? clamp((0.35 - percentileSpread) * 0.4, "contrast") : 0; + + return { + adjust: { + exposure: round(suggestedExposure(normalizedAverage, yLow, yHigh)), + contrast: round(contrast), + blacks: round(clamp(shadowClipRisk * 0.08, "blacks")), + whites: round(clamp(-highlightClipRisk * 0.08, "whites")), + temperature: round(suggestedTemperature(uAvg, vAvg)), + tint: round(suggestedTint(uAvg, vAvg)), + }, + measured: { + frames: Number(stats.frames ?? 1), + yMin: round(yMin), + yLow: round(yLow), + yAvg: round(yAvg), + yHigh: round(yHigh), + yMax: round(yMax), + uAvg: round(uAvg), + vAvg: round(vAvg), + satAvg: round(Number(stats.satAvg ?? 0)), + shadowClipRisk: round(shadowClipRisk), + highlightClipRisk: round(highlightClipRisk), + }, + }; +} + +export function summarizeMediaTreatmentAnalysis( + probe: GradeMediaProbe, + frames: readonly GradeSignalFrame[], +): MediaTreatmentAnalysis { + const result = statsToAdjust(summarizeFrames(frames)); + const hdr = ["smpte2084", "arib-std-b67"].includes(probe.transfer); + const diagnosis: string[] = []; + if (result.measured.shadowClipRisk > 0) { + diagnosis.push("sampled frames contain deep or clipped shadows"); + } + if (result.measured.highlightClipRisk > 0) { + diagnosis.push("sampled frames contain bright or clipped highlights"); + } + if (diagnosis.length === 0) diagnosis.push("no obvious technical imbalance in sampled frames"); + + const warnings: string[] = []; + if (hdr) warnings.push("HDR transfer detected; the realtime treatment path is SDR/Rec.709."); + if (probe.colorSpace === "unknown" || probe.transfer === "unknown") { + warnings.push( + "Source color metadata is incomplete; camera LOG cannot be identified reliably from container metadata alone.", + ); + } + + return { + ...result, + source: { + colorSpace: probe.colorSpace, + transfer: probe.transfer, + primaries: probe.primaries, + pixelFormat: probe.pixelFormat, + hdr, + log: "unknown", + }, + diagnosis, + warnings, + }; +} + +export function analyzeMediaGrade( + mediaPath: string, + options: { ffmpegPath?: string; ffprobePath?: string } = {}, +): MediaTreatmentAnalysis { + const ffmpegPath = options.ffmpegPath ?? "ffmpeg"; + const ffprobePath = options.ffprobePath ?? "ffprobe"; + try { + const probe = probeMedia(mediaPath, ffprobePath); + const isImage = IMAGE_EXTENSIONS.has(extname(mediaPath).toLowerCase()); + const fps = + !isImage && probe.duration + ? Math.max(0.1, Math.min(2, SAMPLE_FRAMES / probe.duration)) + : null; + const filters = [ + fps ? `fps=${fps.toFixed(4)}` : null, + "format=yuv444p", + "signalstats", + "metadata=print:file=-", + ] + .filter(Boolean) + .join(","); + const raw = execFileSync( + ffmpegPath, + [ + "-hide_banner", + "-nostdin", + "-v", + "error", + "-i", + mediaPath, + "-vf", + filters, + "-frames:v", + String(SAMPLE_FRAMES), + "-f", + "null", + "-", + ], + { + encoding: "utf8", + timeout: Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return summarizeMediaTreatmentAnalysis(probe, parseMediaTreatmentSignalStats(raw)); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`grade analysis failed for ${mediaPath}: ${message}`); + } +} + +export function formatMeasuredNote( + mediaPath: string, + measured: MediaTreatmentMeasurements, +): string { + return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YLOW=${measured.yLow}, YAVG=${measured.yAvg}, YHIGH=${measured.yHigh}, YMAX=${measured.yMax}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`; +} diff --git a/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts b/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts new file mode 100644 index 0000000000..7f0c55f693 --- /dev/null +++ b/packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts @@ -0,0 +1,100 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; +import { + formatMeasuredNote, + parseMediaTreatmentSignalStats, + statsToAdjust, + summarizeMediaTreatmentAnalysis, + type GradeMediaProbe, +} from "./mediaGradeAnalyzer"; + +const SIGNALSTATS = `frame:0 pts:0 pts_time:0 +lavfi.signalstats.YMIN=4 +lavfi.signalstats.YLOW=8 +lavfi.signalstats.YAVG=100 +lavfi.signalstats.YHIGH=240 +lavfi.signalstats.YMAX=250 +lavfi.signalstats.UAVG=120 +lavfi.signalstats.VAVG=140 +lavfi.signalstats.SATAVG=40 +frame:1 pts:1 pts_time:1 +lavfi.signalstats.YMIN=20 +lavfi.signalstats.YLOW=20 +lavfi.signalstats.YAVG=120 +lavfi.signalstats.YHIGH=220 +lavfi.signalstats.YMAX=230 +lavfi.signalstats.UAVG=125 +lavfi.signalstats.VAVG=135 +lavfi.signalstats.SATAVG=60`; + +const UNDEREXPOSED_SIGNALSTATS = `frame:0 pts:0 pts_time:0 +lavfi.signalstats.YMIN=2 +lavfi.signalstats.YLOW=8 +lavfi.signalstats.YAVG=50 +lavfi.signalstats.YHIGH=120 +lavfi.signalstats.YMAX=135 +lavfi.signalstats.UAVG=128 +lavfi.signalstats.VAVG=128 +lavfi.signalstats.SATAVG=30`; + +const BLOWN_HIGHLIGHT_SIGNALSTATS = `frame:0 pts:0 pts_time:0 +lavfi.signalstats.YMIN=70 +lavfi.signalstats.YLOW=90 +lavfi.signalstats.YAVG=200 +lavfi.signalstats.YHIGH=248 +lavfi.signalstats.YMAX=255 +lavfi.signalstats.UAVG=128 +lavfi.signalstats.VAVG=128 +lavfi.signalstats.SATAVG=30`; + +const PROBE: GradeMediaProbe = { + duration: 4, + colorSpace: "bt2020nc", + transfer: "smpte2084", + primaries: "bt2020", + pixelFormat: "yuv420p10le", +}; + +function productResult(signalStats = SIGNALSTATS) { + const frames = parseMediaTreatmentSignalStats(signalStats); + const summary = summarizeMediaTreatmentAnalysis(PROBE, frames); + return { + frames, + summary, + adjust: statsToAdjust(summary.measured), + note: formatMeasuredNote("/tmp/frame.png", summary.measured), + }; +} + +async function vendoredResult(signalStats = SIGNALSTATS) { + const moduleUrl = new URL( + "../../../skills/media-use/scripts/lib/grade-analyzer.mjs", + import.meta.url, + ); + const analyzer = await import(moduleUrl.href); + const frames = analyzer.parseMediaTreatmentSignalStats(signalStats); + const summary = analyzer.summarizeMediaTreatmentAnalysis(PROBE, frames); + return { + frames, + summary, + adjust: analyzer.statsToAdjust(summary.measured), + note: analyzer.formatMeasuredNote("/tmp/frame.png", summary.measured), + }; +} + +describe("vendored media grade analyzer parity", () => { + it("keeps the standalone skill copy aligned with the typed product module", async () => { + expect(await vendoredResult()).toEqual(productResult()); + }); + + it.each([ + ["underexposed", UNDEREXPOSED_SIGNALSTATS, 1], + ["blown-highlight", BLOWN_HIGHLIGHT_SIGNALSTATS, -1], + ])("covers the %s exposure branch", async (_, signalStats, expectedSign) => { + const product = productResult(signalStats); + + expect(Math.sign(product.adjust.adjust.exposure)).toBe(expectedSign); + expect(await vendoredResult(signalStats)).toEqual(product); + }); +}); diff --git a/skills-manifest.json b/skills-manifest.json index eaf2d70205..4cb98f52e0 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -46,7 +46,7 @@ "files": 10 }, "media-use": { - "hash": "470beb651f5a5068", + "hash": "68500ab488f7a5ef", "files": 151 }, "motion-graphics": { diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index da79f530ec..5655ccb439 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -76,7 +76,7 @@ Surface an opportunity only when a concrete signal is present: | Image that is a placeholder, tiny, or upscaled-looking | a better `image` (and/or upscale — see `references/operations.md`) | | Hard scene cuts / transitions with no sound | transition `sfx` | | A piece over ~10s with no music bed | `bgm` | -| Footage that reads under/over-exposed or color-cast | a corrective `grade` (inspect with `grade --for --analyze`, preview with `hyperframes grade-compare`) | +| Footage that reads under/over-exposed or color-cast | a corrective grade (inspect it with `hyperframes media-treatment --selector '#hero' --analyze --json`) | | Photographic media that feels visually flat or off-topic | one specific source-appropriate preset or custom treatment, with the intended target named | | A meaningful media entrance/reveal that feels static | one supported seek-safe treatment animation; preserve color unless the request also justifies a preset | diff --git a/skills/media-use/references/grading.md b/skills/media-use/references/grading.md index 1db04795f7..2292eef785 100644 --- a/skills/media-use/references/grading.md +++ b/skills/media-use/references/grading.md @@ -61,11 +61,12 @@ canonical toolbox first: hyperframes media-treatment --capabilities --json ``` -It reports valid adjustment, finishing, effect, palette, preset, and animated -property ranges from Core. Compose one nested payload and pass it back through -`hyperframes media-treatment`; the command rejects unknown keys before mutation. -Do not generate or hand-edit a LUT merely to combine controls already owned by -the realtime shader. +It reports a concise family map. Read `--capability grading` for the processing +order, then request only the focused family needed to get its legal controls +and ranges from Core. Compose one nested payload and pass it back through +`hyperframes media-treatment`; the command rejects unknown keys before +mutation. Do not generate or hand-edit a LUT merely to combine controls already +owned by the realtime shader. For seek-safe effect motion, animate only the runtime-supported CSS properties on that same real media element with its registered paused GSAP timeline: @@ -131,18 +132,20 @@ For visual selection, list reusable LUT candidates with `hyperframes grade-compare --for --grades grades.json`, then commit the winner with `resolve -t grade` as the final `data-color-grading` block. -Use `grade --for --analyze` when you only need side-effect-free -`ffmpeg`/`ffprobe` signalstats evidence. It returns a bounded `adjust` -suggestion without writing a manifest record. The suggestion is a starting +For media already selected in a composition, use `media-treatment --analyze` +when you need side-effect-free `ffmpeg`/`ffprobe` signalstats evidence. It +returns source metadata, HDR/unknown-LOG warnings, and a bounded `adjust` +suggestion without modifying the composition. The suggestion is a starting point for visual review, not an automatic neutralization of intentional color. ```bash -node /scripts/resolve.mjs --type grade --for ./frame.png --analyze --project . --json +hyperframes media-treatment --project . --file index.html \ + --selector '#hero' --analyze --json ``` -Without `--analyze`, `grade --for` merges measured values into the resolved -grade and records that candidate in `.media`; use that form only when you -intend to keep the candidate. +For an unbound source file, `resolve --type grade --for ... --analyze` remains +available. Without `--analyze`, that resolver records a grade candidate in +`.media`; use that form only when you intend to keep the candidate. Library LUT entries live in `luts/index.json`. Each entry keeps `id`, `description`, `tags`, and `intensity`, then supplies either compact `params` diff --git a/skills/media-use/references/media-treatments.md b/skills/media-use/references/media-treatments.md index b8dc0d0045..6651b28049 100644 --- a/skills/media-use/references/media-treatments.md +++ b/skills/media-use/references/media-treatments.md @@ -30,6 +30,7 @@ or assembling a custom treatment: | User intent | Lane | | --------------------------------------------------- | ------------------------------------------ | | too dark, flat, too warm, too many shadows | correction | +| shape shadows/highlights or selected colors | wheels, curves, or HSL secondary | | polished, premium, warm, cinematic, fit the topic | preset or custom treatment | | retro, print, ASCII, glitch, camcorder | shader effect or effect-bearing preset | | obscure the whole selected media | privacy Blur or Pixelate | @@ -151,8 +152,8 @@ drops frames. Do not impose or claim a universal hard cap from one machine. contact sheet justifies it. Do not invent keys, exceed reported ranges, or stack effects without a visual reason. Do not run the generic grade/LUT resolver first; it adds irrelevant candidates and may download an unused - LUT. Use `grade --for --analyze` only when correction needs measured signal - evidence. + LUT. Use `media-treatment --selector "#hero" --analyze --json` only when + correction needs measured signal evidence. 4. Persist pixel settings with `hyperframes media-treatment`; it validates and merges a patch into the existing nested `data-color-grading` contract. Use registered GSAP only for supported animated values and Registry overlay blocks only @@ -195,5 +196,7 @@ drops frames. Do not impose or claim a universal hard cap from one machine. composed controls, optional overlays, and the frames/render that were actually checked. Do not report visual quality from command success alone. -`resolve --type grade --for ... --analyze --json` provides deterministic -clipping and signal evidence, not subject recognition or automatic taste. +`media-treatment --analyze` provides deterministic clipping and signal +evidence for local composition media, not subject recognition or automatic +taste. It reports HDR/metadata caveats and a bounded primary-correction patch; +it does not invent wheels, curves, or HSL selections from statistics. diff --git a/skills/media-use/references/meta.md b/skills/media-use/references/meta.md index 668d4f35b4..776d1ed5d5 100644 --- a/skills/media-use/references/meta.md +++ b/skills/media-use/references/meta.md @@ -16,7 +16,7 @@ HyperFrames owns media _playback_; media-use owns everything else. Each row is e | No transcript-driven cutting | `scripts/transcript-cut.mjs` compiles word-timestamp edits into cut lists | | No auto-duck / publish loudness | `scripts/audio-duck.mjs` + `references/operations.md` loudnorm/sidechain recipes | | No cross-project memory | global content-addressed cache + auto-promote (`~/.media`) | -| No color-grade authoring | `resolve --type grade` emits a paste-ready `data-color-grading` block; `resolve --type lut` freezes validated `.cube` files | +| Grade recipes and LUT freezing | `resolve --type grade` emits a paste-ready recipe and `resolve --type lut` freezes validated `.cube` files; direct element analysis/authoring lives in `hyperframes media-treatment` | | No image generation | RAM-graded local mflux (FLUX) via `scripts/lib/mflux-provider.mjs`, codex `image_gen` upsell (`scripts/lib/codex-provider.mjs`) | | No video generation | `resolve --type video` — HeyGen avatar video first (free-usage path, sign-in nudge on auth failure), local LTX fallback (`videogen` in `scripts/lib/local-models.mjs`); image-to-video, photo-avatar, dub/translate remain manual `heygen` CLI recipes (`references/operations.md`) | | Weak local-model defaults | HeyGen free-usage path via the `heygen` CLI; local open-source tools only as opt-in alternatives (`scripts/lib/local-run.mjs`) | diff --git a/skills/media-use/scripts/lib/grade-analyzer.mjs b/skills/media-use/scripts/lib/grade-analyzer.mjs index 054e5119bf..b673a65a58 100644 --- a/skills/media-use/scripts/lib/grade-analyzer.mjs +++ b/skills/media-use/scripts/lib/grade-analyzer.mjs @@ -1,23 +1,19 @@ +// Vendored plain-JS copy of packages/core/src/mediaGradeAnalyzer.ts. +// packages/core/src/mediaGradeAnalyzer.vendoredParity.test.ts guards behavior drift. import { execFileSync } from "node:child_process"; import { basename, extname } from "node:path"; const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff"]); const SAMPLE_FRAMES = 5; -// A long HD clip on slow storage can exceed the default 15s signalstats window; -// override without a code change via HYPERFRAMES_ANALYZE_TIMEOUT_MS. -const SIGNALSTATS_TIMEOUT_MS = Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || 15000; +const DEFAULT_TIMEOUT_MS = 15_000; const ADJUST_LIMITS = { exposure: { min: -2, max: 2 }, contrast: { min: -1, max: 1 }, - highlights: { min: -1, max: 1 }, - shadows: { min: -1, max: 1 }, whites: { min: -1, max: 1 }, blacks: { min: -1, max: 1 }, temperature: { min: -1, max: 1 }, tint: { min: -1, max: 1 }, - vibrance: { min: -1, max: 1 }, - saturation: { min: -1, max: 1 }, }; function clamp(value, key) { @@ -27,116 +23,214 @@ function clamp(value, key) { } function round(value) { - return Math.round(value * 1000) / 1000; + const rounded = Math.round(value * 1000) / 1000; + return Object.is(rounded, -0) ? 0 : rounded; } -function avg(values) { - if (values.length === 0) return 0; - return values.reduce((sum, value) => sum + value, 0) / values.length; +function average(values) { + return values.reduce((sum, value) => sum + value, 0) / Math.max(1, values.length); } -function probeDuration(mediaPath) { +function probeMedia(mediaPath, ffprobePath) { try { const raw = execFileSync( - "ffprobe", - ["-v", "quiet", "-print_format", "json", "-show_format", mediaPath], - { encoding: "utf8", timeout: 5000 }, + ffprobePath, + [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration", + "-of", + "json", + mediaPath, + ], + { encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] }, ); const parsed = JSON.parse(raw); - const duration = Number(parsed.format?.duration); - return Number.isFinite(duration) && duration > 0 ? duration : null; + const stream = Array.isArray(parsed.streams) ? (parsed.streams[0] ?? {}) : {}; + const duration = Number(stream.duration ?? parsed.format?.duration); + const text = (key) => + typeof stream[key] === "string" && stream[key] ? stream[key] : "unknown"; + return { + duration: Number.isFinite(duration) && duration > 0 ? duration : null, + colorSpace: text("color_space"), + transfer: text("color_transfer"), + primaries: text("color_primaries"), + pixelFormat: text("pix_fmt"), + }; } catch { - return null; + return { + duration: null, + colorSpace: "unknown", + transfer: "unknown", + primaries: "unknown", + pixelFormat: "unknown", + }; } } -function filterFor(mediaPath) { - const ext = extname(mediaPath).toLowerCase(); - if (IMAGE_EXT.has(ext)) return "signalstats,metadata=print:file=-"; - const duration = probeDuration(mediaPath); - if (!duration || duration <= 1) return "signalstats,metadata=print:file=-"; - const fps = Math.max(0.1, Math.min(2, SAMPLE_FRAMES / duration)); - return `fps=${fps.toFixed(4)},signalstats,metadata=print:file=-`; -} - -function parseSignalStats(raw) { +export function parseMediaTreatmentSignalStats(raw) { const frames = []; let current = null; for (const line of String(raw).split(/\r?\n/)) { - const frameMatch = line.match(/^frame:/); - if (frameMatch) { + const frame = line.match(/^frame:\d+.*pts_time:([+-]?(?:\d+(?:\.\d+)?|\.\d+))/); + if (frame) { if (current) frames.push(current); - current = {}; + current = { ptsTime: Number(frame[1]) }; continue; } - const match = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/); - if (!match) continue; - if (!current) current = {}; - current[match[1]] = Number(match[2]); + const stat = line.match(/lavfi\.signalstats\.([A-Z]+)=([+-]?(?:\d+(?:\.\d+)?|\.\d+))/); + if (!stat) continue; + current ??= {}; + current[stat[1]] = Number(stat[2]); } if (current) frames.push(current); - const complete = frames.filter( + return frames.filter( (frame) => Number.isFinite(frame.YMIN) && - Number.isFinite(frame.YMAX) && + Number.isFinite(frame.YLOW) && Number.isFinite(frame.YAVG) && + Number.isFinite(frame.YHIGH) && + Number.isFinite(frame.YMAX) && Number.isFinite(frame.UAVG) && Number.isFinite(frame.VAVG), ); - if (complete.length === 0) { - throw new Error("no signalstats frames found"); - } +} + +function summarizeFrames(frames) { + if (frames.length === 0) throw new Error("FFmpeg returned no analyzable video frames"); return { - frames: complete.length, - yMin: Math.min(...complete.map((frame) => frame.YMIN)), - yMax: Math.max(...complete.map((frame) => frame.YMAX)), - yAvg: avg(complete.map((frame) => frame.YAVG)), - uAvg: avg(complete.map((frame) => frame.UAVG)), - vAvg: avg(complete.map((frame) => frame.VAVG)), + frames: frames.length, + yMin: Math.min(...frames.map((frame) => frame.YMIN)), + yLow: average(frames.map((frame) => frame.YLOW)), + yAvg: average(frames.map((frame) => frame.YAVG)), + yHigh: average(frames.map((frame) => frame.YHIGH)), + yMax: Math.max(...frames.map((frame) => frame.YMAX)), + uAvg: average(frames.map((frame) => frame.UAVG)), + vAvg: average(frames.map((frame) => frame.VAVG)), + satAvg: average(frames.map((frame) => frame.SATAVG ?? 0)), + shadowClipRisk: average(frames.map((frame) => (frame.YLOW <= 16 ? 1 : 0))), + highlightClipRisk: average(frames.map((frame) => (frame.YHIGH >= 235 ? 1 : 0))), }; } +function suggestedExposure(normalizedAverage, yLow, yHigh) { + if (normalizedAverage < 0.28 && yHigh / 255 < 0.65) { + return clamp((0.32 - normalizedAverage) * 1.2, "exposure"); + } + if (normalizedAverage > 0.72 && yLow / 255 > 0.3) { + return clamp((0.68 - normalizedAverage) * 1.2, "exposure"); + } + return 0; +} + +function suggestedTemperature(uAverage, vAverage) { + const chromaWarmth = (vAverage - 128 + (128 - uAverage)) / 128; + return Math.abs(chromaWarmth) >= 0.08 ? clamp(-chromaWarmth * 0.25, "temperature") : 0; +} + +function suggestedTint(uAverage, vAverage) { + const cast = uAverage + vAverage - 256; + return Math.abs(cast) >= 10 ? clamp(-cast / 512, "tint") : 0; +} + export function statsToAdjust(stats) { const yMin = Number(stats.yMin); + const yLow = Number(stats.yLow ?? stats.yMin); const yMax = Number(stats.yMax); + const yHigh = Number(stats.yHigh ?? stats.yMax); const yAvg = Number(stats.yAvg); const uAvg = Number(stats.uAvg); const vAvg = Number(stats.vAvg); - const spread = (yMax - yMin) / 255; - const normalizedAvg = yAvg / 255; - const exposure = clamp((0.45 - normalizedAvg) * 1.8, "exposure"); - const contrast = clamp((0.42 - spread) * 0.9, "contrast"); - const whites = - yMax > 230 ? clamp(-((yMax - 230) / 40 + Math.max(0, normalizedAvg - 0.74)), "whites") : 0; - const blacks = yMin < 12 ? clamp((12 - yMin) / 80, "blacks") : 0; - const chromaWarmth = (vAvg - 128 + (128 - uAvg)) / 128; - const temperature = clamp(-chromaWarmth * 0.7, "temperature"); - const tint = clamp(-(uAvg - 128 + (vAvg - 128)) / 256, "tint"); + const shadowClipRisk = Number(stats.shadowClipRisk ?? (yLow <= 16 ? 1 : 0)); + const highlightClipRisk = Number(stats.highlightClipRisk ?? (yHigh >= 235 ? 1 : 0)); + const percentileSpread = (yHigh - yLow) / 255; + const normalizedAverage = yAvg / 255; + const contrast = percentileSpread < 0.35 ? clamp((0.35 - percentileSpread) * 0.4, "contrast") : 0; return { adjust: { - exposure: round(exposure), + exposure: round(suggestedExposure(normalizedAverage, yLow, yHigh)), contrast: round(contrast), - blacks: round(blacks), - whites: round(whites), - temperature: round(temperature), - tint: round(tint), + blacks: round(clamp(shadowClipRisk * 0.08, "blacks")), + whites: round(clamp(-highlightClipRisk * 0.08, "whites")), + temperature: round(suggestedTemperature(uAvg, vAvg)), + tint: round(suggestedTint(uAvg, vAvg)), }, measured: { frames: Number(stats.frames ?? 1), yMin: round(yMin), - yMax: round(yMax), + yLow: round(yLow), yAvg: round(yAvg), + yHigh: round(yHigh), + yMax: round(yMax), uAvg: round(uAvg), vAvg: round(vAvg), + satAvg: round(Number(stats.satAvg ?? 0)), + shadowClipRisk: round(shadowClipRisk), + highlightClipRisk: round(highlightClipRisk), + }, + }; +} + +export function summarizeMediaTreatmentAnalysis(probe, frames) { + const result = statsToAdjust(summarizeFrames(frames)); + const hdr = ["smpte2084", "arib-std-b67"].includes(probe.transfer); + const diagnosis = []; + if (result.measured.shadowClipRisk > 0) { + diagnosis.push("sampled frames contain deep or clipped shadows"); + } + if (result.measured.highlightClipRisk > 0) { + diagnosis.push("sampled frames contain bright or clipped highlights"); + } + if (diagnosis.length === 0) diagnosis.push("no obvious technical imbalance in sampled frames"); + + const warnings = []; + if (hdr) warnings.push("HDR transfer detected; the realtime treatment path is SDR/Rec.709."); + if (probe.colorSpace === "unknown" || probe.transfer === "unknown") { + warnings.push( + "Source color metadata is incomplete; camera LOG cannot be identified reliably from container metadata alone.", + ); + } + + return { + ...result, + source: { + colorSpace: probe.colorSpace, + transfer: probe.transfer, + primaries: probe.primaries, + pixelFormat: probe.pixelFormat, + hdr, + log: "unknown", }, + diagnosis, + warnings, }; } -export function analyzeMediaGrade(mediaPath) { +export function analyzeMediaGrade( + mediaPath, + { ffmpegPath = "ffmpeg", ffprobePath = "ffprobe" } = {}, +) { try { + const probe = probeMedia(mediaPath, ffprobePath); + const isImage = IMAGE_EXT.has(extname(mediaPath).toLowerCase()); + const fps = + !isImage && probe.duration + ? Math.max(0.1, Math.min(2, SAMPLE_FRAMES / probe.duration)) + : null; + const filters = [ + fps ? `fps=${fps.toFixed(4)}` : null, + "format=yuv444p", + "signalstats", + "metadata=print:file=-", + ] + .filter(Boolean) + .join(","); const raw = execFileSync( - "ffmpeg", + ffmpegPath, [ "-hide_banner", "-nostdin", @@ -145,21 +239,25 @@ export function analyzeMediaGrade(mediaPath) { "-i", mediaPath, "-vf", - filterFor(mediaPath), + filters, "-frames:v", String(SAMPLE_FRAMES), "-f", "null", "-", ], - { encoding: "utf8", timeout: SIGNALSTATS_TIMEOUT_MS, stdio: ["ignore", "pipe", "pipe"] }, + { + encoding: "utf8", + timeout: Number(process.env.HYPERFRAMES_ANALYZE_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS, + stdio: ["ignore", "pipe", "pipe"], + }, ); - return statsToAdjust(parseSignalStats(raw)); - } catch (err) { - throw new Error(`grade analysis failed for ${mediaPath}: ${err.message}`); + return summarizeMediaTreatmentAnalysis(probe, parseMediaTreatmentSignalStats(raw)); + } catch (error) { + throw new Error(`grade analysis failed for ${mediaPath}: ${error.message}`); } } export function formatMeasuredNote(mediaPath, measured) { - return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YMAX=${measured.yMax}, YAVG=${measured.yAvg}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`; + return `media-use: measured ${basename(mediaPath)}: frames=${measured.frames}, YMIN=${measured.yMin}, YLOW=${measured.yLow}, YAVG=${measured.yAvg}, YHIGH=${measured.yHigh}, YMAX=${measured.yMax}, UAVG=${measured.uAvg}, VAVG=${measured.vAvg}; adjust is a starting suggestion`; } diff --git a/skills/media-use/scripts/lib/grade-analyzer.test.mjs b/skills/media-use/scripts/lib/grade-analyzer.test.mjs index 86e6632a62..0a649029d6 100644 --- a/skills/media-use/scripts/lib/grade-analyzer.test.mjs +++ b/skills/media-use/scripts/lib/grade-analyzer.test.mjs @@ -4,7 +4,32 @@ import { existsSync, mkdtempSync, rmSync, unlinkSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { test } from "node:test"; -import { analyzeMediaGrade, formatMeasuredNote, statsToAdjust } from "./grade-analyzer.mjs"; +import { + analyzeMediaGrade, + formatMeasuredNote, + parseMediaTreatmentSignalStats, + statsToAdjust, + summarizeMediaTreatmentAnalysis, +} from "./grade-analyzer.mjs"; + +const SIGNALSTATS = `frame:0 pts:0 pts_time:0 +lavfi.signalstats.YMIN=0 +lavfi.signalstats.YLOW=8 +lavfi.signalstats.YAVG=100 +lavfi.signalstats.YHIGH=240 +lavfi.signalstats.YMAX=255 +lavfi.signalstats.UAVG=120 +lavfi.signalstats.VAVG=140 +lavfi.signalstats.SATAVG=40 +frame:1 pts:2000 pts_time:2 +lavfi.signalstats.YMIN=10 +lavfi.signalstats.YLOW=20 +lavfi.signalstats.YAVG=130 +lavfi.signalstats.YHIGH=220 +lavfi.signalstats.YMAX=245 +lavfi.signalstats.UAVG=130 +lavfi.signalstats.VAVG=125 +lavfi.signalstats.SATAVG=60`; // The "Test: skills" CI job runs bare `node --test` with no ffmpeg on PATH (by // design — skills tests are meant to be node-builtin-only). Tests that shell to @@ -70,18 +95,22 @@ test("under-exposed synthetic frame suggests positive exposure", { skip: FFMPEG_ } }); -test("over-exposed synthetic frame pulls exposure and whites down", { skip: FFMPEG_SKIP }, () => { - const dir = mkdtempSync(join(tmpdir(), "mu-grade-over-")); - try { - const file = makeFrame(dir, "over.png", "white"); - const { adjust } = analyzeMediaGrade(file); - assert.ok(adjust.exposure < 0, `expected negative exposure, got ${adjust.exposure}`); - assert.ok(adjust.whites < 0, `expected negative whites, got ${adjust.whites}`); - assertWithinLimits(adjust); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -}); +test( + "over-exposed synthetic frame pulls exposure down without inventing clipping", + { skip: FFMPEG_SKIP }, + () => { + const dir = mkdtempSync(join(tmpdir(), "mu-grade-over-")); + try { + const file = makeFrame(dir, "over.png", "white"); + const { adjust } = analyzeMediaGrade(file); + assert.ok(adjust.exposure < 0, `expected negative exposure, got ${adjust.exposure}`); + assert.ok(adjust.whites <= 0, `expected non-positive whites, got ${adjust.whites}`); + assertWithinLimits(adjust); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, +); test( "warm-cast synthetic frame suggests negative temperature correction", @@ -150,6 +179,107 @@ test("measured note is a stderr-safe single-line summary", () => { vAvg: 140, }); assert.match(note, /^media-use: measured /); + assert.match(note, /YMIN=10/); + assert.match(note, /YMAX=240/); assert.match(note, /YAVG=80/); assert.equal(note.includes("\n"), false); }); + +test("parses deterministic FFmpeg signalstats frames", () => { + const frames = parseMediaTreatmentSignalStats(SIGNALSTATS); + assert.equal(frames.length, 2); + assert.deepEqual( + { + ptsTime: frames[0]?.ptsTime, + YLOW: frames[0]?.YLOW, + YAVG: frames[0]?.YAVG, + YHIGH: frames[0]?.YHIGH, + SATAVG: frames[0]?.SATAVG, + }, + { ptsTime: 0, YLOW: 8, YAVG: 100, YHIGH: 240, SATAVG: 40 }, + ); +}); + +test("keeps adjust output while adding HDR metadata and warnings", () => { + const result = summarizeMediaTreatmentAnalysis( + { + duration: 4, + colorSpace: "bt2020nc", + transfer: "smpte2084", + primaries: "bt2020", + pixelFormat: "yuv420p10le", + }, + parseMediaTreatmentSignalStats(SIGNALSTATS), + ); + + assert.deepEqual( + { + blacks: result.adjust.blacks, + whites: result.adjust.whites, + frames: result.measured.frames, + yLow: result.measured.yLow, + yHigh: result.measured.yHigh, + satAvg: result.measured.satAvg, + shadowClipRisk: result.measured.shadowClipRisk, + highlightClipRisk: result.measured.highlightClipRisk, + }, + { + blacks: 0.04, + whites: -0.04, + frames: 2, + yLow: 14, + yHigh: 230, + satAvg: 50, + shadowClipRisk: 0.5, + highlightClipRisk: 0.5, + }, + ); + assert.equal(result.source.hdr, true); + assert.equal(result.source.log, "unknown"); + assert.match(result.warnings[0] ?? "", /HDR/); +}); + +test("does not force middle-gray exposure onto healthy intentional contrast", () => { + const result = summarizeMediaTreatmentAnalysis( + { + duration: 4, + colorSpace: "bt709", + transfer: "bt709", + primaries: "bt709", + pixelFormat: "yuv420p", + }, + [ + { + YMIN: 20, + YLOW: 42, + YAVG: 122, + YHIGH: 190, + YMAX: 220, + UAVG: 128, + VAVG: 128, + SATAVG: 40, + }, + { + YMIN: 18, + YLOW: 38, + YAVG: 118, + YHIGH: 196, + YMAX: 225, + UAVG: 128, + VAVG: 128, + SATAVG: 42, + }, + ], + ); + + assert.deepEqual( + { + exposure: result.adjust.exposure, + blacks: result.adjust.blacks, + whites: result.adjust.whites, + temperature: result.adjust.temperature, + tint: result.adjust.tint, + }, + { exposure: 0, blacks: 0, whites: 0, temperature: 0, tint: 0 }, + ); +});