diff --git a/scripts/import-pathway-data.test.ts b/scripts/import-pathway-data.test.ts index 96cfae48..1085f57f 100644 --- a/scripts/import-pathway-data.test.ts +++ b/scripts/import-pathway-data.test.ts @@ -18,6 +18,11 @@ import { buildDataAvailability, descriptionFromV1, keyFeaturesFromV1, + charCount, + withinLimit, + buildCoreDrivers, + buildDependencies, + buildDoc, } from "./import-pathway-data.ts"; describe("publisherGroup", () => { @@ -370,3 +375,161 @@ describe("buildDataAvailability", () => { expect(build([]).out).toBeNull(); }); }); + +describe("character limits (tpr-tpc decision 0028)", () => { + /** A string of exactly `n` characters, ending in a period. */ + const text = (n: number) => `${"a".repeat(n - 1)}.`; + const errorsOf = (fn: (report: Report) => unknown) => { + const report = new Report(); + fn(report); + return report.errors.join("\n"); + }; + + it("counts code points, as the cookbook and JSON-schema maxLength do", () => { + expect(charCount("CO₂")).toBe(3); + expect(charCount("🌍")).toBe(1); + }); + + it("reports text over the limit as an error, and none at the limit", () => { + expect(errorsOf((r) => withinLimit(text(500), 500, "T/x", r))).toBe(""); + expect(errorsOf((r) => withinLimit(text(501), 500, "T/x", r))).toBe( + "T/x: 501 chars > 500 -- shorten it in the workbook", + ); + }); + + it("errors on a core driver over 500, rather than leaving it null", () => { + expect( + errorsOf((r) => buildCoreDrivers({ Policy: text(500) }, r, "T")), + ).toBe(""); + expect( + errorsOf((r) => buildCoreDrivers({ Policy: text(501) }, r, "T")), + ).toContain("T/coreDrivers.policies: 501 chars > 500"); + }); + + it("counts the period the importer adds", () => { + expect( + errorsOf((r) => buildCoreDrivers({ Policy: "a".repeat(500) }, r, "T")), + ).toContain("T/coreDrivers.policies: 501 chars > 500"); + }); + + it("errors on a dependency over 500, rather than skipping it", () => { + const deps = (prose: string) => + errorsOf((r) => + buildDependencies( + [{ Sector: "Power", Technology: prose }], + new Set(["Power"]), + r, + "T", + ), + ); + expect(deps(text(500))).toBe(""); + expect(deps(text(501))).toContain( + 'T/dependencies "Technology" (Power): 501 chars > 500', + ); + }); + + it("errors on scope limitations over 500", () => { + const da = (scope: string) => + errorsOf((r) => + buildDataAvailability( + [ + { + "Sector": "Power", + "Metric": "Capacity", + "Sector segment": "Power generation", + "Granularity": "Coal", + "Scope limitations": scope, + "Geography coverage": "TH", + "Time resolution": "5-year steps", + "Data Format": "Tabular", + }, + ], + new Set(["Power"]), + new Set(["TH"]), + r, + "T", + ), + ); + expect(da(text(500))).toBe(""); + expect(da(text(501))).toContain( + "T/dataAvailability Power Capacity: scope limitations: 501 chars > 500", + ); + }); + + describe("buildDoc", () => { + const sub = { + keyFeatures: [], + coreDrivers: undefined, + dependencies: [], + dataAvailability: [], + }; + const meta = (over: Record = {}) => ({ + "Name of Pathway": "Current Policies Scenario (CPS)", + "Name of publication": "World Energy Outlook 2025", + "Description": "A short description.", + "Pathway Overview": "An overview.", + ...over, + }); + const newDoc = (over: Record = {}) => + errorsOf((r) => + buildDoc( + { mode: "new", file: "T.json", id: "T", templateFile: "t.json" }, + meta(over), + sub, + null, + null, + r, + ), + ); + + it("accepts every field at its limit", () => { + expect( + newDoc({ + // Cookbook format: geography suffix inside the brackets, counted in the 20. + "Name of Pathway": `${"a".repeat(177)} (${"B".repeat(17)} ID)`, + "Name of publication": "p".repeat(200), + "Description": text(100), + "Pathway Overview": text(2500), + }), + ).toBe(""); + }); + + it("errors on each field one over its limit", () => { + expect(newDoc({ "Name of Pathway": `${"a".repeat(194)} (CODE)` })).toBe( + "T/name: 201 chars > 200 -- shorten it in the workbook", + ); + expect( + newDoc({ "Name of Pathway": "Plan (Indonesia on-grid power)" }), + ).toContain( + 'T/name: short name "(Indonesia on-grid power)": 23 chars > 20', + ); + expect(newDoc({ "Name of publication": "p".repeat(201) })).toContain( + "T/publication.title: 201 chars > 200", + ); + expect(newDoc({ Description: text(101) })).toContain( + "T/description: 101 chars > 100", + ); + expect(newDoc({ "Pathway Overview": text(2501) })).toContain( + "T/pathwayDescription: 2501 chars > 2500", + ); + }); + + it("does not check the name, description or publication of an update, which keeps its own", () => { + const errors = errorsOf((r) => + buildDoc( + { mode: "update", file: "T.json" }, + meta({ + "Name of Pathway": `${"a".repeat(300)} (${"B".repeat(30)})`, + "Name of publication": "p".repeat(300), + "Description": text(150), + }), + sub, + { id: "T", name: { full: "Kept" }, description: "Kept." }, + null, + r, + ), + ); + expect(errors).toBe(""); + }); + }); +}); diff --git a/scripts/import-pathway-data.ts b/scripts/import-pathway-data.ts index 39b74fce..71abb731 100644 --- a/scripts/import-pathway-data.ts +++ b/scripts/import-pathway-data.ts @@ -14,8 +14,10 @@ * * This is a development tool, not a runtime path. It writes files in place and * prints a report of everything it could not map cleanly (unresolved scopes, - * enum/vocabulary mismatches, over-length prose) rather than emitting anything - * that would fail `npm run schema:check`. + * enum/vocabulary mismatches) rather than emitting anything that would fail + * `npm run schema:check`. Over-length prose is an authoring error that blocks + * the import: the limits are the cookbook's (tpr-tpc decision 0028), and text + * over one would otherwise be lost. * * Sheet names are parameterized (`--sheet-prefix`, default `draft_`) because the * final `pathway_*` sheets are not populated yet; switch the prefix once they are. @@ -662,6 +664,31 @@ export class Report { } } +/** + * Length in code points -- what the cookbook and JSON-schema `maxLength` count. + * `.length` counts UTF-16 units, which disagrees for e.g. emoji. + */ +export function charCount(text: string): number { + return [...text].length; +} + +/** + * Whether `text` fits in `max` characters; reports it as an authoring error + * when it does not. Over-length text is never truncated or dropped, since that + * would lose content nobody chose to lose. + */ +export function withinLimit( + text: string, + max: number, + where: string, + report: Report, +): boolean { + const n = charCount(text); + if (n <= max) return true; + report.error(`${where}: ${n} chars > ${max} -- shorten it in the workbook`); + return false; +} + // -------------------------------------------------------------------------- // Field builders. // -------------------------------------------------------------------------- @@ -908,7 +935,7 @@ function buildKeyFeatures( return fields; } -function buildCoreDrivers( +export function buildCoreDrivers( row: Row | undefined, report: Report, id: string, @@ -921,19 +948,13 @@ function buildCoreDrivers( continue; } const text = ensurePeriod(raw); - if (text.length > 500) { - report.note( - `${id}/coreDrivers.${field}: ${text.length} chars > 500 -- left null, needs trimming`, - ); - out[field] = null; - } else { - out[field] = text; - } + withinLimit(text, 500, `${id}/coreDrivers.${field}`, report); + out[field] = text; } return out; } -function buildDependencies( +export function buildDependencies( rows: Row[], declaredSectors: Set, report: Report, @@ -959,12 +980,12 @@ function buildDependencies( const prose = row[column] ?? ""; if (isAbsent(prose)) continue; const description = ensurePeriod(prose); - if (description.length > 500) { - report.note( - `${id}/dependencies "${name}" (${sector}): ${description.length} chars > 500 -- skipped`, - ); - continue; - } + withinLimit( + description, + 500, + `${id}/dependencies "${name}" (${sector})`, + report, + ); const evidence = matchEnum(row[`${column}_Evidence type`] ?? "", EVIDENCE_TYPES) ?? "No evidence"; @@ -1136,6 +1157,12 @@ export function buildDataAvailability( bad, ); const granularity = parseGranularity(row["Granularity"] ?? "", bad); + withinLimit( + scopeLimitations, + 500, + `${id}/dataAvailability ${sector} ${rawMetric || "(no metric)"}: scope limitations`, + report, + ); // Every column is required (a row for every allowable pair, per the // cookbook), so a blank or unrecognised cell is an authoring error that @@ -1146,7 +1173,6 @@ export function buildDataAvailability( !timeResolution && "time resolution", !dataFormat && "data format", !scopeLimitations && "scope limitations", - scopeLimitations.length > 500 && "scope limitations > 500 chars", sectorSegment.length === 0 && "sector segment", geography.length === 0 && "geography", granularity.length === 0 && "granularity", @@ -1317,7 +1343,7 @@ function parseTemp(raw: string): number | undefined { type Json = Record; /** Assemble the full v2 document for one pathway. */ -function buildDoc( +export function buildDoc( target: Target, meta: Row, sub: { @@ -1418,14 +1444,8 @@ function buildDoc( let pathwayDescription: string | null; const overview = (meta["Pathway Overview"] ?? "").trim(); if (overview && !isAbsent(overview)) { - const text = ensurePeriod(overview); - if (text.length <= 2500) pathwayDescription = text; - else { - report.note( - `${id}/pathwayDescription: ${text.length} chars > 2500 -- kept existing`, - ); - pathwayDescription = (existing?.pathwayDescription as string) ?? null; - } + pathwayDescription = ensurePeriod(overview); + withinLimit(pathwayDescription, 2500, `${id}/pathwayDescription`, report); } else { pathwayDescription = (existing?.pathwayDescription as string | null) ?? @@ -1460,21 +1480,20 @@ function buildDoc( let name: unknown; let description: unknown; if (target.mode === "new") { - const { code } = scenarioParts(meta["Name of Pathway"] ?? ""); - const full = (meta["Name of Pathway"] ?? "") + // Checked only here: an update keeps its existing name, description and + // publication, so those cells are not imported for it. + const pathwayName = (meta["Name of Pathway"] ?? "").trim(); + withinLimit(pathwayName, 200, `${id}/name`, report); + const { code } = scenarioParts(pathwayName); + withinLimit(code, 20, `${id}/name: short name "(${code})"`, report); + const full = pathwayName .replace(/\s*\([^)]*\)/g, "") .replace(/\s*\[[A-Z]{2}\]/g, "") .trim(); name = { full, short: code }; const desc = ensurePeriod(meta["Description"] ?? ""); - description = - desc.length <= 100 - ? desc - : (template?.description ?? desc.slice(0, 99) + "."); - if (desc.length > 100) - report.note( - `${id}/description: ${desc.length} chars > 100 -- used template description`, - ); + withinLimit(desc, 100, `${id}/description`, report); + description = desc; publication = buildPublicationForNew( template?.publication as Json, meta, @@ -1573,6 +1592,7 @@ function buildPublicationForNew( ): Json { const pub: Json = template ? JSON.parse(JSON.stringify(template)) : {}; const title = (meta["Name of publication"] ?? "").trim(); + withinLimit(title, 200, `${id}/publication.title`, report); const year = parseYear(meta["Year of Publication"] ?? ""); const oldYear = pub.year as number | undefined; if (title) pub.title = { ...(pub.title as Json), full: title };