fix(import): fail the metadata import on over-length free text

scripts/import-pathway-data.test.ts

@@ -18,6 +18,11 @@ import {
   buildDataAvailability,
   descriptionFromV1,
   keyFeaturesFromV1,
+  charCount,
+  withinLimit,
+  buildCoreDrivers,
+  buildDependencies,
+  buildDoc,
 } from "./import-pathway-data.ts";
 
 describe("publisherGroup", () => {
@@ -370,3 +375,161 @@ describe("buildDataAvailability", () => {
     expect(build([]).out).toBeNull();
   });
 });
+
+describe("character limits (tpr-tpc decision 0028)", () => {
+  /** A string of exactly `n` characters, ending in a period. */
+  const text = (n: number) => `${"a".repeat(n - 1)}.`;
+  const errorsOf = (fn: (report: Report) => unknown) => {
+    const report = new Report();
+    fn(report);
+    return report.errors.join("\n");
+  };
+
+  it("counts code points, as the cookbook and JSON-schema maxLength do", () => {
+    expect(charCount("CO₂")).toBe(3);
+    expect(charCount("🌍")).toBe(1);
+  });
+
+  it("reports text over the limit as an error, and none at the limit", () => {
+    expect(errorsOf((r) => withinLimit(text(500), 500, "T/x", r))).toBe("");
+    expect(errorsOf((r) => withinLimit(text(501), 500, "T/x", r))).toBe(
+      "T/x: 501 chars > 500 -- shorten it in the workbook",
+    );
+  });
+
+  it("errors on a core driver over 500, rather than leaving it null", () => {
+    expect(
+      errorsOf((r) => buildCoreDrivers({ Policy: text(500) }, r, "T")),
+    ).toBe("");
+    expect(
+      errorsOf((r) => buildCoreDrivers({ Policy: text(501) }, r, "T")),
+    ).toContain("T/coreDrivers.policies: 501 chars > 500");
+  });
+
+  it("counts the period the importer adds", () => {
+    expect(
+      errorsOf((r) => buildCoreDrivers({ Policy: "a".repeat(500) }, r, "T")),
+    ).toContain("T/coreDrivers.policies: 501 chars > 500");
+  });
+
+  it("errors on a dependency over 500, rather than skipping it", () => {
+    const deps = (prose: string) =>
+      errorsOf((r) =>
+        buildDependencies(
+          [{ Sector: "Power", Technology: prose }],
+          new Set(["Power"]),
+          r,
+          "T",
+        ),
+      );
+    expect(deps(text(500))).toBe("");
+    expect(deps(text(501))).toContain(
+      'T/dependencies "Technology" (Power): 501 chars > 500',
+    );
+  });
+
+  it("errors on scope limitations over 500", () => {
+    const da = (scope: string) =>
+      errorsOf((r) =>
+        buildDataAvailability(
+          [
+            {
+              "Sector": "Power",
+              "Metric": "Capacity",
+              "Sector segment": "Power generation",
+              "Granularity": "Coal",
+              "Scope limitations": scope,
+              "Geography coverage": "TH",
+              "Time resolution": "5-year steps",
+              "Data Format": "Tabular",
+            },
+          ],
+          new Set(["Power"]),
+          new Set(["TH"]),
+          r,
+          "T",
+        ),
+      );
+    expect(da(text(500))).toBe("");
+    expect(da(text(501))).toContain(
+      "T/dataAvailability Power Capacity: scope limitations: 501 chars > 500",
+    );
+  });
+
+  describe("buildDoc", () => {
+    const sub = {
+      keyFeatures: [],
+      coreDrivers: undefined,
+      dependencies: [],
+      dataAvailability: [],
+    };
+    const meta = (over: Record<string, string> = {}) => ({
+      "Name of Pathway": "Current Policies Scenario (CPS)",
+      "Name of publication": "World Energy Outlook 2025",
+      "Description": "A short description.",
+      "Pathway Overview": "An overview.",
+      ...over,
+    });
+    const newDoc = (over: Record<string, string> = {}) =>
+      errorsOf((r) =>
+        buildDoc(
+          { mode: "new", file: "T.json", id: "T", templateFile: "t.json" },
+          meta(over),
+          sub,
+          null,
+          null,
+          r,
+        ),
+      );
+
+    it("accepts every field at its limit", () => {
+      expect(
+        newDoc({
+          // Cookbook format: geography suffix inside the brackets, counted in the 20.
+          "Name of Pathway": `${"a".repeat(177)} (${"B".repeat(17)} ID)`,
+          "Name of publication": "p".repeat(200),
+          "Description": text(100),
+          "Pathway Overview": text(2500),
+        }),
+      ).toBe("");
+    });
+
+    it("errors on each field one over its limit", () => {
+      expect(newDoc({ "Name of Pathway": `${"a".repeat(194)} (CODE)` })).toBe(
+        "T/name: 201 chars > 200 -- shorten it in the workbook",
+      );
+      expect(
+        newDoc({ "Name of Pathway": "Plan (Indonesia on-grid power)" }),
+      ).toContain(
+        'T/name: short name "(Indonesia on-grid power)": 23 chars > 20',
+      );
+      expect(newDoc({ "Name of publication": "p".repeat(201) })).toContain(
+        "T/publication.title: 201 chars > 200",
+      );
+      expect(newDoc({ Description: text(101) })).toContain(
+        "T/description: 101 chars > 100",
+      );
+      expect(newDoc({ "Pathway Overview": text(2501) })).toContain(
+        "T/pathwayDescription: 2501 chars > 2500",
+      );
+    });
+
+    it("does not check the name, description or publication of an update, which keeps its own", () => {
+      const errors = errorsOf((r) =>
+        buildDoc(
+          { mode: "update", file: "T.json" },
+          meta({
+            "Name of Pathway": `${"a".repeat(300)} (${"B".repeat(30)})`,
+            "Name of publication": "p".repeat(300),
+            "Description": text(150),
+          }),
+          sub,
+          { id: "T", name: { full: "Kept" }, description: "Kept." },
+          null,
+          r,
+        ),
+      );
+      expect(errors).toBe("");
+    });
+  });
+});

scripts/import-pathway-data.ts

@@ -14,8 +14,10 @@
  *
  * This is a development tool, not a runtime path. It writes files in place and
  * prints a report of everything it could not map cleanly (unresolved scopes,
- * enum/vocabulary mismatches, over-length prose) rather than emitting anything
- * that would fail `npm run schema:check`.
+ * enum/vocabulary mismatches) rather than emitting anything that would fail
+ * `npm run schema:check`. Over-length prose is an authoring error that blocks
+ * the import: the limits are the cookbook's (tpr-tpc decision 0028), and text
+ * over one would otherwise be lost.
  *
  * Sheet names are parameterized (`--sheet-prefix`, default `draft_`) because the
  * final `pathway_*` sheets are not populated yet; switch the prefix once they are.
@@ -662,6 +664,31 @@ export class Report {
   }
 }
 
+/**
+ * Length in code points -- what the cookbook and JSON-schema `maxLength` count.
+ * `.length` counts UTF-16 units, which disagrees for e.g. emoji.
+ */
+export function charCount(text: string): number {
+  return [...text].length;
+}
+
+/**
+ * Whether `text` fits in `max` characters; reports it as an authoring error
+ * when it does not. Over-length text is never truncated or dropped, since that
+ * would lose content nobody chose to lose.
+ */
+export function withinLimit(
+  text: string,
+  max: number,
+  where: string,
+  report: Report,
+): boolean {
+  const n = charCount(text);
+  if (n <= max) return true;
+  report.error(`${where}: ${n} chars > ${max} -- shorten it in the workbook`);
+  return false;
+}
+
 // --------------------------------------------------------------------------
 // Field builders.
 // --------------------------------------------------------------------------
@@ -908,7 +935,7 @@ function buildKeyFeatures(
   return fields;
 }
 
-function buildCoreDrivers(
+export function buildCoreDrivers(
   row: Row | undefined,
   report: Report,
   id: string,
@@ -921,19 +948,13 @@ function buildCoreDrivers(
       continue;
     }
     const text = ensurePeriod(raw);
-    if (text.length > 500) {
-      report.note(
-        `${id}/coreDrivers.${field}: ${text.length} chars > 500 -- left null, needs trimming`,
-      );
-      out[field] = null;
-    } else {
-      out[field] = text;
-    }
+    withinLimit(text, 500, `${id}/coreDrivers.${field}`, report);
+    out[field] = text;
   }
   return out;
 }
 
-function buildDependencies(
+export function buildDependencies(
   rows: Row[],
   declaredSectors: Set<string>,
   report: Report,
@@ -959,12 +980,12 @@ function buildDependencies(
       const prose = row[column] ?? "";
       if (isAbsent(prose)) continue;
       const description = ensurePeriod(prose);
-      if (description.length > 500) {
-        report.note(
-          `${id}/dependencies "${name}" (${sector}): ${description.length} chars > 500 -- skipped`,
-        );
-        continue;
-      }
+      withinLimit(
+        description,
+        500,
+        `${id}/dependencies "${name}" (${sector})`,
+        report,
+      );
       const evidence =
         matchEnum(row[`${column}_Evidence type`] ?? "", EVIDENCE_TYPES) ??
         "No evidence";
@@ -1136,6 +1157,12 @@ export function buildDataAvailability(
       bad,
     );
     const granularity = parseGranularity(row["Granularity"] ?? "", bad);
+    withinLimit(
+      scopeLimitations,
+      500,
+      `${id}/dataAvailability ${sector} ${rawMetric || "(no metric)"}: scope limitations`,
+      report,
+    );
 
     // Every column is required (a row for every allowable pair, per the
     // cookbook), so a blank or unrecognised cell is an authoring error that
@@ -1146,7 +1173,6 @@ export function buildDataAvailability(
       !timeResolution && "time resolution",
       !dataFormat && "data format",
       !scopeLimitations && "scope limitations",
-      scopeLimitations.length > 500 && "scope limitations > 500 chars",
       sectorSegment.length === 0 && "sector segment",
       geography.length === 0 && "geography",
       granularity.length === 0 && "granularity",
@@ -1317,7 +1343,7 @@ function parseTemp(raw: string): number | undefined {
 type Json = Record<string, unknown>;
 
 /** Assemble the full v2 document for one pathway. */
-function buildDoc(
+export function buildDoc(
   target: Target,
   meta: Row,
   sub: {
@@ -1418,14 +1444,8 @@ function buildDoc(
   let pathwayDescription: string | null;
   const overview = (meta["Pathway Overview"] ?? "").trim();
   if (overview && !isAbsent(overview)) {
-    const text = ensurePeriod(overview);
-    if (text.length <= 2500) pathwayDescription = text;
-    else {
-      report.note(
-        `${id}/pathwayDescription: ${text.length} chars > 2500 -- kept existing`,
-      );
-      pathwayDescription = (existing?.pathwayDescription as string) ?? null;
-    }
+    pathwayDescription = ensurePeriod(overview);
+    withinLimit(pathwayDescription, 2500, `${id}/pathwayDescription`, report);
   } else {
     pathwayDescription =
       (existing?.pathwayDescription as string | null) ??
@@ -1460,21 +1480,20 @@ function buildDoc(
   let name: unknown;
   let description: unknown;
   if (target.mode === "new") {
-    const { code } = scenarioParts(meta["Name of Pathway"] ?? "");
-    const full = (meta["Name of Pathway"] ?? "")
+    // Checked only here: an update keeps its existing name, description and
+    // publication, so those cells are not imported for it.
+    const pathwayName = (meta["Name of Pathway"] ?? "").trim();
+    withinLimit(pathwayName, 200, `${id}/name`, report);
+    const { code } = scenarioParts(pathwayName);
+    withinLimit(code, 20, `${id}/name: short name "(${code})"`, report);
+    const full = pathwayName
       .replace(/\s*\([^)]*\)/g, "")
       .replace(/\s*\[[A-Z]{2}\]/g, "")
       .trim();
     name = { full, short: code };
     const desc = ensurePeriod(meta["Description"] ?? "");
-    description =
-      desc.length <= 100
-        ? desc
-        : (template?.description ?? desc.slice(0, 99) + ".");
-    if (desc.length > 100)
-      report.note(
-        `${id}/description: ${desc.length} chars > 100 -- used template description`,
-      );
+    withinLimit(desc, 100, `${id}/description`, report);
+    description = desc;
     publication = buildPublicationForNew(
       template?.publication as Json,
       meta,
@@ -1573,6 +1592,7 @@ function buildPublicationForNew(
 ): Json {
   const pub: Json = template ? JSON.parse(JSON.stringify(template)) : {};
   const title = (meta["Name of publication"] ?? "").trim();
+  withinLimit(title, 200, `${id}/publication.title`, report);
   const year = parseYear(meta["Year of Publication"] ?? "");
   const oldYear = pub.year as number | undefined;
   if (title) pub.title = { ...(pub.title as Json), full: title };