feat: sector segments in the timeseries download and benchmark plots

scripts/build-timeseries-files.ts

@@ -26,6 +26,7 @@ import {
   getTechnologyDefinition,
   UnknownTaxonomyError,
 } from "../src/utils/timeseriesTaxonomy.ts";
+import { jsonToCsv, sectorSegmentColumns } from "./timeseries-csv.ts";
 
 const STRICT = process.env.TS_INDEX_STRICT === "1";
 const DEBUG = process.env.TS_INDEX_DEBUG === "1";
@@ -211,6 +212,7 @@ async function main() {
       const sectors = new Set<string>();
       const geos = new Set<string>();
       const metrics = new Set<string>();
+      const segments = new Set<string>();
 
       for (const row of rawData as any[]) {
         rowCount++;
@@ -239,6 +241,10 @@ async function main() {
         // Unique metrics
         if (typeof row?.metric === "string" && row.metric)
           metrics.add(row.metric);
+        // Unique sector segments (v2 rows carry a list)
+        if (Array.isArray(row?.sectorSegment))
+          for (const seg of row.sectorSegment)
+            if (typeof seg === "string" && seg) segments.add(seg);
       }
 
       if (rowCount > 0) computed.rowCount = rowCount;
@@ -253,6 +259,7 @@ async function main() {
         computed.geographyCount = geos.size;
         computed.geographies = [...geos].sort();
       }
+      if (segments.size > 0) computed.sectorSegments = [...segments].sort();
       if (metrics.size > 0) computed.metrics = [...metrics].sort();
     }
 
@@ -310,21 +317,6 @@ async function main() {
   const PUBLIC_DATA_DIR = path.join(ROOT, "public", "data");
   await fs.mkdir(PUBLIC_DATA_DIR, { recursive: true });
 
-  function jsonToCsv(data: any[]): string {
-    if (!Array.isArray(data) || data.length === 0) return "";
-    // Merge metadata into each row
-    const keys = Array.from(new Set(data.flatMap((row) => Object.keys(row))));
-    const escape = (v: any) =>
-      typeof v === "string"
-        ? `"${v.replace(/"/g, '""')}"`
-        : v === null || v === undefined
-          ? ""
-          : String(v);
-    const header = keys.join(",");
-    const rows = data.map((row) => keys.map((k) => escape(row[k])).join(","));
-    return [header, ...rows].join("\n");
-  }
-
   for (const [dsId, ds] of Object.entries(out.byDataset)) {
     logDebug(`Processing datasetId: ${dsId}`);
     // ds.path currently looks like "/data/…file.json" (derived from src earlier).
@@ -393,7 +385,6 @@ async function main() {
           source,
           sector: sectorDef.displayName,
           emissionsScope: parsed.emissionsScope,
-          sectorScope: metricDef?.sectorScope ?? "",
           metric: metricDef?.displayName ?? "",
           definitionMetric: metricDef?.definition ?? "",
           technology: techDef?.displayName ?? "",
@@ -437,6 +428,12 @@ async function main() {
                   : String(row.technology);
 
               const rowMetadata = getRowMetadata(sector, metric, technology);
+              const segmentColumns = sectorSegmentColumns(
+                sector,
+                Array.isArray(row.sectorSegment)
+                  ? (row.sectorSegment as string[])
+                  : undefined,
+              );
 
               const out = {
                 publisher: rowMetadata.publisher,
@@ -452,9 +449,11 @@ async function main() {
                 unit: row.unit,
                 source: rowMetadata.source,
                 emissions_scope: rowMetadata.emissionsScope,
-                sector_scope: rowMetadata.sectorScope,
+                sector_segment: segmentColumns.sector_segment,
                 definition_technology: rowMetadata.definitionTechnology,
                 definition_metric: rowMetadata.definitionMetric,
+                definition_sector_segment:
+                  segmentColumns.definition_sector_segment,
               };
               logDebug(`    Export row: ${JSON.stringify(out)}`);
               return out;

scripts/timeseries-csv.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect } from "vitest";
+import { csvCell, jsonToCsv, sectorSegmentColumns } from "./timeseries-csv.ts";
+
+describe("sectorSegmentColumns", () => {
+  it("lists the row's segments and their definitions in the same order", () => {
+    const cols = sectorSegmentColumns("power", [
+      "Power generation",
+      "Energy storage",
+    ]);
+    expect(cols.sector_segment).toBe("Power generation; Energy storage");
+    const definitions = cols.definition_sector_segment.split("; ");
+    expect(definitions).toHaveLength(2);
+    expect(definitions[0]).toMatch(/Generation of electricity/);
+    expect(definitions[1]).toMatch(/Storing electricity/);
+  });
+
+  it("leaves both cells empty for a row with no segments", () => {
+    expect(sectorSegmentColumns("power", undefined)).toEqual({
+      sector_segment: "",
+      definition_sector_segment: "",
+    });
+  });
+
+  it("throws on a segment the sector does not define", () => {
+    expect(() => sectorSegmentColumns("power", ["Ironmaking"])).toThrow(
+      /Unknown segment "Ironmaking"/,
+    );
+  });
+});
+
+describe("CSV writing", () => {
+  it("quotes strings and joins a list into one quoted cell", () => {
+    expect(csvCell('say "hi"')).toBe('"say ""hi"""');
+    expect(csvCell(["Power generation", "Energy storage"])).toBe(
+      '"Power generation; Energy storage"',
+    );
+    expect(csvCell(42)).toBe("42");
+    expect(csvCell(null)).toBe("");
+  });
+
+  it("keeps a list in its own column", () => {
+    const csv = jsonToCsv([
+      { geography: "Southeast Asia", sector_segment: ["A", "B"], value: 1 },
+    ]);
+    const [header, row] = csv.split("\n");
+    expect(header).toBe("geography,sector_segment,value");
+    expect(row).toBe('"Southeast Asia","A; B",1');
+  });
+});

scripts/timeseries-csv.ts

@@ -0,0 +1,61 @@
+/**
+ * CSV pieces of the timeseries download, kept out of
+ * build-timeseries-files.ts so they can be tested: that script runs on import.
+ */
+import {
+  getSectorDefinition,
+  segmentDefinitionFor,
+  UnknownTaxonomyError,
+} from "../src/utils/timeseriesTaxonomy.ts";
+
+/** How a list value is written into one CSV cell. */
+export const LIST_SEPARATOR = "; ";
+
+/**
+ * The download's segment columns for one row: the segments it covers and, in
+ * the same order, what each means.
+ *
+ * They replace the former `sector_scope` column, which repeated one taxonomy
+ * string per metric; segments now come from the row itself (#915). A row
+ * without segments (a v1 file) gets empty cells rather than a guess.
+ */
+export function sectorSegmentColumns(
+  sectorKey: string,
+  segments: readonly string[] | undefined,
+): { sector_segment: string; definition_sector_segment: string } {
+  const list = segments ?? [];
+  const sector = getSectorDefinition(sectorKey).displayName;
+  const definitions = list.map((segment) => {
+    const def = segmentDefinitionFor(sector, segment);
+    // Schema validation guarantees this; failing loudly beats a blank cell.
+    if (!def)
+      throw new UnknownTaxonomyError(
+        `Unknown segment "${segment}" for sector "${sector}"`,
+      );
+    return def.definition;
+  });
+  return {
+    sector_segment: list.join(LIST_SEPARATOR),
+    definition_sector_segment: definitions.join(LIST_SEPARATOR),
+  };
+}
+
+/**
+ * One CSV cell. Strings are always quoted; a list is joined first, so it can
+ * never spill its commas into the next columns.
+ */
+export function csvCell(value: unknown): string {
+  if (Array.isArray(value)) return csvCell(value.join(LIST_SEPARATOR));
+  if (typeof value === "string") return `"${value.replace(/"/g, '""')}"`;
+  if (value === null || value === undefined) return "";
+  return String(value);
+}
+
+/** Rows to CSV text; the header is the union of the rows' keys. */
+export function jsonToCsv(data: readonly Record<string, unknown>[]): string {
+  if (data.length === 0) return "";
+  const keys = Array.from(new Set(data.flatMap((row) => Object.keys(row))));
+  const header = keys.join(",");
+  const rows = data.map((row) => keys.map((k) => csvCell(row[k])).join(","));
+  return [header, ...rows].join("\n");
+}

src/components/ComparisonPlots.test.tsx

@@ -19,6 +19,7 @@ function makeEntry(
   pathwayId: string,
   geos: string[],
   metrics: string[] = ["technologyMix"],
+  sectorSegment: string[] = ["Power generation"],
 ): ComparisonPlotsEntry {
   return {
     pathwayId,
@@ -27,6 +28,7 @@ function makeEntry(
         metrics.flatMap((metric) => [
           {
             sector: "power",
+            sectorSegment,
             metric,
             geography: geo,
             year: "2020",
@@ -36,6 +38,7 @@ function makeEntry(
           },
           {
             sector: "power",
+            sectorSegment,
             metric,
             geography: geo,
             year: "2030",
@@ -62,6 +65,7 @@ function makeMetricEntry(
     timeseriesdata: {
       data: values.map((value, i) => ({
         sector: "power",
+        sectorSegment: ["Power generation"],
         metric,
         geography,
         year: String(2020 + i * 10),
@@ -202,11 +206,33 @@ describe("ComparisonPlots", () => {
         />,
       );
 
-      // One per column: the segment follows the shared plot type, but it
-      // belongs in each column's caption next to that column's geography.
+      // One per column, next to that column's geography.
       expect(await screen.findAllByText("Power generation")).toHaveLength(2);
     });
 
+    it("reads each column's segments from its own rows", async () => {
+      // Recorded per row (#915), so two pathways' series of the same metric
+      // can cover different parts of the sector.
+      const entries = [
+        makeEntry("p1", ["Global"]),
+        makeEntry(
+          "p2",
+          ["Global"],
+          ["technologyMix"],
+          ["Power generation", "Energy storage"],
+        ),
+      ];
+      render(
+        <ComparisonPlots
+          entries={entries}
+          requestedGeographies={{ p1: "Global", p2: "Global" }}
+        />,
+      );
+
+      expect(await screen.findAllByText("Power generation")).toHaveLength(2);
+      expect(screen.getAllByText("Energy storage")).toHaveLength(1);
+    });
+
     it("carries the segment's definition as a tooltip", async () => {
       const entries = [
         makeEntry("p1", ["Global"]),
@@ -247,6 +273,21 @@ describe("ComparisonPlots", () => {
     });
   });
 
+  it("omits the segment badge for a column whose series has one year", () => {
+    // PlotPanel draws nothing for a single year, so a badge would describe
+    // an empty chart.
+    const oneYear = makeEntry("p2", ["Global"]);
+    oneYear.timeseriesdata!.data = oneYear.timeseriesdata!.data.slice(0, 1);
+    render(
+      <ComparisonPlots
+        entries={[makeEntry("p1", ["Global"]), oneYear]}
+        requestedGeographies={{ p1: "Global", p2: "Global" }}
+      />,
+    );
+
+    expect(screen.getAllByText("Power generation")).toHaveLength(1);
+  });
+
   it("explains itself rather than relabelling Power series for another sector", () => {
     const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["Global"])];
     render(

src/components/ComparisonPlots.tsx

@@ -1,6 +1,11 @@
 import React, { useCallback, useEffect, useMemo, useState } from "react";
 import type { PlotType, TimeSeries, HoveredPoint } from "./PlotSelector";
-import { PlotPanel, PLOT_OPTIONS, hasDataForMetric } from "./PlotPanel";
+import {
+  PlotPanel,
+  PLOT_OPTIONS,
+  hasDataForMetric,
+  hasDataForMetricAndGeo,
+} from "./PlotPanel";
 import Badge from "./Badge";
 import RegionMembersTooltip from "./RegionMembersTooltip";
 import type { Geography } from "../types";
@@ -14,10 +19,8 @@ import {
   geographyFallbackNote,
   resolveGeography,
 } from "../utils/geographyFallback";
-import {
-  getMetricDefinition,
-  getSectorDefinition,
-} from "../utils/timeseriesTaxonomy";
+import { getSectorDefinition } from "../utils/timeseriesTaxonomy";
+import { plottedSegments } from "../utils/timeseriesSegments";
 import { getSectorSegmentTooltip } from "../utils/tooltipUtils";
 import { useElementWidth } from "../hooks/useElementWidth";
 
@@ -212,13 +215,6 @@ const ComparisonPlots: React.FC<ComparisonPlotsProps> = ({
 
   const hasAnyData = availablePlotOptions.length > 0;
 
-  // Which part of the sector the selected metric describes. One value for the
-  // whole grid, because it follows the shared plot type rather than the pathway.
-  const segment = getMetricDefinition(
-    PLOT_SECTOR.key,
-    selectedPlot,
-  ).sectorScope;
-
   const handlePlotChange = useCallback(
     (e: React.ChangeEvent<HTMLSelectElement>) => {
       setSelectedPlot(e.target.value as PlotType);
@@ -279,20 +275,30 @@ const ComparisonPlots: React.FC<ComparisonPlotsProps> = ({
       </div>
 
       {/*
-        Which part of the sector the selected metric describes. A property of
-        the metric, not of the pathway, so every column shows the same one —
-        but it belongs in each column's caption alongside the geography, the
-        way the detail page captions its small multiples.
-
-        `sectorScope` is optional on MetricDefinition, so a metric may declare
-        no segment; render the badge only when one exists rather than an empty
-        pill.
+        Which parts of the sector each column's series covers. Read from that
+        column's own rows (#915): segments are recorded per row, so two
+        pathways' series of the same metric can differ, and each column says
+        what it shows, the way the detail page captions its small multiples.
+        A column whose rows name no segment renders no segment badge.
       */}
       {entries.map((entry, idx) => {
         const resolution = resolutions[idx];
         const used = resolution?.used ?? "";
         const usedKind = geographyKind(used);
         const note = resolution ? geographyFallbackNote(resolution) : null;
+        // Only for a series PlotPanel draws; see PlotGrid.
+        const segments = hasDataForMetricAndGeo(
+          entry.timeseriesdata,
+          selectedPlot,
+          used,
+        )
+          ? plottedSegments(
+              entry.timeseriesdata?.data,
+              PLOT_SECTOR,
+              selectedPlot,
+              used,
+            )
+          : [];
 
         return (
           <div
@@ -333,8 +339,9 @@ const ComparisonPlots: React.FC<ComparisonPlotsProps> = ({
                 >
                   {geographyLabel(normalizeGeography(used))}
                 </Badge>
-                {segment ? (
+                {segments.map((segment) => (
                   <Badge
+                    key={segment}
                     variant="sectorSegment"
                     tooltip={getSectorSegmentTooltip(
                       PLOT_SECTOR.displayName,
@@ -343,7 +350,7 @@ const ComparisonPlots: React.FC<ComparisonPlotsProps> = ({
                   >
                     {segment}
                   </Badge>
-                ) : null}
+                ))}
               </div>
             ) : null}
             {note ? (