From 0c50c22b2540e28c73db736ddc5a60bc183dc161 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 6 Jul 2026 14:31:18 -0400 Subject: [PATCH] Emit certified populace-us URI in US national Reproduce-in-Python snippets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports policyengine-app v1 PR #2846 to app-v2. The "Reproduce in Python" generator's US national branch (getDatasetUrl in reproducibilityCode.ts) constructed hf://policyengine/policyengine-us-data/{name}_{year}.h5 for a bare national dataset name. That repo is deprecated/archived and policyengine.py no longer resolves it as the default, so a copied snippet pointed users at an uncertified dataset. - Centralize the certified URI as POPULACE_US_DEFAULT_DATASET_URI in constants.ts (single source of truth), pinned to the revision the bundle manifest resolves as data_releases.us.default_dataset_uri. - getDatasetUrl now returns that URI for a bare US national name. Fully-qualified URIs (the exact dataset a run used) still pass through verbatim; default runs still omit dataset= entirely. - Update the bare-name national test to assert the certified URI and the absence of any policyengine-us-data reference. Scope: the subnational (state/CD) and place fallbacks still reference policyengine-us-data/{states,districts}/*.h5 by design, pending Populace geo/place scoping — tracked in app-v2#1079. Documented at each call site; deliberately NOT changed here to avoid colliding with that issue. Co-Authored-By: Claude Fable 5 --- app/src/constants.ts | 13 +++++++++ .../unit/utils/reproducibilityCode.test.ts | 8 +++--- app/src/utils/reproducibilityCode.ts | 27 +++++++++++++++---- changelog_entry.yaml | 2 ++ 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/app/src/constants.ts b/app/src/constants.ts index 2c947b469..d08a8f708 100644 --- a/app/src/constants.ts +++ b/app/src/constants.ts @@ -2,6 +2,19 @@ export const FOREVER = '2100-12-31'; export const BASE_URL = 'https://api.policyengine.org'; export const CURRENT_YEAR = '2026'; +// Certified default US microdata dataset URI, pinned to the revision that +// policyengine.py's bundle manifest resolves as data_releases.us.default_dataset_uri. +// The legacy `policyengine-us-data` Hugging Face repo is deprecated/archived, so any +// generated "Reproduce in Python" snippet for a US national run must point here rather +// than at a `policyengine-us-data` path. Pinned (rather than tracking a floating branch) +// so a copied snippet reproduces the exact certified dataset the app ran against. +// Ported from policyengine-app v1 (POPULACE_US_DEFAULT_DATASET_URI, PR #2846). +// NOTE: subnational (state/CD) and place fallbacks still reference `policyengine-us-data` +// pending Populace place/geo scoping — tracked in policyengine-app-v2#1079 — and are +// intentionally NOT switched to this national URI here. +export const POPULACE_US_DEFAULT_DATASET_URI = + 'hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-sparse-l0-refit-57k-71a0887-national-only-20260701'; + // App URLs for the split website/calculator architecture // In dev mode, these are set via VITE_* env vars to localhost URLs // In production, they fall back to the prod URLs diff --git a/app/src/tests/unit/utils/reproducibilityCode.test.ts b/app/src/tests/unit/utils/reproducibilityCode.test.ts index bf430c94d..ca2f94ab9 100644 --- a/app/src/tests/unit/utils/reproducibilityCode.test.ts +++ b/app/src/tests/unit/utils/reproducibilityCode.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test, vi } from 'vitest'; +import { POPULACE_US_DEFAULT_DATASET_URI } from '@/constants'; import { TEST_COUNTRIES } from '@/tests/fixtures/constants'; import { BASELINE_AND_REFORM_POLICY, @@ -596,9 +597,10 @@ describe('reproducibilityCode', () => { ); const code = lines.join('\n'); - // Then - expect(code).toContain('enhanced_cps_2024.h5'); - expect(code).toContain('dataset='); + // Then - a bare US national dataset name resolves to the certified + // populace-us URI, never the deprecated policyengine-us-data repo. + expect(code).toContain(`dataset="${POPULACE_US_DEFAULT_DATASET_URI}"`); + expect(code).not.toContain('policyengine-us-data'); }); test('given full dataset url then uses it verbatim', () => { diff --git a/app/src/utils/reproducibilityCode.ts b/app/src/utils/reproducibilityCode.ts index a31f0963f..ff622ceb0 100644 --- a/app/src/utils/reproducibilityCode.ts +++ b/app/src/utils/reproducibilityCode.ts @@ -6,7 +6,7 @@ * https://github.com/PolicyEngine/policyengine-app/blob/main/src/data/reformDefinitionCode.js */ -import { CURRENT_YEAR } from '@/constants'; +import { CURRENT_YEAR, POPULACE_US_DEFAULT_DATASET_URI } from '@/constants'; import type { Household } from '@/models/Household'; import { addVariationAxesToPythonPackageHouseholdData, @@ -38,15 +38,21 @@ function normalizeDatasetUrlForReproducibility(countryId: string, datasetName: s /** * Build a HuggingFace dataset URL for a US national-level dataset. - * Dataset files follow the pattern: {name}_{year}.h5 + * + * A fully-qualified URI (e.g. the exact `hf://...` the backend ran against) is + * returned verbatim. For a bare US national dataset name, emit the certified + * `populace-us` URI rather than a legacy `policyengine-us-data/{name}_{year}.h5` + * path: that repo is deprecated/archived and policyengine.py no longer resolves + * it as the default, so a copied snippet must point at the certified dataset. + * Ported from policyengine-app v1 PR #2846. */ -function getDatasetUrl(countryId: string, datasetName: string, year: number): string | null { +function getDatasetUrl(countryId: string, datasetName: string): string | null { if (datasetName.includes('://')) { return normalizeDatasetUrlForReproducibility(countryId, datasetName); } if (countryId === 'us') { - return `hf://policyengine/policyengine-us-data/${datasetName}_${year}.h5`; + return POPULACE_US_DEFAULT_DATASET_URI; } return null; } @@ -57,6 +63,12 @@ function getDatasetUrl(countryId: string, datasetName: string, year: number): st * - "state/ca" → states/CA.h5 * - "congressional_district/CA-01" → districts/CA-01.h5 * Note: place/ regions are handled separately via getPlaceStateDatasetUrl. + * + * These `policyengine-us-data/{states,districts}/*.h5` paths are the geography- + * scoped fallback used only when a non-default subnational dataset is selected. + * They intentionally still reference the legacy repo pending Populace geo scoping; + * see policyengine-app-v2#1079. Default state/CD runs already use national Populace + * scoping via getScopedUsRegionImplementationCode, so they do not hit this path. */ function getSubnationalDatasetUrl(region: string): string | null { for (const [prefix, folder] of Object.entries(US_REGION_PREFIX_TO_FOLDER)) { @@ -90,6 +102,11 @@ function isDefaultUsScopedRegion( /** * For place/ regions, get the parent state's dataset URL. * "place/NJ-57000" → states/NJ.h5 + * + * Places load the parent state's `policyengine-us-data` H5 and filter by + * place_fips because the certified Populace export does not yet carry place + * geography. This legacy reference is deliberately retained pending Populace + * place scoping; see policyengine-app-v2#1079. */ function getPlaceStateDatasetUrl(region: string): string | null { if (!region.startsWith('place/')) { @@ -318,7 +335,7 @@ function getImplementationCode( const isNational = region === countryId; const year = timePeriod || DEFAULT_YEAR; const resolvedDatasetUrl = - dataset && !isDefaultDataset ? getDatasetUrl(countryId, dataset, year) : null; + dataset && !isDefaultDataset ? getDatasetUrl(countryId, dataset) : null; if (isDefaultUsScopedRegion(countryId, region, dataset, isDefaultDataset)) { return getScopedUsRegionImplementationCode(region, year, hasBaseline, hasReform); diff --git a/changelog_entry.yaml b/changelog_entry.yaml index eaceeef42..d9fb06aad 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -8,3 +8,5 @@ changed: - Proxy UK Python package docs at /uk/python and UK API docs at /uk/api so the country-parameterized "Python" and "API" nav links work on UK pages - Align the calculator app header (e.g. /us/reports) with the main website header — hover-open dropdowns, per-item underline, Model dropdown with sub-items, Python link, and Events under About + fixed: + - Point "Reproduce in Python" snippets for a US national run at the certified populace-us dataset instead of the deprecated policyengine-us-data repo