From 8acb0b0ad2c616b193ffdda8ea04f99b83fecfe9 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 16:16:36 +1000 Subject: [PATCH 1/2] Name the expansion-id filter for what it does, and fill out the constraint docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _keep_rows_for_enabled_elements is shared by the network and constraints translators, but "enabled elements" only described the network caller — the constraints side passes constraint_ids. Renamed to _keep_rows_for_expansion_ids and reworded the docstrings so each caller's selection is described where the decision is made. The constraint helpers' I/O examples now show full tables, including the blank-cell wildcard forms the options and costs accept. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 84 ++++++++++++++++++++------- src/ispypsa/translator/network.py | 46 ++++++++------- 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 3996697d..e0fea38b 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -106,7 +106,7 @@ _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, ) from ispypsa.translator.network import ( - _keep_rows_for_enabled_elements, + _keep_rows_for_expansion_ids, _pair_forward_and_reverse_options, _prepare_expansion_costs, _resolve_expansion_options, @@ -294,10 +294,11 @@ def _translate_custom_constraints_from_network_tables( ) rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + relaxation_generators = _finalise_generators(relaxation_generators) return { "custom_constraints_lhs": lhs, "custom_constraints_rhs": rhs, - "custom_constraints_generators": _finalise_generators(relaxation_generators), + "custom_constraints_generators": relaxation_generators, } @@ -357,9 +358,20 @@ def _add_constraint_type( matching the vocabulary pypsa_build applies constraints with). I/O Example: - rhs: constraint_id=SWQLD1, rhs=3000 - custom_constraints: constraint_id=SWQLD1, direction="<=" - -> constraint_id=SWQLD1, rhs=3000, constraint_type="<=" + rhs: + constraint_id timeslice rhs investment_period + SWQLD1 qld_peak_demand 3000 2026 + NQ1 qld_peak_demand 2650 2026 + + custom_constraints: + constraint_id direction + SWQLD1 <= + NQ1 = + + returns: + constraint_id timeslice rhs investment_period constraint_type + SWQLD1 qld_peak_demand 3000 2026 <= + NQ1 qld_peak_demand 2650 2026 == """ rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) @@ -383,9 +395,17 @@ def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: belongs to. I/O Example: - term_type=generator_output -> component=Generator, attribute=p - term_type=link_flow -> component=Link, attribute=p - term_type=storage_output -> component=Storage, attribute=p + lhs: + constraint_id term_type variable_name coefficient investment_period + SWQLD1 link_flow NSW-QLD 0.84 2026 + SWQLD1 generator_output KINGASF1 0.14 2026 + SWQLD1 storage_output Q8 Battery - 2h 0.43 2026 + + returns: + constraint_id variable_name coefficient investment_period component attribute + SWQLD1 NSW-QLD 0.84 2026 Link p + SWQLD1 KINGASF1 0.14 2026 Generator p + SWQLD1 Q8 Battery - 2h 0.43 2026 Storage p """ lhs = lhs.copy() lhs["component"] = lhs["term_type"].map( @@ -477,23 +497,37 @@ def _create_constraint_relaxation_generators( The generators' p_nom enters the parent constraint's LHS with coefficient -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the constraint at the option's cost; total relaxation is capped at the - option's allowed_expansion by the expansion-limit constraints. The - constraints in the model (constraint_ids) are the enabled elements the - options and costs wildcards resolve against, gated as a whole by the - config's rez_transmission_expansion flag. + option's allowed_expansion by the expansion-limit constraints. - I/O Example: + Options and costs may use blank key cells as wildcards: a blank + expansion_id means "every constraint" — here, every constraint in the + model (constraint_ids) — and a blank cost year means "every investment + period", so a single blank-id row gives all constraints the same + relaxation option or cost, and a blank-year cost row is a static cost + across the periods. If the config's rez_transmission_expansion flag is + off, no relaxation generators are built at all. + + I/O Example (blank cells are wildcards): ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option SWQLD1 constraint_relaxation 500 Option 2 + constraint_relaxation 200 Default ispypsa_tables["network_transmission_path_expansion_costs"]: expansion_id year cost SWQLD1 2030 100000 + 80000 # every constraint, every period + + constraint_ids = ["SWQLD1", "NQ1"] - constraint_ids=["SWQLD1"], investment_periods=[2030] returns (abridged): - name isp_name bus p_nom build_year allowed_expansion - SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 500 + config: rez_transmission_expansion = True, investment_periods = [2030, 2040] + + returns: + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(100000) 500 + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 500 + NQ1_exp_2030 NQ1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(80000) 200 + NQ1_exp_2040 NQ1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 200 """ if not config.network.rez_transmission_expansion: return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) @@ -543,7 +577,7 @@ def _resolve_relaxation_options( options = options[ expansion_type.isna() | (expansion_type == "constraint_relaxation") ] - options = _keep_rows_for_enabled_elements(options, constraint_ids) + options = _keep_rows_for_expansion_ids(options, constraint_ids) allowed_values = { "expansion_id": constraint_ids, "expansion_type": ["constraint_relaxation"], @@ -557,9 +591,15 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: """Adds the PyPSA generator attributes shared by all relaxation generators. I/O Example: - expansion_id=SWQLD1, year=2030, capital_cost=8140, allowed_expansion=500 - -> name=SWQLD1_exp_2030, isp_name=SWQLD1, p_nom=0.0, - p_nom_extendable=True, build_year=2030, lifetime=inf + generators: + expansion_id year capital_cost allowed_expansion + SWQLD1 2030 8140 500 + SWQLD1 2040 7900 500 + + returns: + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf 8140 500 + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf 7900 500 """ generators = generators.rename(columns={"expansion_id": "isp_name"}) generators["name"] = ( @@ -589,7 +629,9 @@ def _relaxation_generator_lhs_terms( SWQLD1_exp_2030 SWQLD1 2030 SWQLD1_exp_2040 SWQLD1 2040 - investment_periods=[2030, 2040] returns: + investment_periods=[2030, 2040] + + returns: constraint_id investment_period variable_name component attribute coefficient SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index 66233841..f274eab8 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -398,7 +398,7 @@ def _enabled_expansion_element_ids( ``transmission_expansion`` gates flow paths between (sub)regions; ``rez_transmission_expansion`` gates REZ connection paths. The result is the enabled expansion_id set the options and costs tables are filtered to (see - _keep_rows_for_enabled_elements) and then resolved against, so an option or + _keep_rows_for_expansion_ids) and then resolved against, so an option or cost for a disabled or non-modelled element drops out before resolution. I/O Example: @@ -462,7 +462,7 @@ def _resolve_expansion_options( # not dropped, so they are set aside before wildcard resolution (which would # otherwise raise on them). options = options[options["expansion_type"] != "constraint_relaxation"] - options = _keep_rows_for_enabled_elements(options, enabled_ids) + options = _keep_rows_for_expansion_ids(options, enabled_ids) allowed_values = { "expansion_id": enabled_ids, "expansion_type": ["forward", "reverse"], @@ -477,22 +477,23 @@ def _resolve_expansion_options( return options -def _keep_rows_for_enabled_elements( - table: pd.DataFrame, enabled_ids: list[str] +def _keep_rows_for_expansion_ids( + table: pd.DataFrame, expansion_ids: list[str] ) -> pd.DataFrame: - """Keeps rows whose expansion_id is an enabled element or blank (a wildcard). + """Keeps rows whose expansion_id is one of the given ids or blank (a wildcard). - Selecting the enabled elements is config-driven, so it happens before - wildcard resolution rather than inside it. Rows for disabled or non-modelled - elements drop out here, as do rows for constraint groups (their expansion_ids - are constraint_ids, routed to ispypsa.translator.constraints instead). + Which ids to keep is a config-driven selection made by the caller — the + enabled paths here, the constraints in the model in + ispypsa.translator.constraints — so it happens before wildcard resolution + rather than inside it. Rows for any other id drop out; the blank rows are + kept because they are wildcards that resolve against the given ids. I/O Example: - table: enabled_ids = ["CQ-NQ"] + table: expansion_ids = ["CQ-NQ"] expansion_id year cost CQ-NQ 2026 1000000 - Q1-NQ 2026 500000 # disabled element: dropped - SWQLD1 2026 400000 # constraint group: dropped + Q1-NQ 2026 500000 # not in expansion_ids: dropped + SWQLD1 2026 400000 # not in expansion_ids: dropped 2026 900000 # blank: a wildcard, kept returns: @@ -501,7 +502,7 @@ def _keep_rows_for_enabled_elements( 2026 900000 """ ids = table["expansion_id"] - keep = ids.isna() | ids.isin(enabled_ids) + keep = ids.isna() | ids.isin(expansion_ids) return table[keep] @@ -591,19 +592,20 @@ def _prepare_expansion_costs( wacc: float, asset_lifetime: int, ) -> pd.DataFrame: - """Resolves the expansion-cost wildcards to the enabled elements and + """Resolves the expansion-cost wildcards to the given expansion ids and investment periods, then annuitises them. Blank expansion_id or year cells are wildcards (see the network_transmission_path_expansion_costs schema): an empty expansion_id is a table-wide default cost, an empty year a static cost across the investment - periods. Costs for disabled elements, constraint groups (routed to - ispypsa.translator.constraints) and years outside the investment periods are - all designed selection, filtered out first. _resolve_wildcards - then expands the blanks against the enabled elements and the investment - periods. A blank cost then resolves to free — the nan_fill the schema - declares for the cost column. Year values are labels matched against the - config's investment periods as ints — no financial vs calendar year + periods. The caller decides which ids the costs are for — the enabled paths + here, the constraints in the model in ispypsa.translator.constraints — so + rows for any other id, and rows for years that are not investment periods, + are dropped before resolution: they are selection, not bad data. + _resolve_wildcards then expands the blanks against the given ids and the + investment periods. A blank cost then resolves to free — the nan_fill the + schema declares for the cost column. Year values are labels matched against + the config's investment periods as ints — no financial vs calendar year interpretation happens here; the config's year_type decides what span of time the labels denote, so the table just has to label years the same way the config does (templated tables carry financial-year ending years). @@ -622,7 +624,7 @@ def _prepare_expansion_costs( CQ-NQ 2026 annuitise(1000) # the static row fills 2026 CQ-NQ 2028 annuitise(1200) # the 2028 override beats the static row """ - costs = _keep_rows_for_enabled_elements(expansion_costs, enabled_ids) + costs = _keep_rows_for_expansion_ids(expansion_costs, enabled_ids) costs = _keep_rows_for_investment_period_years(costs, investment_periods) allowed_values = {"expansion_id": enabled_ids, "year": investment_periods} costs = _resolve_wildcards(costs, allowed_values, ["cost"]) From 39d80eba3d720d65051c657143ac586108afcf3a Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Mon, 24 Aug 2026 14:34:03 +1000 Subject: [PATCH 2/2] Give the financial-year boundary one home in translator.helpers The start of a model year under FY nomenclature (1 July of the year before the label) was encoded three times: forward as a triple in _get_iteration_start_and_end_time, forward again as Timestamps in constraints._investment_period_start_dates, and in reverse in snapshots._add_investment_periods. A change to any one would silently leave the custom-constraint date_from resolution and the snapshots disagreeing about where a period begins. _period_start and its inverse _period_of now own the convention; the triple helper and the two other sites derive from them, with the triple's shape pinned by a test so the snapshot builders are unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01XRWFpjCqEbjKuYNv7mVsUk --- src/ispypsa/translator/constraints.py | 10 ++-- src/ispypsa/translator/helpers.py | 53 +++++++++++++++---- src/ispypsa/translator/snapshots.py | 14 ++--- .../test_translator_helpers.py | 31 +++++++++++ 4 files changed, 83 insertions(+), 25 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index e0fea38b..a6a29276 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -100,7 +100,7 @@ import pandas as pd from ispypsa.config import ModelConfig -from ispypsa.translator.helpers import _resolve_wildcards +from ispypsa.translator.helpers import _period_start, _resolve_wildcards from ispypsa.translator.mappings import ( _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE, _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, @@ -305,15 +305,15 @@ def _translate_custom_constraints_from_network_tables( def _investment_period_start_dates( investment_periods: list[int], year_type: str ) -> dict[int, pd.Timestamp]: - """The datetime each investment period starts at. + """The datetime each investment period starts at — the same financial or + calendar year boundary the snapshots use (see _period_start in + ispypsa.translator.helpers). I/O Example: [2030], "fy" -> {2030: 2029-07-01} # FY ending nomenclature [2030], "calendar" -> {2030: 2030-01-01} """ - if year_type == "fy": - return {p: pd.Timestamp(year=p - 1, month=7, day=1) for p in investment_periods} - return {p: pd.Timestamp(year=p, month=1, day=1) for p in investment_periods} + return {p: _period_start(year_type, p) for p in investment_periods} def _resolve_values_active_at_period_starts( diff --git a/src/ispypsa/translator/helpers.py b/src/ispypsa/translator/helpers.py index 90b57c03..40387b71 100644 --- a/src/ispypsa/translator/helpers.py +++ b/src/ispypsa/translator/helpers.py @@ -3,19 +3,50 @@ import pandas as pd -def _get_iteration_start_and_end_time(year_type: str, start_year: int, end_year: int): - """Get the model start year, end year, and start/end month for iteration, which depend on - financial vs calendar year. +def _period_start(year_type: str, year: int) -> pd.Timestamp: + """The datetime a model year starts at. + + Model years are labelled by the calendar year they end in under financial + year nomenclature, so FY 2030 starts on 1 July 2029; a calendar year starts + on 1 January of its own year. This is the one place that boundary lives: + the snapshots (through _get_iteration_start_and_end_time and _period_of) + and the custom-constraint date_from resolution all take it from here. + + I/O Example: + ("fy", 2030) -> 2029-07-01 + ("calendar", 2030) -> 2030-01-01 """ if year_type == "fy": - start_year = start_year - 1 - end_year = end_year - month = 7 - else: - start_year = start_year - end_year = end_year + 1 - month = 1 - return start_year, end_year, month + return pd.Timestamp(year=year - 1, month=7, day=1) + return pd.Timestamp(year=year, month=1, day=1) + + +def _period_of(year_type: str, timestamps: pd.Series) -> pd.Series: + """The model year each timestamp falls in — the inverse of _period_start. + + I/O Example: + ("fy", [2029-06-30 23:00, 2029-07-01 00:00]) -> [2029, 2030] + ("calendar", [2029-06-30 23:00, 2029-07-01 00:00]) -> [2029, 2029] + """ + years = timestamps.dt.year.astype("int64") + if year_type == "fy": + return years + (timestamps.dt.month >= 7).astype("int64") + return years + + +def _get_iteration_start_and_end_time(year_type: str, start_year: int, end_year: int): + """The (start_year, end_year, month) triple the snapshot builders iterate + over: the calendar year and month the first model year starts in, and the + calendar year the model year after end_year starts in (an exclusive end). + Both boundaries come from _period_start. + + I/O Example: + ("fy", 2025, 2030) -> (2024, 2030, 7) # 2024-07-01 up to 2030-07-01 + ("calendar", 2025, 2030) -> (2025, 2031, 1) # 2025-01-01 up to 2031-01-01 + """ + first = _period_start(year_type, start_year) + end = _period_start(year_type, end_year + 1) + return first.year, end.year, first.month def _annuitised_investment_costs( diff --git a/src/ispypsa/translator/snapshots.py b/src/ispypsa/translator/snapshots.py index 609629c5..e5263c86 100644 --- a/src/ispypsa/translator/snapshots.py +++ b/src/ispypsa/translator/snapshots.py @@ -7,7 +7,10 @@ from ispypsa.config import ModelConfig, load_config from ispypsa.data_fetch import read_csvs -from ispypsa.translator.helpers import _get_iteration_start_and_end_time +from ispypsa.translator.helpers import ( + _get_iteration_start_and_end_time, + _period_of, +) from ispypsa.translator.temporal_filters import _filter_snapshots @@ -159,14 +162,7 @@ def _add_investment_periods( Returns: pd.DataFrame with column "investment_periods" and "snapshots". """ snapshots = snapshots.copy() - snapshots["calendar_year"] = snapshots["snapshots"].dt.year - snapshots["effective_year"] = snapshots["calendar_year"].astype("int64") - - if year_type == "fy": - mask = snapshots["snapshots"].dt.month >= 7 - snapshots.loc[mask, "effective_year"] = ( - snapshots.loc[mask, "effective_year"] + 1 - ) + snapshots["effective_year"] = _period_of(year_type, snapshots["snapshots"]) inv_periods_df = pd.DataFrame({"investment_periods": investment_periods}) inv_periods_df = inv_periods_df.sort_values("investment_periods") diff --git a/tests/test_translator/test_translator_helpers.py b/tests/test_translator/test_translator_helpers.py index dc5c75b0..86d30fcd 100644 --- a/tests/test_translator/test_translator_helpers.py +++ b/tests/test_translator/test_translator_helpers.py @@ -6,10 +6,41 @@ from ispypsa.translator.helpers import ( _add_investment_periods_as_build_years, _get_financial_year_int_from_string, + _get_iteration_start_and_end_time, + _period_of, + _period_start, _resolve_wildcards, ) +def test_period_start_is_the_financial_or_calendar_year_boundary(): + assert _period_start("fy", 2030) == pd.Timestamp("2029-07-01") + assert _period_start("calendar", 2030) == pd.Timestamp("2030-01-01") + + +def test_period_of_is_the_inverse_of_period_start(): + timestamps = pd.Series( + pd.to_datetime(["2029-06-30 23:00", "2029-07-01 00:00", "2030-06-30 23:00"]) + ) + + financial_years = _period_of("fy", timestamps) + calendar_years = _period_of("calendar", timestamps) + + pd.testing.assert_series_equal( + financial_years, pd.Series([2029, 2030, 2030], dtype="int64"), check_names=False + ) + pd.testing.assert_series_equal( + calendar_years, pd.Series([2029, 2029, 2030], dtype="int64"), check_names=False + ) + + +def test_get_iteration_start_and_end_time_runs_first_start_to_exclusive_end(): + """Pins the triple's shape so the snapshot builders see no change from it + being derived from _period_start.""" + assert _get_iteration_start_and_end_time("fy", 2025, 2030) == (2024, 2030, 7) + assert _get_iteration_start_and_end_time("calendar", 2025, 2030) == (2025, 2031, 1) + + def test_get_financial_year_int_from_string(): """Test financial year string translation to integer.""" # Standard financial year format