From 1a53dc2536588d96a5381f11800124862d4cfaa3 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:27:57 +1000 Subject: [PATCH 01/20] Translate the new-format custom constraints with temporal scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each RHS value becomes one constraint instance per investment period (and per timeslice where tagged), with date_from resolved as the value active at the start of each period. LHS terms keep generator and battery names even though those components aren't translated yet — pypsa_build skips-and-logs missing components, and the logging silences itself once generator translation lands. The generators/batteries parameters are accepted now as the future name-mapping hook. Constraint-relaxation expansion options become dummy generators on bus_for_custom_constraint_gens, with their LHS terms filtered per period by build_year so a generator built for 2040 can't relax a 2030 constraint instance — an improvement over the old single-constraint formulation, which couldn't express this. Extendable links and relaxation generators get "_expansion_limit" constraints capping total p_nom at the option's allowed expansion; the suffix keeps their names clear of the parent constraints'. Duplicate input rows and RHS rows without LHS pairs raise rather than collapse silently; the "load" term type maps to Load/p, which pypsa_build logs as unimplemented and skips, matching the old path. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 587 ++++++++++++++++++++++ src/ispypsa/translator/mappings.py | 5 + tests/test_translator/test_constraints.py | 327 ++++++++++++ 3 files changed, 919 insertions(+) create mode 100644 src/ispypsa/translator/constraints.py create mode 100644 tests/test_translator/test_constraints.py diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py new file mode 100644 index 00000000..1c6cff04 --- /dev/null +++ b/src/ispypsa/translator/constraints.py @@ -0,0 +1,587 @@ +"""Translates the new-format custom-constraint tables into PyPSA friendly form. + +Consumes ``custom_constraints`` (constraint senses), ``custom_constraints_lhs`` +(terms) and ``custom_constraints_rhs`` (limit values), plus the unified +network expansion tables for constraint-relaxation generators and +expansion-limit constraints. + +The PyPSA friendly outputs carry two temporal columns the old format lacked: + +- ``investment_period``: time-varying inputs (``date_from``) are resolved by + holding, for each investment period, the value active at the period's + start. Rows with a NaN investment_period apply in every period. +- ``timeslice`` (RHS only): the constraint binds only at snapshots inside the + timeslice's windows (see ispypsa.translator.timeslices). NaN means the + limit applies at all snapshots. +""" + +import logging + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.mappings import ( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE, + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, +) +from ispypsa.translator.network import ( + _pivot_physical_expansion_options, + _prepare_expansion_costs, +) + +logger = logging.getLogger(__name__) + +_LHS_COLUMNS = [ + "constraint_name", + "investment_period", + "variable_name", + "component", + "attribute", + "coefficient", +] + +_RHS_COLUMNS = [ + "constraint_name", + "investment_period", + "timeslice", + "rhs", + "constraint_type", +] + +_GENERATOR_COLUMNS = [ + "name", + "isp_name", + "bus", + "p_nom", + "p_nom_extendable", + "build_year", + "lifetime", + "capital_cost", +] + +_DIRECTION_TO_CONSTRAINT_TYPE = {"<=": "<=", ">=": ">=", "=": "=="} + +# Working column orders before constraint_id is renamed to constraint_name. +_INTERNAL_LHS_COLUMNS = ["constraint_id"] + [ + c for c in _LHS_COLUMNS if c != "constraint_name" +] +_INTERNAL_RHS_COLUMNS = ["constraint_id"] + [ + c for c in _RHS_COLUMNS if c != "constraint_name" +] + + +def _raise_on_duplicate_input_rows( + table: pd.DataFrame, keys: list[str], label: str +) -> None: + """Raise on input rows sharing the same key — duplicates would otherwise + silently collapse to one arbitrary row during date_from resolution.""" + duplicates = table[table.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint {label} rows for: " + f"{sorted(set(duplicates['constraint_id']))}" + ) + + +def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.DataFrame: + """Concatenates the non-empty frames (concatenating all-empty frames is + deprecated by pandas), returning a header-only frame when all are empty.""" + non_empty = [frame for frame in frames if not frame.empty] + if not non_empty: + return pd.DataFrame(columns=columns) + return pd.concat(non_empty, ignore_index=True) + + +def _translate_custom_constraints_from_network_tables( + ispypsa_tables: dict[str, pd.DataFrame], + links: pd.DataFrame, + config: ModelConfig, + generators: pd.DataFrame | None = None, + batteries: pd.DataFrame | None = None, +) -> dict[str, pd.DataFrame]: + """Translates the custom-constraint tables and builds the endogenous + expansion-limit constraints. + + Generator and battery terms pass through with their IASR IDs as + variable_names: pypsa_build skips (and logs) terms whose components are + not in the model. The ``generators`` and ``batteries`` arguments are + accepted so the templated generator/battery tables can be used to map + those IDs to model component names once generator templating lands; they + are currently unused. + + Args: + ispypsa_tables: dictionary of new-format `ISPyPSA` input tables; + consumes custom_constraints, custom_constraints_lhs, + custom_constraints_rhs, network_expansion_options and + network_transmission_path_expansion_costs. + links: PyPSA friendly links table from + ispypsa.translator.network (existing plus expansion links). + config: `ispypsa.config.ModelConfig` object. + generators: templated generator table (future name-mapping hook). + batteries: templated battery table (future name-mapping hook). + + Returns: dict with the PyPSA friendly custom_constraints_lhs, + custom_constraints_rhs and custom_constraints_generators tables. + """ + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice", "date_from"], + "RHS", + ) + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name", "date_from"], + "LHS", + ) + period_starts = _investment_period_start_dates( + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + ) + rhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice"], + period_starts, + ) + rhs = _add_constraint_type(rhs, ispypsa_tables["custom_constraints"]) + lhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name"], + period_starts, + ) + lhs = _add_component_and_attribute(lhs) + lhs = _expand_link_flow_terms(lhs, links) + rhs = _drop_rhs_without_lhs_terms(rhs, lhs) + + relaxation_generators = _create_constraint_relaxation_generators( + ispypsa_tables, set(rhs["constraint_id"]), config + ) + relaxation_generator_lhs = _relaxation_generator_lhs_terms( + relaxation_generators, + config.temporal.capacity_expansion.investment_periods, + ) + expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( + links, relaxation_generators, ispypsa_tables["network_expansion_options"] + ) + + lhs = _concat_non_empty( + [lhs, relaxation_generator_lhs, expansion_limit_lhs], _INTERNAL_LHS_COLUMNS + ) + rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) + lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + return { + "custom_constraints_lhs": lhs, + "custom_constraints_rhs": rhs, + "custom_constraints_generators": _finalise_generators(relaxation_generators), + } + + +def _investment_period_start_dates( + investment_periods: list[int], year_type: str +) -> dict[int, pd.Timestamp]: + """The datetime each investment period starts at. + + I/O Example: + [2030], "fy" -> {2030: 2029-07-01} # FY ending nomenclature + [2030], "calendar" -> {2030: 2030-01-01} + """ + if year_type == "fy": + return {p: pd.Timestamp(year=p - 1, month=7, day=1) for p in investment_periods} + return {p: pd.Timestamp(year=p, month=1, day=1) for p in investment_periods} + + +def _resolve_values_active_at_period_starts( + table: pd.DataFrame, + group_columns: list[str], + period_starts: dict[int, pd.Timestamp], +) -> pd.DataFrame: + """Resolves date_from-varying rows into one row per investment period. + + For each period, each group keeps the row active at the period's start: + the latest date_from on or before it, with no-date_from rows acting as + the baseline. A group whose earliest date_from is after a period's start + contributes no row for that period. + + I/O Example: + table (group_columns=["constraint_id", "timeslice"]): + constraint_id timeslice rhs date_from + SWQLD1 qld_peak_demand 3000 + SWQLD1 qld_peak_demand 2500 2032-12-01T00:00:00 + + period_starts={2030: 2029-07-01, 2035: 2034-07-01} returns: + constraint_id timeslice rhs investment_period + SWQLD1 qld_peak_demand 3000 2030 + SWQLD1 qld_peak_demand 2500 2035 # 2032 value held from period start + """ + table = table.copy() + table["date_from"] = pd.to_datetime(table["date_from"]).fillna(pd.Timestamp.min) + resolved = [] + for period, start in period_starts.items(): + active = table[table["date_from"] <= start] + active = active.sort_values("date_from") + active = active.groupby(group_columns, dropna=False).tail(1).copy() + active["investment_period"] = period + resolved.append(active) + return pd.concat(resolved, ignore_index=True).drop(columns="date_from") + + +def _add_constraint_type( + rhs: pd.DataFrame, custom_constraints: pd.DataFrame +) -> pd.DataFrame: + """Adds each constraint's sense as constraint_type ("=" becomes "==", + matching the vocabulary pypsa_build applies constraints with). + + I/O Example: + rhs: constraint_id=SWQLD1, rhs=3000 + custom_constraints: constraint_id=SWQLD1, direction="<=" + -> constraint_id=SWQLD1, rhs=3000, constraint_type="<=" + """ + rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") + rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) + _raise_on_missing_constraint_type(rhs) + return rhs.drop(columns="direction") + + +def _raise_on_missing_constraint_type(rhs: pd.DataFrame) -> None: + """Raise if any RHS row's constraint has no (or an unmapped) sense — the + constraint can't be applied without one.""" + missing = rhs.loc[rhs["constraint_type"].isna(), "constraint_id"] + if not missing.empty: + raise ValueError( + f"Custom constraints with RHS values but no direction in the " + f"custom_constraints table: {sorted(set(missing))}" + ) + + +def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: + """Maps each term_type to the PyPSA component and attribute its variable + belongs to. + + I/O Example: + term_type=generator_output -> component=Generator, attribute=p + term_type=link_flow -> component=Link, attribute=p + term_type=storage_output -> component=Storage, attribute=p + """ + lhs = lhs.copy() + lhs["component"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE + ) + lhs["attribute"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE + ) + _raise_on_unmapped_term_types(lhs) + return lhs.drop(columns="term_type") + + +def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: + """Raise if any term_type has no component mapping — silently dropping a + term would weaken its constraint.""" + unmapped = lhs.loc[lhs["component"].isna(), "term_type"] + if not unmapped.empty: + raise ValueError( + f"Custom constraint LHS term_types with no component mapping: " + f"{sorted(set(unmapped))}" + ) + + +def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFrame: + """Replaces each link term's path_id with the model's link names — the + existing link plus each expansion link — one term per link. Terms for + paths not in the model are dropped and logged. + + I/O Example: + lhs: + constraint_id variable_name component coefficient + SWQLD1 NSW-QLD Link 0.84 + SWQLD1 KINGASF1 Generator 0.14 + + links (isp_name -> name): NSW-QLD -> NSW-QLD_existing, NSW-QLD_exp_2030 + + returns: + constraint_id variable_name component coefficient + SWQLD1 KINGASF1 Generator 0.14 + SWQLD1 NSW-QLD_existing Link 0.84 + SWQLD1 NSW-QLD_exp_2030 Link 0.84 + """ + link_terms = lhs[lhs["component"] == "Link"] + other_terms = lhs[lhs["component"] != "Link"] + _log_link_terms_not_in_model(link_terms, links) + expanded = link_terms.merge( + links.loc[:, ["isp_name", "name"]], left_on="variable_name", right_on="isp_name" + ) + expanded = expanded.drop(columns=["variable_name", "isp_name"]) + expanded = expanded.rename(columns={"name": "variable_name"}) + return pd.concat([other_terms, expanded], ignore_index=True) + + +def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) -> None: + missing = set(link_terms["variable_name"]) - set(links["isp_name"]) + if missing: + logger.info( + f"Custom constraint link_flow terms dropped (paths not in model): " + f"{sorted(missing)}" + ) + + +def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.DataFrame: + """Drops (and logs) RHS rows for constraints left with no LHS terms — a + constraint with an empty LHS can't be applied.""" + without_lhs = set(rhs["constraint_id"]) - set(lhs["constraint_id"]) + if without_lhs: + logger.info( + f"Custom constraints dropped (no LHS terms in model): {sorted(without_lhs)}" + ) + return rhs[~rhs["constraint_id"].isin(without_lhs)] + + +def _create_constraint_relaxation_generators( + ispypsa_tables: dict[str, pd.DataFrame], + constraint_ids: set[str], + config: ModelConfig, +) -> pd.DataFrame: + """Builds one extendable dummy generator per relaxable constraint and + investment period, with the selected expansion option's annualised cost. + + The generators' p_nom enters the parent constraint's LHS with coefficient + -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the + constraint at the option's cost; total relaxation is capped at the + option's allowed_expansion by the expansion-limit constraints. + + I/O Example: + network_expansion_options: + expansion_id expansion_type allowed_expansion + SWQLD1 constraint_relaxation 500 + + network_transmission_path_expansion_costs: + expansion_id year cost + SWQLD1 2030 100000 + + constraint_ids={"SWQLD1"}, investment_periods=[2030] returns (abridged): + name isp_name bus p_nom build_year + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 + """ + if not config.network.rez_transmission_expansion: + return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) + options = ispypsa_tables["network_expansion_options"] + relaxations = options[options["expansion_type"] == "constraint_relaxation"] + relaxations = _drop_relaxations_without_constraints(relaxations, constraint_ids) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + generators = costs.merge( + relaxations.loc[:, ["expansion_id", "allowed_expansion"]], on="expansion_id" + ) + return _format_relaxation_generators(generators) + + +def _drop_relaxations_without_constraints( + relaxations: pd.DataFrame, constraint_ids: set[str] +) -> pd.DataFrame: + """Drops (and logs) relaxation options whose constraint is not in the + model — there is nothing for them to relax.""" + missing = set(relaxations["expansion_id"]) - constraint_ids + if missing: + logger.info( + f"Constraint relaxation expansion options dropped (their constraints " + f"are not in the model): {sorted(missing)}" + ) + return relaxations[~relaxations["expansion_id"].isin(missing)] + + +def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: + """Adds the PyPSA generator attributes shared by all relaxation generators. + + I/O Example: + expansion_id=SWQLD1, year=2030, capital_cost=8140, allowed_expansion=500 + -> name=SWQLD1_exp_2030, isp_name=SWQLD1, p_nom=0.0, + p_nom_extendable=True, build_year=2030, lifetime=inf + """ + generators = generators.rename(columns={"expansion_id": "isp_name"}) + generators["name"] = ( + generators["isp_name"] + "_exp_" + generators["year"].astype(str) + ) + generators["bus"] = "bus_for_custom_constraint_gens" + generators["p_nom"] = 0.0 + generators["p_nom_extendable"] = True + generators["build_year"] = generators["year"] + generators["lifetime"] = np.inf + return generators.loc[:, _GENERATOR_COLUMNS + ["allowed_expansion"]] + + +def _relaxation_generator_lhs_terms( + relaxation_generators: pd.DataFrame, investment_periods: list[int] +) -> pd.DataFrame: + """LHS terms subtracting each relaxation generator's capacity from its + parent constraint. + + Terms are per investment period and only include generators already built + by that period — capacity built in a later period can't relax an earlier + period's constraint. + + I/O Example: + relaxation_generators: + name isp_name build_year + SWQLD1_exp_2030 SWQLD1 2030 + SWQLD1_exp_2040 SWQLD1 2040 + + investment_periods=[2030, 2040] returns: + constraint_id investment_period variable_name component attribute coefficient + SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2040 Generator p_nom -1.0 + """ + terms = [] + for period in investment_periods: + built = relaxation_generators[ + relaxation_generators["build_year"] <= period + ].copy() + built["investment_period"] = period + terms.append(built) + terms = pd.concat(terms, ignore_index=True) + terms = terms.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + terms["component"] = "Generator" + terms["attribute"] = "p_nom" + terms["coefficient"] = -1.0 + columns = [c for c in _LHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return terms.loc[:, columns] + + +def _create_expansion_limit_constraints( + links: pd.DataFrame, + relaxation_generators: pd.DataFrame, + expansion_options: pd.DataFrame, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Caps the total capacity built across each expandable element's + per-period components at the selected option's capacity. + + For physical paths the cap is the option's forward capacity (reverse + capacity scales with it through each expansion link's p_min_pu); for + constraint relaxations it is the option's allowed_expansion. The + constraints have no investment_period or timeslice — they apply to the + p_nom variables globally. Names get an "_expansion_limit" suffix so a + relaxation cap doesn't collide with the constraint it relaxes. + + I/O Example: + links (extendable): CQ-NQ_exp_2030, CQ-NQ_exp_2040 (isp_name CQ-NQ) + expansion_options: CQ-NQ forward 1000 / reverse 900 + + returns lhs: + constraint_id variable_name component attribute coefficient investment_period + CQ-NQ_expansion_limit CQ-NQ_exp_2030 Link p_nom 1.0 NaN + CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN + + and rhs: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ_expansion_limit 1000 <= NaN NaN + """ + lhs = pd.concat( + [ + _expansion_limit_lhs(links[links["p_nom_extendable"]], "Link"), + _expansion_limit_lhs(relaxation_generators, "Generator"), + ], + ignore_index=True, + ) + rhs = _expansion_limit_rhs(expansion_options, set(lhs["constraint_id"])) + lhs["constraint_id"] = lhs["constraint_id"] + "_expansion_limit" + rhs["constraint_id"] = rhs["constraint_id"] + "_expansion_limit" + return lhs, rhs + + +def _expansion_limit_lhs(components: pd.DataFrame, component_type: str) -> pd.DataFrame: + """One LHS term per expandable component, summing p_nom across the + investment periods of its parent element. + + I/O Example: + components: name=CQ-NQ_exp_2030, isp_name=CQ-NQ; component_type="Link" + -> constraint_id=CQ-NQ, variable_name=CQ-NQ_exp_2030, component=Link, + attribute=p_nom, coefficient=1.0, investment_period=NaN + """ + lhs = components.loc[:, ["isp_name", "name"]].copy() + lhs = lhs.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + lhs["component"] = component_type + lhs["attribute"] = "p_nom" + lhs["coefficient"] = 1.0 + lhs["investment_period"] = np.nan + return lhs + + +def _expansion_limit_rhs( + expansion_options: pd.DataFrame, expandable_ids: set[str] +) -> pd.DataFrame: + """One RHS row per expandable element with components in the model: the + selected option's forward capacity (physical paths) or allowed_expansion + (constraint relaxations). + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion + CQ-NQ forward 1000 + CQ-NQ reverse 900 + SWQLD1 constraint_relaxation 500 + + expandable_ids={"CQ-NQ", "SWQLD1"} returns: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ 1000 <= NaN NaN + SWQLD1 500 <= NaN NaN + """ + caps = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "constraint_relaxation"]) + ] + caps = caps[caps["expansion_id"].isin(expandable_ids)] + rhs = caps.rename(columns={"expansion_id": "constraint_id"}).copy() + rhs["rhs"] = rhs["allowed_expansion"] + rhs["constraint_type"] = "<=" + rhs["investment_period"] = np.nan + rhs["timeslice"] = np.nan + columns = [c for c in _RHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return rhs.loc[:, columns] + + +def _finalise_lhs_and_rhs( + lhs: pd.DataFrame, rhs: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Suffixes expansion-limit names, validates the pairing, and sets the + final PyPSA friendly column orders.""" + lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) + rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) + _raise_on_duplicate_rhs_rows(rhs) + _raise_on_unpaired_constraints(lhs, rhs) + return ( + lhs.loc[:, _LHS_COLUMNS].reset_index(drop=True), + rhs.loc[:, _RHS_COLUMNS].reset_index(drop=True), + ) + + +def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: + """Raise on duplicate (constraint, period, timeslice) RHS rows — pypsa_build + would create two constraints with the same name.""" + keys = ["constraint_name", "investment_period", "timeslice"] + duplicates = rhs[rhs.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint RHS rows for: " + f"{sorted(set(duplicates['constraint_name']))}" + ) + + +def _raise_on_unpaired_constraints(lhs: pd.DataFrame, rhs: pd.DataFrame) -> None: + """Raise if any constraint appears on only one side — a one-sided + constraint can't be applied.""" + lhs_names = set(lhs["constraint_name"]) + rhs_names = set(rhs["constraint_name"]) + if lhs_names != rhs_names: + raise ValueError( + f"Custom constraints with LHS terms but no RHS: " + f"{sorted(lhs_names - rhs_names)}; with RHS but no LHS terms: " + f"{sorted(rhs_names - lhs_names)}" + ) + + +def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: + """Drops the allowed_expansion working column carried for the + expansion-limit RHS, leaving the PyPSA generator columns.""" + return relaxation_generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index ed6e25b2..615203b5 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -152,6 +152,10 @@ "generator_capacity": "Generator", "generator_output": "Generator", "load_consumption": "Load", + # "load" is the new-format vocabulary for "load_consumption" (PLEXOS Node + # Load Coefficient terms). Load variables aren't implemented in + # pypsa_build, which skips these terms with a log line. + "load": "Load", "storage_output": "Storage", } @@ -160,6 +164,7 @@ "generator_capacity": "p_nom", "generator_output": "p", "load_consumption": "p", + "load": "p", "storage_output": "p", } diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py new file mode 100644 index 00000000..0aa7d64b --- /dev/null +++ b/tests/test_translator/test_constraints.py @@ -0,0 +1,327 @@ +import pandas as pd +import pytest + +from ispypsa.translator.constraints import ( + _translate_custom_constraints_from_network_tables, +) +from ispypsa.translator.helpers import _annuitised_investment_costs + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25). +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def _constraint_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """One PLEXOS-derived constraint (SWQLD1) with a link, a generator and a + storage term, timeslice-varying RHS, and a relaxation expansion option. + The sample_model_config's investment periods are 2026 and 2028.""" + tables = {} + tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """) + tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, NSW-QLD, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, storage_output, Q8 Battery - 2h, 0.43, + """) + tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_winter_reference, 3500, + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + NSW-QLD, 2026, 500000 + SWQLD1, 2026, 100000 + """) + return tables + + +def _links(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + isp_name, name, p_nom_extendable + NSW-QLD, NSW-QLD_existing, False + NSW-QLD, NSW-QLD_exp_2026, True + """) + + +def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, qld_winter_reference, 3500, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_winter_reference, 3500, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_relaxation_generators( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], + expected_generators, + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_custom_constraints_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], expected_generators, check_dtype=False + ) + lhs = result["custom_constraints_lhs"] + assert "SWQLD1_exp_2026" not in set(lhs["variable_name"]) + + +def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config): + """A value dated mid-FY2027 (i.e. before FY2028 starts on 2027-07-01) does + not apply in the 2026 period but does in 2028.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, 2026-12-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + rhs = rhs[rhs["constraint_name"] == "SWQLD1"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_peak_demand, 2500, <= + """) + pd.testing.assert_frame_equal( + rhs.sort_values("investment_period").reset_index(drop=True), + expected, + check_dtype=False, + ) + + +def test_date_from_after_all_periods_contributes_nothing( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, generator_output, LATEGEN, 0.5, 2040-01-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert "LATEGEN" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, = + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + assert set(rhs.loc[rhs["constraint_name"] == "SWQLD1", "constraint_type"]) == {"=="} + + +def test_link_terms_not_in_model_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, TAS-SEV, 0.5, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraint link_flow terms dropped (paths not in model): ['TAS-SEV']" + ) in caplog.text + assert "TAS-SEV" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_constraint_with_no_lhs_terms_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + NQ1, <= + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + NQ1, qld_peak_demand, 2650, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraints dropped (no LHS terms in model): ['NQ1']" + ) in caplog.text + assert "NQ1" not in set(result["custom_constraints_rhs"]["constraint_name"]) + + +def test_raises_on_rhs_without_direction(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + """) + + with pytest.raises(ValueError, match=r"no direction.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_raises_on_duplicate_rhs_rows(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, + """) + + with pytest.raises(ValueError, match=r"Duplicate custom constraint RHS.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): + """At coarser granularities the custom-constraint tables are header-only; + the expansion-limit constraints for links are still produced.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = pd.DataFrame( + columns=["constraint_id", "direction"] + ) + ispypsa_tables["custom_constraints_lhs"] = pd.DataFrame( + columns=[ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + ] + ) + ispypsa_tables["custom_constraints_rhs"] = pd.DataFrame( + columns=["constraint_id", "timeslice", "rhs", "date_from"] + ) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, Option 1 + NSW-QLD, reverse, 900, Option 1 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + """) + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"], expected_lhs, check_dtype=False + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NSW-QLD_expansion_limit, , , 1000, <= + """) + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"], expected_rhs, check_dtype=False + ) From dcd82a0097f1d7ebfde678e8ba12a36fd937bb31 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 15:02:01 +1000 Subject: [PATCH 02/20] Resolve expansion options and costs through the wildcard convention #126 changed the expansion tables to allow blank key cells as wildcards, changed _prepare_expansion_costs to take the enabled element ids, and made expansion links carry forward/reverse per unit of max(forward, reverse). The cherry-picked module still read unresolved options, capped paths at their forward capacity and imported a helper #126 deleted. The expansion-limit cap is now max(forward, reverse), re-resolved for the paths that have expansion links with the network module's own helpers, and constraint relaxations resolve their options and costs against the constraints in the model so blank-id defaults apply to every constraint. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 408 +++++++++++++++++----- tests/test_translator/test_constraints.py | 131 +++++++ 2 files changed, 442 insertions(+), 97 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 1c6cff04..9cfa93bf 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -1,18 +1,97 @@ -"""Translates the new-format custom-constraint tables into PyPSA friendly form. - -Consumes ``custom_constraints`` (constraint senses), ``custom_constraints_lhs`` -(terms) and ``custom_constraints_rhs`` (limit values), plus the unified -network expansion tables for constraint-relaxation generators and -expansion-limit constraints. - -The PyPSA friendly outputs carry two temporal columns the old format lacked: - -- ``investment_period``: time-varying inputs (``date_from``) are resolved by - holding, for each investment period, the value active at the period's - start. Rows with a NaN investment_period apply in every period. -- ``timeslice`` (RHS only): the constraint binds only at snapshots inside the - timeslice's windows (see ispypsa.translator.timeslices). NaN means the - limit applies at all snapshots. +"""Translate the new-format custom-constraint tables into PyPSA friendly form. + +This module sits in the translator stage alongside ispypsa.translator.network. +It turns the templated custom-constraint tables (PLEXOS-derived group +constraints such as SWQLD1) into the LHS/RHS tables pypsa_build applies as +linopy constraints, and adds the endogenous expansion-limit constraints, with +their constraint-relaxation generators, that cap how much capacity the model +can build for each expandable network element. + +The inputs are the three custom-constraint tables — custom_constraints (one +row per constraint), custom_constraints_lhs (one row per term per date_from) +and custom_constraints_rhs (one row per timeslice per date_from): + + custom_constraints: custom_constraints_rhs: + constraint_id direction constraint_id timeslice rhs date_from + SWQLD1 <= SWQLD1 qld_peak_demand 3000 + SWQLD1 qld_peak_demand 2500 2027-07-01T00:00:00 + + custom_constraints_lhs: + constraint_id term_type variable_name coefficient date_from + SWQLD1 link_flow NSW-QLD 0.84 + SWQLD1 generator_output KINGASF1 0.14 + +plus network_expansion_options and network_transmission_path_expansion_costs +(the unified expansion tables, see ispypsa.translator.network) and the PyPSA +friendly links table (existing plus expansion links). + +The outputs are the PyPSA friendly custom_constraints_rhs (one row per +constraint, investment period and timeslice), custom_constraints_lhs (one row +per constraint, investment period and term) and custom_constraints_generators +(one row per relaxable constraint and investment period): + + custom_constraints_rhs: + constraint_name investment_period timeslice rhs constraint_type + SWQLD1 2026 qld_peak_demand 3000 <= + SWQLD1 2028 qld_peak_demand 2500 <= + NSW-QLD_expansion_limit 1000 <= + SWQLD1_expansion_limit 400 <= + + custom_constraints_lhs (2028 rows mirror 2026): + constraint_name investment_period variable_name component attribute coefficient + SWQLD1 2026 NSW-QLD_existing Link p 0.84 + SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 + SWQLD1 2026 KINGASF1 Generator p 0.14 + SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 + NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 + SWQLD1_expansion_limit SWQLD1_exp_2026 Generator p_nom 1.0 + + custom_constraints_generators (abridged): + name isp_name bus p_nom p_nom_extendable build_year capital_cost + SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2026 annuitise(100000) + +A blank investment_period means the row applies in every period; a blank +timeslice means the RHS binds at every snapshot (otherwise only at snapshots +inside the timeslice's windows, see ispypsa.translator.timeslices). + +The pipeline runs as follows. Duplicate input rows are rejected first, since +they would silently collapse during date resolution. Then the date_from +column of the LHS and RHS tables is resolved into one row per investment +period: for each period, each group (a constraint's term, or a constraint's +timeslice) keeps the row active at the period's start — the latest date_from +on or before it, with no-date_from rows as the baseline. The RHS gains each +constraint's sense from custom_constraints as constraint_type. LHS term_types +map to the PyPSA component and attribute their variable belongs to, and each +link_flow term is expanded from its path_id to every link the model has on +that path — the existing link and each expansion link — so flow through new +builds counts towards the constraint too. Terms for paths not in the model +are dropped, and constraints left with no LHS terms are dropped with them. + +Constraint relaxation comes next, gated by the config's +rez_transmission_expansion flag. Each constraint that has a +constraint_relaxation expansion option gets one extendable dummy generator +per investment period at the option's annualised cost; the generator's p_nom +enters the parent constraint's LHS with coefficient -1.0, so building it +relaxes the constraint. Finally the expansion-limit constraints cap the total +p_nom built across each expandable element's per-period components: for a +path the cap is max(forward, reverse) of its option, matching the per-unit +ratings ispypsa.translator.network gives its expansion links; for a +relaxation it is the option's allowed_expansion. Both tables are then +finalised — constraint_id becomes constraint_name and one-sided or duplicate +constraints are rejected. + +Reference detail: + +- direction to constraint_type: "<=" and ">=" pass through, "=" becomes "==". +- term_type to component/attribute lives in ispypsa.translator.mappings + (_CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE and _..._ATTRIBUTE_TYPE). +- Expansion-limit constraints are named "_expansion_limit" so a + relaxation cap doesn't collide with the constraint it relaxes. +- Dropped rows: link_flow terms whose path is not in the model (logged); RHS + rows for constraints with no LHS terms (logged); relaxation options and + costs for constraints not in the model, or all of them when + rez_transmission_expansion is off; date_from rows that only start after + every investment period. """ import logging @@ -21,13 +100,16 @@ import pandas as pd from ispypsa.config import ModelConfig +from ispypsa.translator.helpers import _resolve_wildcards from ispypsa.translator.mappings import ( _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE, _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, ) from ispypsa.translator.network import ( - _pivot_physical_expansion_options, + _keep_rows_for_enabled_elements, + _pair_forward_and_reverse_options, _prepare_expansion_costs, + _resolve_expansion_options, ) logger = logging.getLogger(__name__) @@ -86,7 +168,12 @@ def _raise_on_duplicate_input_rows( def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.DataFrame: """Concatenates the non-empty frames (concatenating all-empty frames is - deprecated by pandas), returning a header-only frame when all are empty.""" + deprecated by pandas), returning a header-only frame when all are empty. + + I/O Example: + [empty, 2-row frame, empty], columns -> the 2-row frame + [empty, empty], columns -> header-only frame with columns + """ non_empty = [frame for frame in frames if not frame.empty] if not non_empty: return pd.DataFrame(columns=columns) @@ -97,32 +184,63 @@ def _translate_custom_constraints_from_network_tables( ispypsa_tables: dict[str, pd.DataFrame], links: pd.DataFrame, config: ModelConfig, - generators: pd.DataFrame | None = None, - batteries: pd.DataFrame | None = None, ) -> dict[str, pd.DataFrame]: """Translates the custom-constraint tables and builds the endogenous expansion-limit constraints. - Generator and battery terms pass through with their IASR IDs as + Consumes the custom_constraints, custom_constraints_lhs, + custom_constraints_rhs, network_expansion_options and + network_transmission_path_expansion_costs tables, plus the PyPSA friendly + links table from ispypsa.translator.network (existing plus expansion + links). Generator and battery terms pass through with their IASR IDs as variable_names: pypsa_build skips (and logs) terms whose components are - not in the model. The ``generators`` and ``batteries`` arguments are - accepted so the templated generator/battery tables can be used to map - those IDs to model component names once generator templating lands; they - are currently unused. - - Args: - ispypsa_tables: dictionary of new-format `ISPyPSA` input tables; - consumes custom_constraints, custom_constraints_lhs, - custom_constraints_rhs, network_expansion_options and - network_transmission_path_expansion_costs. - links: PyPSA friendly links table from - ispypsa.translator.network (existing plus expansion links). - config: `ispypsa.config.ModelConfig` object. - generators: templated generator table (future name-mapping hook). - batteries: templated battery table (future name-mapping hook). - - Returns: dict with the PyPSA friendly custom_constraints_lhs, - custom_constraints_rhs and custom_constraints_generators tables. + not in the model. + + I/O Example (config: investment periods 2026 and 2028): + custom_constraints: custom_constraints_rhs: + constraint_id direction constraint_id timeslice rhs date_from + SWQLD1 <= SWQLD1 qld_peak_demand 3000 + + custom_constraints_lhs: + constraint_id term_type variable_name coefficient date_from + SWQLD1 link_flow NSW-QLD 0.84 + SWQLD1 generator_output KINGASF1 0.14 + + network_expansion_options: + expansion_id expansion_type allowed_expansion expansion_option + NSW-QLD forward 1000 Option 1 + NSW-QLD reverse 900 Option 1 + SWQLD1 constraint_relaxation 400 Option 2 + + network_transmission_path_expansion_costs: + expansion_id year cost + NSW-QLD 2026 500000 + SWQLD1 2026 100000 + + links: + isp_name name p_nom_extendable + NSW-QLD NSW-QLD_existing False + NSW-QLD NSW-QLD_exp_2026 True + + returns custom_constraints_rhs: + constraint_name investment_period timeslice rhs constraint_type + SWQLD1 2026 qld_peak_demand 3000 <= + SWQLD1 2028 qld_peak_demand 3000 <= + NSW-QLD_expansion_limit 1000 <= + SWQLD1_expansion_limit 400 <= + + custom_constraints_lhs (2028 rows mirror 2026): + constraint_name investment_period variable_name component attribute coefficient + SWQLD1 2026 NSW-QLD_existing Link p 0.84 + SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 + SWQLD1 2026 KINGASF1 Generator p 0.14 + SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 + NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 + SWQLD1_expansion_limit SWQLD1_exp_2026 Generator p_nom 1.0 + + custom_constraints_generators (abridged): + name isp_name bus p_nom build_year capital_cost + SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 2026 annuitise(100000) """ _raise_on_duplicate_input_rows( ispypsa_tables["custom_constraints_rhs"], @@ -154,14 +272,17 @@ def _translate_custom_constraints_from_network_tables( rhs = _drop_rhs_without_lhs_terms(rhs, lhs) relaxation_generators = _create_constraint_relaxation_generators( - ispypsa_tables, set(rhs["constraint_id"]), config + ispypsa_tables, sorted(set(rhs["constraint_id"])), config ) relaxation_generator_lhs = _relaxation_generator_lhs_terms( relaxation_generators, config.temporal.capacity_expansion.investment_periods, ) + path_caps = _resolve_path_expansion_caps( + ispypsa_tables["network_expansion_options"], links + ) expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( - links, relaxation_generators, ispypsa_tables["network_expansion_options"] + links, relaxation_generators, path_caps ) lhs = _concat_non_empty( @@ -315,6 +436,8 @@ def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFr def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) -> None: + """Logs the link_flow terms whose path has no link in the model (they are + dropped by the merge in _expand_link_flow_terms).""" missing = set(link_terms["variable_name"]) - set(links["isp_name"]) if missing: logger.info( @@ -325,7 +448,12 @@ def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.DataFrame: """Drops (and logs) RHS rows for constraints left with no LHS terms — a - constraint with an empty LHS can't be applied.""" + constraint with an empty LHS can't be applied. + + I/O Example: + rhs constraint_ids {SWQLD1, NQ1}, lhs constraint_ids {SWQLD1} + -> NQ1's RHS rows dropped and logged; SWQLD1's kept + """ without_lhs = set(rhs["constraint_id"]) - set(lhs["constraint_id"]) if without_lhs: logger.info( @@ -336,7 +464,7 @@ def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.Data def _create_constraint_relaxation_generators( ispypsa_tables: dict[str, pd.DataFrame], - constraint_ids: set[str], + constraint_ids: list[str], config: ModelConfig, ) -> pd.DataFrame: """Builds one extendable dummy generator per relaxable constraint and @@ -345,30 +473,33 @@ def _create_constraint_relaxation_generators( The generators' p_nom enters the parent constraint's LHS with coefficient -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the constraint at the option's cost; total relaxation is capped at the - option's allowed_expansion by the expansion-limit constraints. + option's allowed_expansion by the expansion-limit constraints. The + constraints in the model (constraint_ids) are the enabled elements the + options and costs wildcards resolve against, gated as a whole by the + config's rez_transmission_expansion flag. I/O Example: network_expansion_options: - expansion_id expansion_type allowed_expansion - SWQLD1 constraint_relaxation 500 + expansion_id expansion_type allowed_expansion expansion_option + SWQLD1 constraint_relaxation 500 Option 2 network_transmission_path_expansion_costs: expansion_id year cost SWQLD1 2030 100000 - constraint_ids={"SWQLD1"}, investment_periods=[2030] returns (abridged): - name isp_name bus p_nom build_year - SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 + constraint_ids=["SWQLD1"], investment_periods=[2030] returns (abridged): + name isp_name bus p_nom build_year allowed_expansion + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 500 """ if not config.network.rez_transmission_expansion: return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) - options = ispypsa_tables["network_expansion_options"] - relaxations = options[options["expansion_type"] == "constraint_relaxation"] - relaxations = _drop_relaxations_without_constraints(relaxations, constraint_ids) + relaxations = _resolve_relaxation_options( + ispypsa_tables["network_expansion_options"], constraint_ids + ) costs = _prepare_expansion_costs( ispypsa_tables["network_transmission_path_expansion_costs"], + constraint_ids, config.temporal.capacity_expansion.investment_periods, - config.temporal.year_type, config.wacc, config.network.annuitisation_lifetime, ) @@ -378,18 +509,44 @@ def _create_constraint_relaxation_generators( return _format_relaxation_generators(generators) -def _drop_relaxations_without_constraints( - relaxations: pd.DataFrame, constraint_ids: set[str] +def _resolve_relaxation_options( + options: pd.DataFrame, constraint_ids: list[str] ) -> pd.DataFrame: - """Drops (and logs) relaxation options whose constraint is not in the - model — there is nothing for them to relax.""" - missing = set(relaxations["expansion_id"]) - constraint_ids - if missing: - logger.info( - f"Constraint relaxation expansion options dropped (their constraints " - f"are not in the model): {sorted(missing)}" - ) - return relaxations[~relaxations["expansion_id"].isin(missing)] + """Resolves the expansion-options wildcards to one constraint_relaxation + row per constraint in the model that has an option. + + The physical forward/reverse rows are set aside first (they become + expansion links in ispypsa.translator.network); a blank expansion_type + covers constraint_relaxation too, so it is kept. Options for constraints + not in the model are dropped, then _resolve_wildcards fans blank cells out + against the model's constraints, most specific row winning. + + I/O Example (blank cells are wildcards): + options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 BigLine # physical: set aside + SWQLD1 constraint_relaxation 400 Relax + constraint_relaxation 200 Default # blank id: every constraint + + constraint_ids = ["SWQLD1", "SWV1"] + + returns: + expansion_id expansion_type allowed_expansion expansion_option + SWQLD1 constraint_relaxation 400 Relax + SWV1 constraint_relaxation 200 Default + """ + expansion_type = options["expansion_type"] + options = options[ + expansion_type.isna() | (expansion_type == "constraint_relaxation") + ] + options = _keep_rows_for_enabled_elements(options, constraint_ids) + allowed_values = { + "expansion_id": constraint_ids, + "expansion_type": ["constraint_relaxation"], + } + return _resolve_wildcards( + options, allowed_values, ["allowed_expansion", "expansion_option"] + ) def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: @@ -450,33 +607,85 @@ def _relaxation_generator_lhs_terms( return terms.loc[:, columns] +def _resolve_path_expansion_caps( + options: pd.DataFrame, links: pd.DataFrame +) -> pd.DataFrame: + """The capacity cap for each path with expansion links in the model: + max(forward, reverse) of its resolved expansion option. + + Each expansion link's p_max_pu and p_min_pu are the option's forward and + reverse capacities per unit of that max (see + ispypsa.translator.network._build_expansion_links), so capping the p_nom + built across a path's expansion links at the max delivers the option's + full capacity in both directions. The paths with extendable links are the + enabled elements the options wildcards resolve against — the same set + ispypsa.translator.network resolved them for. + + I/O Example: + options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 800 BigLine + CQ-NQ reverse 1000 BigLine + + links: + isp_name name p_nom_extendable + CQ-NQ CQ-NQ_existing False + CQ-NQ CQ-NQ_exp_2030 True + + returns: + expansion_id allowed_expansion + CQ-NQ 1000 + """ + expandable_ids = sorted(set(links.loc[links["p_nom_extendable"], "isp_name"])) + options = _resolve_expansion_options(options, expandable_ids) + options = _pair_forward_and_reverse_options(options) + caps = options.loc[:, ["expansion_id"]].copy() + caps["allowed_expansion"] = options[["forward_capacity", "reverse_capacity"]].max( + axis=1 + ) + return caps + + def _create_expansion_limit_constraints( links: pd.DataFrame, relaxation_generators: pd.DataFrame, - expansion_options: pd.DataFrame, + path_caps: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Caps the total capacity built across each expandable element's per-period components at the selected option's capacity. - For physical paths the cap is the option's forward capacity (reverse - capacity scales with it through each expansion link's p_min_pu); for - constraint relaxations it is the option's allowed_expansion. The - constraints have no investment_period or timeslice — they apply to the - p_nom variables globally. Names get an "_expansion_limit" suffix so a - relaxation cap doesn't collide with the constraint it relaxes. + For physical paths the cap is path_caps' max(forward, reverse); for + constraint relaxations it is the option's allowed_expansion carried on the + relaxation generators. The constraints have no investment_period or + timeslice — they apply to the p_nom variables globally. Names get an + "_expansion_limit" suffix so a relaxation cap doesn't collide with the + constraint it relaxes. I/O Example: - links (extendable): CQ-NQ_exp_2030, CQ-NQ_exp_2040 (isp_name CQ-NQ) - expansion_options: CQ-NQ forward 1000 / reverse 900 + links: + isp_name name p_nom_extendable + CQ-NQ CQ-NQ_existing False + CQ-NQ CQ-NQ_exp_2030 True + CQ-NQ CQ-NQ_exp_2040 True + + relaxation_generators: + name isp_name allowed_expansion + SWQLD1_exp_2030 SWQLD1 500 + + path_caps: + expansion_id allowed_expansion + CQ-NQ 1000 returns lhs: - constraint_id variable_name component attribute coefficient investment_period - CQ-NQ_expansion_limit CQ-NQ_exp_2030 Link p_nom 1.0 NaN - CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN + constraint_id variable_name component attribute coefficient investment_period + CQ-NQ_expansion_limit CQ-NQ_exp_2030 Link p_nom 1.0 NaN + CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN + SWQLD1_expansion_limit SWQLD1_exp_2030 Generator p_nom 1.0 NaN and rhs: - constraint_id rhs constraint_type investment_period timeslice - CQ-NQ_expansion_limit 1000 <= NaN NaN + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ_expansion_limit 1000 <= NaN NaN + SWQLD1_expansion_limit 500 <= NaN NaN """ lhs = pd.concat( [ @@ -485,7 +694,10 @@ def _create_expansion_limit_constraints( ], ignore_index=True, ) - rhs = _expansion_limit_rhs(expansion_options, set(lhs["constraint_id"])) + relaxation_caps = relaxation_generators.loc[:, ["isp_name", "allowed_expansion"]] + relaxation_caps = relaxation_caps.rename(columns={"isp_name": "expansion_id"}) + caps = pd.concat([path_caps, relaxation_caps.drop_duplicates()], ignore_index=True) + rhs = _expansion_limit_rhs(caps) lhs["constraint_id"] = lhs["constraint_id"] + "_expansion_limit" rhs["constraint_id"] = rhs["constraint_id"] + "_expansion_limit" return lhs, rhs @@ -509,29 +721,21 @@ def _expansion_limit_lhs(components: pd.DataFrame, component_type: str) -> pd.Da return lhs -def _expansion_limit_rhs( - expansion_options: pd.DataFrame, expandable_ids: set[str] -) -> pd.DataFrame: - """One RHS row per expandable element with components in the model: the - selected option's forward capacity (physical paths) or allowed_expansion - (constraint relaxations). +def _expansion_limit_rhs(caps: pd.DataFrame) -> pd.DataFrame: + """One RHS row per expandable element, capping its components' total p_nom + at the element's allowed_expansion. I/O Example: - expansion_options: - expansion_id expansion_type allowed_expansion - CQ-NQ forward 1000 - CQ-NQ reverse 900 - SWQLD1 constraint_relaxation 500 + caps: + expansion_id allowed_expansion + CQ-NQ 1000 + SWQLD1 500 - expandable_ids={"CQ-NQ", "SWQLD1"} returns: + returns: constraint_id rhs constraint_type investment_period timeslice CQ-NQ 1000 <= NaN NaN SWQLD1 500 <= NaN NaN """ - caps = expansion_options[ - expansion_options["expansion_type"].isin(["forward", "constraint_relaxation"]) - ] - caps = caps[caps["expansion_id"].isin(expandable_ids)] rhs = caps.rename(columns={"expansion_id": "constraint_id"}).copy() rhs["rhs"] = rhs["allowed_expansion"] rhs["constraint_type"] = "<=" @@ -544,8 +748,14 @@ def _expansion_limit_rhs( def _finalise_lhs_and_rhs( lhs: pd.DataFrame, rhs: pd.DataFrame ) -> tuple[pd.DataFrame, pd.DataFrame]: - """Suffixes expansion-limit names, validates the pairing, and sets the - final PyPSA friendly column orders.""" + """Renames constraint_id to constraint_name, validates the LHS/RHS + pairing, and sets the final PyPSA friendly column orders. + + I/O Example: + lhs: constraint_id=SWQLD1, ... rhs: constraint_id=SWQLD1, ... + -> lhs.columns == _LHS_COLUMNS, rhs.columns == _RHS_COLUMNS, both + keyed by constraint_name=SWQLD1 + """ lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) _raise_on_duplicate_rhs_rows(rhs) @@ -583,5 +793,9 @@ def _raise_on_unpaired_constraints(lhs: pd.DataFrame, rhs: pd.DataFrame) -> None def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: """Drops the allowed_expansion working column carried for the - expansion-limit RHS, leaving the PyPSA generator columns.""" + expansion-limit RHS, leaving the PyPSA generator columns. + + I/O Example: + columns [*_GENERATOR_COLUMNS, allowed_expansion] -> _GENERATOR_COLUMNS + """ return relaxation_generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 0aa7d64b..d8389e1c 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -325,3 +325,134 @@ def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): pd.testing.assert_frame_equal( result["custom_constraints_rhs"], expected_rhs, check_dtype=False ) + + +def test_path_expansion_limit_is_max_of_forward_and_reverse( + csv_str_to_df, sample_model_config +): + """Expansion links carry forward/reverse per unit of max(forward, reverse), + so the cap on their total p_nom must be that max, not the forward value.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 800, NSW-QLD Option 1 + NSW-QLD, reverse, 1000, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NSW-QLD_expansion_limit, , , 1000, <= + """) + pd.testing.assert_frame_equal( + rhs[rhs["constraint_name"] == "NSW-QLD_expansion_limit"].reset_index(drop=True), + expected, + check_dtype=False, + ) + + +def test_wildcard_relaxation_option_and_cost_apply_to_every_constraint( + csv_str_to_df, sample_model_config +): + """A blank expansion_id relaxation option (and a blank expansion_id, blank + year cost) is a default for every constraint in the model; a specific + row still wins for its own constraint.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + NQ1, <= + """) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + NQ1, generator_output, KINGASF1, 0.5, + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + NQ1, qld_peak_demand, 2650, + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + , constraint_relaxation, 200, Default relaxation + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + NSW-QLD, 2026, 500000 + , , 100000 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + NQ1_exp_2026, NQ1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + NQ1_exp_2028, NQ1, bus_for_custom_constraint_gens, 0.0, True, 2028, inf, {100000 * _ANNUITY_PER_DOLLAR} + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + SWQLD1_exp_2028, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2028, inf, {100000 * _ANNUITY_PER_DOLLAR} + """) + generators = result["custom_constraints_generators"] + pd.testing.assert_frame_equal( + generators.sort_values("name").reset_index(drop=True), + expected_generators, + check_dtype=False, + rtol=1e-5, + ) + rhs = result["custom_constraints_rhs"] + expected_limits = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NQ1_expansion_limit, , , 200, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + limits = rhs[rhs["constraint_name"].str.endswith("_expansion_limit")] + pd.testing.assert_frame_equal( + limits.sort_values("constraint_name").reset_index(drop=True), + expected_limits, + check_dtype=False, + ) + + +def test_relaxation_option_for_constraint_not_in_model_is_dropped( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + NQ1, constraint_relaxation, 300, NQ1 Option 1 + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + NSW-QLD, 2026, 500000 + SWQLD1, 2026, 100000 + NQ1, 2026, 100000 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], + expected_generators, + check_dtype=False, + rtol=1e-5, + ) From c95de770a80aa4d9854b670d0a2a0e0911c2e3f6 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 15:19:44 +1000 Subject: [PATCH 03/20] Label the dict-packaged tables in the constraints I/O examples Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 9cfa93bf..3996697d 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -197,22 +197,26 @@ def _translate_custom_constraints_from_network_tables( not in the model. I/O Example (config: investment periods 2026 and 2028): - custom_constraints: custom_constraints_rhs: - constraint_id direction constraint_id timeslice rhs date_from - SWQLD1 <= SWQLD1 qld_peak_demand 3000 + ispypsa_tables["custom_constraints"]: + constraint_id direction + SWQLD1 <= - custom_constraints_lhs: + ispypsa_tables["custom_constraints_rhs"]: + constraint_id timeslice rhs date_from + SWQLD1 qld_peak_demand 3000 + + ispypsa_tables["custom_constraints_lhs"]: constraint_id term_type variable_name coefficient date_from SWQLD1 link_flow NSW-QLD 0.84 SWQLD1 generator_output KINGASF1 0.14 - network_expansion_options: + ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option NSW-QLD forward 1000 Option 1 NSW-QLD reverse 900 Option 1 SWQLD1 constraint_relaxation 400 Option 2 - network_transmission_path_expansion_costs: + ispypsa_tables["network_transmission_path_expansion_costs"]: expansion_id year cost NSW-QLD 2026 500000 SWQLD1 2026 100000 @@ -222,14 +226,14 @@ def _translate_custom_constraints_from_network_tables( NSW-QLD NSW-QLD_existing False NSW-QLD NSW-QLD_exp_2026 True - returns custom_constraints_rhs: + returns["custom_constraints_rhs"]: constraint_name investment_period timeslice rhs constraint_type SWQLD1 2026 qld_peak_demand 3000 <= SWQLD1 2028 qld_peak_demand 3000 <= NSW-QLD_expansion_limit 1000 <= SWQLD1_expansion_limit 400 <= - custom_constraints_lhs (2028 rows mirror 2026): + returns["custom_constraints_lhs"] (2028 rows mirror 2026): constraint_name investment_period variable_name component attribute coefficient SWQLD1 2026 NSW-QLD_existing Link p 0.84 SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 @@ -238,7 +242,7 @@ def _translate_custom_constraints_from_network_tables( NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 SWQLD1_expansion_limit SWQLD1_exp_2026 Generator p_nom 1.0 - custom_constraints_generators (abridged): + returns["custom_constraints_generators"] (abridged): name isp_name bus p_nom build_year capital_cost SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 2026 annuitise(100000) """ @@ -479,11 +483,11 @@ def _create_constraint_relaxation_generators( config's rez_transmission_expansion flag. I/O Example: - network_expansion_options: + ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option SWQLD1 constraint_relaxation 500 Option 2 - network_transmission_path_expansion_costs: + ispypsa_tables["network_transmission_path_expansion_costs"]: expansion_id year cost SWQLD1 2030 100000 From 8acb0b0ad2c616b193ffdda8ea04f99b83fecfe9 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 18 Aug 2026 16:16:36 +1000 Subject: [PATCH 04/20] Name the expansion-id filter for what it does, and fill out the constraint docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _keep_rows_for_enabled_elements is shared by the network and constraints translators, but "enabled elements" only described the network caller — the constraints side passes constraint_ids. Renamed to _keep_rows_for_expansion_ids and reworded the docstrings so each caller's selection is described where the decision is made. The constraint helpers' I/O examples now show full tables, including the blank-cell wildcard forms the options and costs accept. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 84 ++++++++++++++++++++------- src/ispypsa/translator/network.py | 46 ++++++++------- 2 files changed, 87 insertions(+), 43 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 3996697d..e0fea38b 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -106,7 +106,7 @@ _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, ) from ispypsa.translator.network import ( - _keep_rows_for_enabled_elements, + _keep_rows_for_expansion_ids, _pair_forward_and_reverse_options, _prepare_expansion_costs, _resolve_expansion_options, @@ -294,10 +294,11 @@ def _translate_custom_constraints_from_network_tables( ) rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + relaxation_generators = _finalise_generators(relaxation_generators) return { "custom_constraints_lhs": lhs, "custom_constraints_rhs": rhs, - "custom_constraints_generators": _finalise_generators(relaxation_generators), + "custom_constraints_generators": relaxation_generators, } @@ -357,9 +358,20 @@ def _add_constraint_type( matching the vocabulary pypsa_build applies constraints with). I/O Example: - rhs: constraint_id=SWQLD1, rhs=3000 - custom_constraints: constraint_id=SWQLD1, direction="<=" - -> constraint_id=SWQLD1, rhs=3000, constraint_type="<=" + rhs: + constraint_id timeslice rhs investment_period + SWQLD1 qld_peak_demand 3000 2026 + NQ1 qld_peak_demand 2650 2026 + + custom_constraints: + constraint_id direction + SWQLD1 <= + NQ1 = + + returns: + constraint_id timeslice rhs investment_period constraint_type + SWQLD1 qld_peak_demand 3000 2026 <= + NQ1 qld_peak_demand 2650 2026 == """ rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) @@ -383,9 +395,17 @@ def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: belongs to. I/O Example: - term_type=generator_output -> component=Generator, attribute=p - term_type=link_flow -> component=Link, attribute=p - term_type=storage_output -> component=Storage, attribute=p + lhs: + constraint_id term_type variable_name coefficient investment_period + SWQLD1 link_flow NSW-QLD 0.84 2026 + SWQLD1 generator_output KINGASF1 0.14 2026 + SWQLD1 storage_output Q8 Battery - 2h 0.43 2026 + + returns: + constraint_id variable_name coefficient investment_period component attribute + SWQLD1 NSW-QLD 0.84 2026 Link p + SWQLD1 KINGASF1 0.14 2026 Generator p + SWQLD1 Q8 Battery - 2h 0.43 2026 Storage p """ lhs = lhs.copy() lhs["component"] = lhs["term_type"].map( @@ -477,23 +497,37 @@ def _create_constraint_relaxation_generators( The generators' p_nom enters the parent constraint's LHS with coefficient -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the constraint at the option's cost; total relaxation is capped at the - option's allowed_expansion by the expansion-limit constraints. The - constraints in the model (constraint_ids) are the enabled elements the - options and costs wildcards resolve against, gated as a whole by the - config's rez_transmission_expansion flag. + option's allowed_expansion by the expansion-limit constraints. - I/O Example: + Options and costs may use blank key cells as wildcards: a blank + expansion_id means "every constraint" — here, every constraint in the + model (constraint_ids) — and a blank cost year means "every investment + period", so a single blank-id row gives all constraints the same + relaxation option or cost, and a blank-year cost row is a static cost + across the periods. If the config's rez_transmission_expansion flag is + off, no relaxation generators are built at all. + + I/O Example (blank cells are wildcards): ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option SWQLD1 constraint_relaxation 500 Option 2 + constraint_relaxation 200 Default ispypsa_tables["network_transmission_path_expansion_costs"]: expansion_id year cost SWQLD1 2030 100000 + 80000 # every constraint, every period + + constraint_ids = ["SWQLD1", "NQ1"] - constraint_ids=["SWQLD1"], investment_periods=[2030] returns (abridged): - name isp_name bus p_nom build_year allowed_expansion - SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 500 + config: rez_transmission_expansion = True, investment_periods = [2030, 2040] + + returns: + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(100000) 500 + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 500 + NQ1_exp_2030 NQ1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(80000) 200 + NQ1_exp_2040 NQ1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 200 """ if not config.network.rez_transmission_expansion: return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) @@ -543,7 +577,7 @@ def _resolve_relaxation_options( options = options[ expansion_type.isna() | (expansion_type == "constraint_relaxation") ] - options = _keep_rows_for_enabled_elements(options, constraint_ids) + options = _keep_rows_for_expansion_ids(options, constraint_ids) allowed_values = { "expansion_id": constraint_ids, "expansion_type": ["constraint_relaxation"], @@ -557,9 +591,15 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: """Adds the PyPSA generator attributes shared by all relaxation generators. I/O Example: - expansion_id=SWQLD1, year=2030, capital_cost=8140, allowed_expansion=500 - -> name=SWQLD1_exp_2030, isp_name=SWQLD1, p_nom=0.0, - p_nom_extendable=True, build_year=2030, lifetime=inf + generators: + expansion_id year capital_cost allowed_expansion + SWQLD1 2030 8140 500 + SWQLD1 2040 7900 500 + + returns: + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf 8140 500 + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf 7900 500 """ generators = generators.rename(columns={"expansion_id": "isp_name"}) generators["name"] = ( @@ -589,7 +629,9 @@ def _relaxation_generator_lhs_terms( SWQLD1_exp_2030 SWQLD1 2030 SWQLD1_exp_2040 SWQLD1 2040 - investment_periods=[2030, 2040] returns: + investment_periods=[2030, 2040] + + returns: constraint_id investment_period variable_name component attribute coefficient SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index 66233841..f274eab8 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -398,7 +398,7 @@ def _enabled_expansion_element_ids( ``transmission_expansion`` gates flow paths between (sub)regions; ``rez_transmission_expansion`` gates REZ connection paths. The result is the enabled expansion_id set the options and costs tables are filtered to (see - _keep_rows_for_enabled_elements) and then resolved against, so an option or + _keep_rows_for_expansion_ids) and then resolved against, so an option or cost for a disabled or non-modelled element drops out before resolution. I/O Example: @@ -462,7 +462,7 @@ def _resolve_expansion_options( # not dropped, so they are set aside before wildcard resolution (which would # otherwise raise on them). options = options[options["expansion_type"] != "constraint_relaxation"] - options = _keep_rows_for_enabled_elements(options, enabled_ids) + options = _keep_rows_for_expansion_ids(options, enabled_ids) allowed_values = { "expansion_id": enabled_ids, "expansion_type": ["forward", "reverse"], @@ -477,22 +477,23 @@ def _resolve_expansion_options( return options -def _keep_rows_for_enabled_elements( - table: pd.DataFrame, enabled_ids: list[str] +def _keep_rows_for_expansion_ids( + table: pd.DataFrame, expansion_ids: list[str] ) -> pd.DataFrame: - """Keeps rows whose expansion_id is an enabled element or blank (a wildcard). + """Keeps rows whose expansion_id is one of the given ids or blank (a wildcard). - Selecting the enabled elements is config-driven, so it happens before - wildcard resolution rather than inside it. Rows for disabled or non-modelled - elements drop out here, as do rows for constraint groups (their expansion_ids - are constraint_ids, routed to ispypsa.translator.constraints instead). + Which ids to keep is a config-driven selection made by the caller — the + enabled paths here, the constraints in the model in + ispypsa.translator.constraints — so it happens before wildcard resolution + rather than inside it. Rows for any other id drop out; the blank rows are + kept because they are wildcards that resolve against the given ids. I/O Example: - table: enabled_ids = ["CQ-NQ"] + table: expansion_ids = ["CQ-NQ"] expansion_id year cost CQ-NQ 2026 1000000 - Q1-NQ 2026 500000 # disabled element: dropped - SWQLD1 2026 400000 # constraint group: dropped + Q1-NQ 2026 500000 # not in expansion_ids: dropped + SWQLD1 2026 400000 # not in expansion_ids: dropped 2026 900000 # blank: a wildcard, kept returns: @@ -501,7 +502,7 @@ def _keep_rows_for_enabled_elements( 2026 900000 """ ids = table["expansion_id"] - keep = ids.isna() | ids.isin(enabled_ids) + keep = ids.isna() | ids.isin(expansion_ids) return table[keep] @@ -591,19 +592,20 @@ def _prepare_expansion_costs( wacc: float, asset_lifetime: int, ) -> pd.DataFrame: - """Resolves the expansion-cost wildcards to the enabled elements and + """Resolves the expansion-cost wildcards to the given expansion ids and investment periods, then annuitises them. Blank expansion_id or year cells are wildcards (see the network_transmission_path_expansion_costs schema): an empty expansion_id is a table-wide default cost, an empty year a static cost across the investment - periods. Costs for disabled elements, constraint groups (routed to - ispypsa.translator.constraints) and years outside the investment periods are - all designed selection, filtered out first. _resolve_wildcards - then expands the blanks against the enabled elements and the investment - periods. A blank cost then resolves to free — the nan_fill the schema - declares for the cost column. Year values are labels matched against the - config's investment periods as ints — no financial vs calendar year + periods. The caller decides which ids the costs are for — the enabled paths + here, the constraints in the model in ispypsa.translator.constraints — so + rows for any other id, and rows for years that are not investment periods, + are dropped before resolution: they are selection, not bad data. + _resolve_wildcards then expands the blanks against the given ids and the + investment periods. A blank cost then resolves to free — the nan_fill the + schema declares for the cost column. Year values are labels matched against + the config's investment periods as ints — no financial vs calendar year interpretation happens here; the config's year_type decides what span of time the labels denote, so the table just has to label years the same way the config does (templated tables carry financial-year ending years). @@ -622,7 +624,7 @@ def _prepare_expansion_costs( CQ-NQ 2026 annuitise(1000) # the static row fills 2026 CQ-NQ 2028 annuitise(1200) # the 2028 override beats the static row """ - costs = _keep_rows_for_enabled_elements(expansion_costs, enabled_ids) + costs = _keep_rows_for_expansion_ids(expansion_costs, enabled_ids) costs = _keep_rows_for_investment_period_years(costs, investment_periods) allowed_values = {"expansion_id": enabled_ids, "year": investment_periods} costs = _resolve_wildcards(costs, allowed_values, ["cost"]) From 97f7f53a23f459a8a56924b96a0e260967497fe1 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:47:43 +1000 Subject: [PATCH 05/20] Choose the relaxation coefficient's sign from the constraint's direction A relaxation generator's p_nom was always subtracted from the parent constraint's LHS, which loosens a "<=" but tightens a ">=". The sign now follows the constraint's direction (-1.0 for "<=", +1.0 for ">=") so building relaxation capacity always loosens the constraint; an option on an "=" is rejected, since no single-signed term can loosen an equality. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 84 ++++++++++++++++++++--- tests/test_translator/test_constraints.py | 75 +++++++++++++++++++- 2 files changed, 146 insertions(+), 13 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index e0fea38b..1111151b 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -70,9 +70,12 @@ Constraint relaxation comes next, gated by the config's rez_transmission_expansion flag. Each constraint that has a constraint_relaxation expansion option gets one extendable dummy generator -per investment period at the option's annualised cost; the generator's p_nom -enters the parent constraint's LHS with coefficient -1.0, so building it -relaxes the constraint. Finally the expansion-limit constraints cap the total +per investment period at the option's annualised cost. The generator's p_nom +enters the parent constraint's LHS with a sign chosen by the constraint's +direction so that building it always loosens the constraint: subtracted from +a "<=" it raises the cap, added to a ">=" it lowers the floor. No +single-signed term can loosen an "=", so a relaxation option on an equality +constraint is rejected. Finally the expansion-limit constraints cap the total p_nom built across each expandable element's per-period components: for a path the cap is max(forward, reverse) of its option, matching the per-unit ratings ispypsa.translator.network gives its expansion links; for a @@ -83,6 +86,8 @@ Reference detail: - direction to constraint_type: "<=" and ">=" pass through, "=" becomes "==". +- relaxation generator coefficient: -1.0 on a "<=" constraint, +1.0 on a + ">="; a relaxation option on an "==" constraint raises. - term_type to component/attribute lives in ispypsa.translator.mappings (_CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE and _..._ATTRIBUTE_TYPE). - Expansion-limit constraints are named "_expansion_limit" so a @@ -144,6 +149,11 @@ _DIRECTION_TO_CONSTRAINT_TYPE = {"<=": "<=", ">=": ">=", "=": "=="} +# The LHS sign that lets a relaxation generator's p_nom loosen a constraint: +# subtracting it from a "<=" raises the cap, adding it to a ">=" lowers the +# floor. No single sign loosens an "==", so it is absent here and rejected. +_CONSTRAINT_TYPE_TO_RELAXATION_COEFFICIENT = {"<=": -1.0, ">=": 1.0} + # Working column orders before constraint_id is renamed to constraint_name. _INTERNAL_LHS_COLUMNS = ["constraint_id"] + [ c for c in _LHS_COLUMNS if c != "constraint_name" @@ -281,6 +291,7 @@ def _translate_custom_constraints_from_network_tables( relaxation_generator_lhs = _relaxation_generator_lhs_terms( relaxation_generators, config.temporal.capacity_expansion.investment_periods, + rhs, ) path_caps = _resolve_path_expansion_caps( ispypsa_tables["network_expansion_options"], links @@ -494,10 +505,11 @@ def _create_constraint_relaxation_generators( """Builds one extendable dummy generator per relaxable constraint and investment period, with the selected expansion option's annualised cost. - The generators' p_nom enters the parent constraint's LHS with coefficient - -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the - constraint at the option's cost; total relaxation is capped at the - option's allowed_expansion by the expansion-limit constraints. + The generators' p_nom enters the parent constraint's LHS with a sign set + by the constraint's direction (see _relaxation_generator_lhs_terms), so + building them relaxes the constraint at the option's cost; total + relaxation is capped at the option's allowed_expansion by the + expansion-limit constraints. Options and costs may use blank key cells as wildcards: a blank expansion_id means "every constraint" — here, every constraint in the @@ -614,28 +626,40 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: def _relaxation_generator_lhs_terms( - relaxation_generators: pd.DataFrame, investment_periods: list[int] + relaxation_generators: pd.DataFrame, + investment_periods: list[int], + rhs: pd.DataFrame, ) -> pd.DataFrame: - """LHS terms subtracting each relaxation generator's capacity from its + """LHS terms letting each relaxation generator's capacity loosen its parent constraint. Terms are per investment period and only include generators already built by that period — capacity built in a later period can't relax an earlier - period's constraint. + period's constraint. Each term's sign follows the parent constraint's + direction (see _relaxation_coefficients) so that building capacity always + loosens the constraint. I/O Example: relaxation_generators: name isp_name build_year SWQLD1_exp_2030 SWQLD1 2030 SWQLD1_exp_2040 SWQLD1 2040 + NQ1_exp_2030 NQ1 2030 investment_periods=[2030, 2040] + rhs (abridged): + constraint_id constraint_type + SWQLD1 <= + NQ1 >= + returns: constraint_id investment_period variable_name component attribute coefficient SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2040 Generator p_nom -1.0 + NQ1 2030 NQ1_exp_2030 Generator p_nom 1.0 # ">=": added, lowering the floor + NQ1 2040 NQ1_exp_2030 Generator p_nom 1.0 """ terms = [] for period in investment_periods: @@ -648,11 +672,49 @@ def _relaxation_generator_lhs_terms( terms = terms.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) terms["component"] = "Generator" terms["attribute"] = "p_nom" - terms["coefficient"] = -1.0 + terms["coefficient"] = _relaxation_coefficients(terms["constraint_id"], rhs) columns = [c for c in _LHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] return terms.loc[:, columns] +def _relaxation_coefficients(constraint_ids: pd.Series, rhs: pd.DataFrame) -> pd.Series: + """The LHS coefficient that lets a relaxation generator's p_nom loosen + each constraint: -1.0 on a "<=" (subtracting from the LHS raises the cap) + and +1.0 on a ">=" (adding to the LHS lowers the floor). Raises for "==" + constraints, which no single-signed term can loosen. + + I/O Example: + constraint_ids: SWQLD1, SWQLD1, NQ1 + + rhs (abridged): + constraint_id constraint_type + SWQLD1 <= + NQ1 >= + + returns: -1.0, -1.0, 1.0 + """ + constraint_type = rhs.drop_duplicates("constraint_id").set_index("constraint_id")[ + "constraint_type" + ] + coefficients = constraint_ids.map(constraint_type).map( + _CONSTRAINT_TYPE_TO_RELAXATION_COEFFICIENT + ) + _raise_on_relaxed_equality_constraints(constraint_ids[coefficients.isna()]) + return coefficients + + +def _raise_on_relaxed_equality_constraints(constraint_ids: pd.Series) -> None: + """Raise when a constraint_relaxation option targets an "==" constraint — + a single-signed slack term could only move the equality one way, which + isn't a relaxation.""" + if not constraint_ids.empty: + raise ValueError( + "Constraint relaxation is only supported for '<=' and '>=' " + "constraints; '==' constraints with a constraint_relaxation " + f"expansion option: {sorted(set(constraint_ids))}" + ) + + def _resolve_path_expansion_caps( options: pd.DataFrame, links: pd.DataFrame ) -> pd.DataFrame: diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index d8389e1c..426cce27 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -199,13 +199,84 @@ def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_co constraint_id, direction SWQLD1, = """) + # No relaxation option: an "==" constraint can't be relaxed. + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + """) result = _translate_custom_constraints_from_network_tables( ispypsa_tables, _links(csv_str_to_df), sample_model_config ) - rhs = result["custom_constraints_rhs"] - assert set(rhs.loc[rhs["constraint_name"] == "SWQLD1", "constraint_type"]) == {"=="} + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, == + SWQLD1, 2026, qld_winter_reference, 3500, == + SWQLD1, 2028, qld_peak_demand, 3000, == + SWQLD1, 2028, qld_winter_reference, 3500, == + NSW-QLD_expansion_limit, , , 1000, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( + csv_str_to_df, sample_model_config +): + """A ">=" constraint is loosened by lowering its floor, so the relaxation + generator's p_nom is added to the LHS (+1.0) rather than subtracted.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, >= + """) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, 1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_raises_on_relaxation_option_for_equality_constraint( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, = + """) + + with pytest.raises( + ValueError, match=r"'==' constraints with a constraint_relaxation.*SWQLD1" + ): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) def test_link_terms_not_in_model_dropped_and_logged( From 3c0abf800e7c022fab9f03e2bc195b2dcaa8f7ab Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:48:09 +1000 Subject: [PATCH 06/20] Move input-integrity checks to the schemas; drop one-sided constraint periods The translator raised on duplicate rows, missing directions, unpaired constraints and relaxed equalities -- all properties of the input tables, so they're now declared as schema rules the translator trusts without re-checking. What's genuinely per-run remains: after date_from resolution a (constraint, period) can end up with terms on one side only, so those periods are dropped from both sides with an INFO line each way. Relaxation-generator terms derive from the post-drop RHS so they can't re-create one-sided periods. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 236 ++++++++---------- .../schemas/custom_constraints_lhs.yaml | 9 + .../schemas/network_expansion_options.yaml | 8 + tests/test_translator/test_constraints.py | 134 +++++++--- 4 files changed, 227 insertions(+), 160 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 1111151b..8d3024d9 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -54,49 +54,57 @@ timeslice means the RHS binds at every snapshot (otherwise only at snapshots inside the timeslice's windows, see ispypsa.translator.timeslices). -The pipeline runs as follows. Duplicate input rows are rejected first, since -they would silently collapse during date resolution. Then the date_from -column of the LHS and RHS tables is resolved into one row per investment -period: for each period, each group (a constraint's term, or a constraint's -timeslice) keeps the row active at the period's start — the latest date_from -on or before it, with no-date_from rows as the baseline. The RHS gains each -constraint's sense from custom_constraints as constraint_type. LHS term_types -map to the PyPSA component and attribute their variable belongs to, and each -link_flow term is expanded from its path_id to every link the model has on -that path — the existing link and each expansion link — so flow through new -builds counts towards the constraint too. Terms for paths not in the model -are dropped, and constraints left with no LHS terms are dropped with them. +The pipeline runs as follows. The date_from column of the LHS and RHS tables +is resolved into one row per investment period: for each period, each group +(a constraint's term, or a constraint's timeslice) keeps the row active at +the period's start — the latest date_from on or before it, with no-date_from +rows as the baseline. The RHS gains each constraint's sense from +custom_constraints as constraint_type. LHS term_types map to the PyPSA +component and attribute their variable belongs to, and each link_flow term +is expanded from its path_id to every link the model has on that path — the +existing link and each expansion link — so flow through new builds counts +towards the constraint too; terms for paths not in the model are dropped. +Because date resolution can leave a constraint with terms but no limit (or a +limit but no terms) in some periods, the two tables are then reconciled +period by period: a constraint is kept only in the periods where it has both +LHS terms and an RHS row, and the one-sided periods are dropped and logged. Constraint relaxation comes next, gated by the config's rez_transmission_expansion flag. Each constraint that has a constraint_relaxation expansion option gets one extendable dummy generator per investment period at the option's annualised cost. The generator's p_nom -enters the parent constraint's LHS with a sign chosen by the constraint's -direction so that building it always loosens the constraint: subtracted from -a "<=" it raises the cap, added to a ">=" it lowers the floor. No -single-signed term can loosen an "=", so a relaxation option on an equality -constraint is rejected. Finally the expansion-limit constraints cap the total +enters the parent constraint's LHS, in each period the constraint binds in, +with a sign chosen by the constraint's direction so that building it always +loosens the constraint: subtracted from a "<=" it raises the cap, added to a +">=" it lowers the floor. Finally the expansion-limit constraints cap the total p_nom built across each expandable element's per-period components: for a path the cap is max(forward, reverse) of its option, matching the per-unit ratings ispypsa.translator.network gives its expansion links; for a relaxation it is the option's allowed_expansion. Both tables are then -finalised — constraint_id becomes constraint_name and one-sided or duplicate -constraints are rejected. +finalised — constraint_id becomes constraint_name and duplicate constraint +names are rejected. + +Input integrity is the table schemas' job, not this module's. The rules the +pipeline relies on without re-checking — unique input rows, a direction for +every constraint with RHS values, the LHS and RHS naming the same +constraints, and no constraint_relaxation option on an "=" constraint — are +declared in src/ispypsa/validation/schemas (custom_constraints*.yaml and +network_expansion_options.yaml). Reference detail: - direction to constraint_type: "<=" and ">=" pass through, "=" becomes "==". -- relaxation generator coefficient: -1.0 on a "<=" constraint, +1.0 on a - ">="; a relaxation option on an "==" constraint raises. +- relaxation generator coefficient: -1.0 on a "<=" constraint, +1.0 on a ">=". - term_type to component/attribute lives in ispypsa.translator.mappings (_CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE and _..._ATTRIBUTE_TYPE). - Expansion-limit constraints are named "_expansion_limit" so a relaxation cap doesn't collide with the constraint it relaxes. -- Dropped rows: link_flow terms whose path is not in the model (logged); RHS - rows for constraints with no LHS terms (logged); relaxation options and - costs for constraints not in the model, or all of them when - rez_transmission_expansion is off; date_from rows that only start after - every investment period. +- Dropped rows: link_flow terms whose path is not in the model (logged); per + investment period, RHS rows of a constraint with no LHS terms in that + period and LHS terms of a constraint with no RHS row in that period (both + logged); relaxation options and costs for constraints not in the model, or + all of them when rez_transmission_expansion is off; date_from rows that + only start after every investment period. """ import logging @@ -151,7 +159,8 @@ # The LHS sign that lets a relaxation generator's p_nom loosen a constraint: # subtracting it from a "<=" raises the cap, adding it to a ">=" lowers the -# floor. No single sign loosens an "==", so it is absent here and rejected. +# floor. No single sign loosens an "==", so the network_expansion_options +# schema forbids relaxing one and it is absent here. _CONSTRAINT_TYPE_TO_RELAXATION_COEFFICIENT = {"<=": -1.0, ">=": 1.0} # Working column orders before constraint_id is renamed to constraint_name. @@ -163,19 +172,6 @@ ] -def _raise_on_duplicate_input_rows( - table: pd.DataFrame, keys: list[str], label: str -) -> None: - """Raise on input rows sharing the same key — duplicates would otherwise - silently collapse to one arbitrary row during date_from resolution.""" - duplicates = table[table.duplicated(subset=keys, keep=False)] - if not duplicates.empty: - raise ValueError( - f"Duplicate custom constraint {label} rows for: " - f"{sorted(set(duplicates['constraint_id']))}" - ) - - def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.DataFrame: """Concatenates the non-empty frames (concatenating all-empty frames is deprecated by pandas), returning a header-only frame when all are empty. @@ -256,16 +252,6 @@ def _translate_custom_constraints_from_network_tables( name isp_name bus p_nom build_year capital_cost SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 2026 annuitise(100000) """ - _raise_on_duplicate_input_rows( - ispypsa_tables["custom_constraints_rhs"], - ["constraint_id", "timeslice", "date_from"], - "RHS", - ) - _raise_on_duplicate_input_rows( - ispypsa_tables["custom_constraints_lhs"], - ["constraint_id", "term_type", "variable_name", "date_from"], - "LHS", - ) period_starts = _investment_period_start_dates( config.temporal.capacity_expansion.investment_periods, config.temporal.year_type, @@ -283,15 +269,13 @@ def _translate_custom_constraints_from_network_tables( ) lhs = _add_component_and_attribute(lhs) lhs = _expand_link_flow_terms(lhs, links) - rhs = _drop_rhs_without_lhs_terms(rhs, lhs) + lhs, rhs = _drop_one_sided_constraint_periods(lhs, rhs) relaxation_generators = _create_constraint_relaxation_generators( ispypsa_tables, sorted(set(rhs["constraint_id"])), config ) relaxation_generator_lhs = _relaxation_generator_lhs_terms( - relaxation_generators, - config.temporal.capacity_expansion.investment_periods, - rhs, + relaxation_generators, rhs ) path_caps = _resolve_path_expansion_caps( ispypsa_tables["network_expansion_options"], links @@ -386,21 +370,9 @@ def _add_constraint_type( """ rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) - _raise_on_missing_constraint_type(rhs) return rhs.drop(columns="direction") -def _raise_on_missing_constraint_type(rhs: pd.DataFrame) -> None: - """Raise if any RHS row's constraint has no (or an unmapped) sense — the - constraint can't be applied without one.""" - missing = rhs.loc[rhs["constraint_type"].isna(), "constraint_id"] - if not missing.empty: - raise ValueError( - f"Custom constraints with RHS values but no direction in the " - f"custom_constraints table: {sorted(set(missing))}" - ) - - def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: """Maps each term_type to the PyPSA component and attribute its variable belongs to. @@ -481,20 +453,53 @@ def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) ) -def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.DataFrame: - """Drops (and logs) RHS rows for constraints left with no LHS terms — a - constraint with an empty LHS can't be applied. +def _drop_one_sided_constraint_periods( + lhs: pd.DataFrame, rhs: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Keeps each constraint only in the investment periods where it has both + LHS terms and an RHS row, dropping (and logging) the one-sided periods. + + A period is one-sided when one side's date_from starts later than the + other's, or when every LHS term was dropped because its path is not in + the model. Either way the constraint can't be applied in that period. I/O Example: - rhs constraint_ids {SWQLD1, NQ1}, lhs constraint_ids {SWQLD1} - -> NQ1's RHS rows dropped and logged; SWQLD1's kept + lhs (abridged): rhs (abridged): + constraint_id investment_period constraint_id investment_period + SWQLD1 2028 SWQLD1 2026 + NQ1 2026 SWQLD1 2028 + NQ1 2028 NQ1 2026 + + returns: + lhs without its NQ1 2028 term (no RHS row that period; logged) + rhs without its SWQLD1 2026 row (no LHS terms that period; logged) """ - without_lhs = set(rhs["constraint_id"]) - set(lhs["constraint_id"]) - if without_lhs: + keys = ["constraint_id", "investment_period"] + lhs_periods = lhs.loc[:, keys].drop_duplicates() + rhs_periods = rhs.loc[:, keys].drop_duplicates() + _log_one_sided_periods( + rhs_periods, lhs_periods, "RHS rows dropped (no LHS terms in that period)" + ) + _log_one_sided_periods( + lhs_periods, rhs_periods, "LHS terms dropped (no RHS row in that period)" + ) + two_sided = lhs_periods.merge(rhs_periods, on=keys) + return lhs.merge(two_sided, on=keys), rhs.merge(two_sided, on=keys) + + +def _log_one_sided_periods( + present: pd.DataFrame, required: pd.DataFrame, message: str +) -> None: + """Logs the (constraint_id, investment_period) pairs in present that have + no partner in required — the pairs _drop_one_sided_constraint_periods is + about to drop.""" + unmatched = present.merge(required, how="left", indicator=True) + unmatched = unmatched[unmatched["_merge"] == "left_only"] + if not unmatched.empty: + pairs = zip(unmatched["constraint_id"], unmatched["investment_period"]) logger.info( - f"Custom constraints dropped (no LHS terms in model): {sorted(without_lhs)}" + f"Custom constraint {message}: {sorted((c, int(p)) for c, p in pairs)}" ) - return rhs[~rhs["constraint_id"].isin(without_lhs)] def _create_constraint_relaxation_generators( @@ -626,18 +631,17 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: def _relaxation_generator_lhs_terms( - relaxation_generators: pd.DataFrame, - investment_periods: list[int], - rhs: pd.DataFrame, + relaxation_generators: pd.DataFrame, rhs: pd.DataFrame ) -> pd.DataFrame: """LHS terms letting each relaxation generator's capacity loosen its parent constraint. - Terms are per investment period and only include generators already built - by that period — capacity built in a later period can't relax an earlier - period's constraint. Each term's sign follows the parent constraint's - direction (see _relaxation_coefficients) so that building capacity always - loosens the constraint. + A term exists for each period the parent constraint has an RHS row in, + and only for generators already built by that period — capacity built in + a later period can't relax an earlier period's constraint. Each term's + sign follows the parent constraint's direction (see + _relaxation_coefficients) so that building capacity always loosens the + constraint. I/O Example: relaxation_generators: @@ -646,30 +650,25 @@ def _relaxation_generator_lhs_terms( SWQLD1_exp_2040 SWQLD1 2040 NQ1_exp_2030 NQ1 2030 - investment_periods=[2030, 2040] - rhs (abridged): - constraint_id constraint_type - SWQLD1 <= - NQ1 >= + constraint_id investment_period constraint_type + SWQLD1 2030 <= + SWQLD1 2040 <= + NQ1 2040 >= # NQ1 has no 2030 row returns: constraint_id investment_period variable_name component attribute coefficient SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 SWQLD1 2040 SWQLD1_exp_2040 Generator p_nom -1.0 - NQ1 2030 NQ1_exp_2030 Generator p_nom 1.0 # ">=": added, lowering the floor - NQ1 2040 NQ1_exp_2030 Generator p_nom 1.0 + NQ1 2040 NQ1_exp_2030 Generator p_nom 1.0 # ">=": added, lowering the floor """ - terms = [] - for period in investment_periods: - built = relaxation_generators[ - relaxation_generators["build_year"] <= period - ].copy() - built["investment_period"] = period - terms.append(built) - terms = pd.concat(terms, ignore_index=True) - terms = terms.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + constraint_periods = rhs.loc[:, ["constraint_id", "investment_period"]] + terms = constraint_periods.drop_duplicates().merge( + relaxation_generators, left_on="constraint_id", right_on="isp_name" + ) + terms = terms[terms["build_year"] <= terms["investment_period"]].copy() + terms = terms.rename(columns={"name": "variable_name"}) terms["component"] = "Generator" terms["attribute"] = "p_nom" terms["coefficient"] = _relaxation_coefficients(terms["constraint_id"], rhs) @@ -680,8 +679,9 @@ def _relaxation_generator_lhs_terms( def _relaxation_coefficients(constraint_ids: pd.Series, rhs: pd.DataFrame) -> pd.Series: """The LHS coefficient that lets a relaxation generator's p_nom loosen each constraint: -1.0 on a "<=" (subtracting from the LHS raises the cap) - and +1.0 on a ">=" (adding to the LHS lowers the floor). Raises for "==" - constraints, which no single-signed term can loosen. + and +1.0 on a ">=" (adding to the LHS lowers the floor). No single-signed + term can loosen an "==", so the network_expansion_options schema forbids + relaxing one. I/O Example: constraint_ids: SWQLD1, SWQLD1, NQ1 @@ -696,23 +696,9 @@ def _relaxation_coefficients(constraint_ids: pd.Series, rhs: pd.DataFrame) -> pd constraint_type = rhs.drop_duplicates("constraint_id").set_index("constraint_id")[ "constraint_type" ] - coefficients = constraint_ids.map(constraint_type).map( + return constraint_ids.map(constraint_type).map( _CONSTRAINT_TYPE_TO_RELAXATION_COEFFICIENT ) - _raise_on_relaxed_equality_constraints(constraint_ids[coefficients.isna()]) - return coefficients - - -def _raise_on_relaxed_equality_constraints(constraint_ids: pd.Series) -> None: - """Raise when a constraint_relaxation option targets an "==" constraint — - a single-signed slack term could only move the equality one way, which - isn't a relaxation.""" - if not constraint_ids.empty: - raise ValueError( - "Constraint relaxation is only supported for '<=' and '>=' " - "constraints; '==' constraints with a constraint_relaxation " - f"expansion option: {sorted(set(constraint_ids))}" - ) def _resolve_path_expansion_caps( @@ -856,8 +842,8 @@ def _expansion_limit_rhs(caps: pd.DataFrame) -> pd.DataFrame: def _finalise_lhs_and_rhs( lhs: pd.DataFrame, rhs: pd.DataFrame ) -> tuple[pd.DataFrame, pd.DataFrame]: - """Renames constraint_id to constraint_name, validates the LHS/RHS - pairing, and sets the final PyPSA friendly column orders. + """Renames constraint_id to constraint_name, rejects duplicate constraint + names, and sets the final PyPSA friendly column orders. I/O Example: lhs: constraint_id=SWQLD1, ... rhs: constraint_id=SWQLD1, ... @@ -867,7 +853,6 @@ def _finalise_lhs_and_rhs( lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) _raise_on_duplicate_rhs_rows(rhs) - _raise_on_unpaired_constraints(lhs, rhs) return ( lhs.loc[:, _LHS_COLUMNS].reset_index(drop=True), rhs.loc[:, _RHS_COLUMNS].reset_index(drop=True), @@ -886,19 +871,6 @@ def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: ) -def _raise_on_unpaired_constraints(lhs: pd.DataFrame, rhs: pd.DataFrame) -> None: - """Raise if any constraint appears on only one side — a one-sided - constraint can't be applied.""" - lhs_names = set(lhs["constraint_name"]) - rhs_names = set(rhs["constraint_name"]) - if lhs_names != rhs_names: - raise ValueError( - f"Custom constraints with LHS terms but no RHS: " - f"{sorted(lhs_names - rhs_names)}; with RHS but no LHS terms: " - f"{sorted(rhs_names - lhs_names)}" - ) - - def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: """Drops the allowed_expansion working column carried for the expansion-limit RHS, leaving the PyPSA generator columns. diff --git a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml index 29a1f15e..fb375179 100644 --- a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml +++ b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml @@ -19,6 +19,15 @@ description: > If absent: Constraints have no terms, so no custom constraints bind. +custom_validation: + - name: lhs_and_rhs_name_the_same_constraints + description: > + Every constraint_id with a row here must have at least one row in + custom_constraints_rhs, and every constraint_id in custom_constraints_rhs + must have at least one row here. A constraint with terms but no limit + value, or a limit value but no terms, cannot be applied. This does not + require every custom_constraints row to have terms: a catalogued + constraint with neither LHS nor RHS rows simply never binds. columns: constraint_id: type: string diff --git a/src/ispypsa/validation/schemas/network_expansion_options.yaml b/src/ispypsa/validation/schemas/network_expansion_options.yaml index 66cfce6d..b503b7cf 100644 --- a/src/ispypsa/validation/schemas/network_expansion_options.yaml +++ b/src/ispypsa/validation/schemas/network_expansion_options.yaml @@ -48,6 +48,14 @@ custom_validation: direction and stands alone. This keeps an element's resolved forward and reverse tied to the same option, so the single cost keyed on expansion_id applies to a coherent pair. + - name: relaxation_options_target_inequality_constraints + description: > + A constraint_relaxation row must not resolve to a constraint whose + direction in custom_constraints is "=", whether keyed on the constraint + directly or reached through a blank expansion_id or expansion_type + wildcard. Relaxation adds a single-signed slack term to the constraint's + LHS, which can loosen a "<=" or ">=" but can only move an equality one + way. columns: expansion_id: type: string diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 426cce27..4d0c7e0c 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -199,7 +199,8 @@ def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_co constraint_id, direction SWQLD1, = """) - # No relaxation option: an "==" constraint can't be relaxed. + # No relaxation option: the network_expansion_options schema forbids + # relaxing an "=" constraint. ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" expansion_id, expansion_type, allowed_expansion, expansion_option NSW-QLD, forward, 1000, NSW-QLD Option 1 @@ -262,23 +263,6 @@ def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( ) -def test_raises_on_relaxation_option_for_equality_constraint( - csv_str_to_df, sample_model_config -): - ispypsa_tables = _constraint_tables(csv_str_to_df) - ispypsa_tables["custom_constraints"] = csv_str_to_df(""" - constraint_id, direction - SWQLD1, = - """) - - with pytest.raises( - ValueError, match=r"'==' constraints with a constraint_relaxation.*SWQLD1" - ): - _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config - ) - - def test_link_terms_not_in_model_dropped_and_logged( csv_str_to_df, sample_model_config, caplog ): @@ -321,36 +305,130 @@ def test_constraint_with_no_lhs_terms_dropped_and_logged( ) assert ( - "Custom constraints dropped (no LHS terms in model): ['NQ1']" + "Custom constraint RHS rows dropped (no LHS terms in that period): " + "[('NQ1', 2026), ('NQ1', 2028)]" ) in caplog.text - assert "NQ1" not in set(result["custom_constraints_rhs"]["constraint_name"]) + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + +def _one_sided_period_expected_outputs(csv_str_to_df): + """SWQLD1 binding in 2028 only, with a single generator term: the outputs + both mid-horizon date_from cases below converge on.""" + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + """) + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2028, qld_peak_demand, 3000, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + return expected_lhs, expected_rhs + + +def _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs): + lhs_sort = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(lhs_sort).reset_index(drop=True), + expected_lhs.sort_values(lhs_sort).reset_index(drop=True), + check_dtype=False, + ) + rhs_sort = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(rhs_sort).reset_index(drop=True), + expected_rhs.sort_values(rhs_sort).reset_index(drop=True), + check_dtype=False, + ) -def test_raises_on_rhs_without_direction(csv_str_to_df, sample_model_config): + +def test_rhs_starting_mid_horizon_drops_lhs_for_earlier_periods_and_logs( + csv_str_to_df, sample_model_config, caplog +): + """An RHS whose date_from falls after the 2026 period start (2025-07-01) + but before 2028's (2027-07-01) binds only in 2028, so SWQLD1's 2026 LHS + terms have nothing to pair with and are dropped — including the + relaxation generator's, which only enter periods the constraint binds in.""" ispypsa_tables = _constraint_tables(csv_str_to_df) - ispypsa_tables["custom_constraints"] = csv_str_to_df(""" - constraint_id, direction + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, 2027-01-01T00:00:00 """) - with pytest.raises(ValueError, match=r"no direction.*SWQLD1"): - _translate_custom_constraints_from_network_tables( + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( ispypsa_tables, _links(csv_str_to_df), sample_model_config ) + assert ( + "Custom constraint LHS terms dropped (no RHS row in that period): " + "[('SWQLD1', 2026)]" + ) in caplog.text + expected_lhs, expected_rhs = _one_sided_period_expected_outputs(csv_str_to_df) + _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) + -def test_raises_on_duplicate_rhs_rows(csv_str_to_df, sample_model_config): +def test_lhs_starting_mid_horizon_drops_rhs_for_earlier_periods_and_logs( + csv_str_to_df, sample_model_config, caplog +): + """The mirror case: LHS terms that only start after the 2026 period start + leave SWQLD1's 2026 RHS row with nothing to constrain, so it is dropped + rather than emitted as an empty (or relaxation-only) constraint.""" ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, 2027-01-01T00:00:00 + """) ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" constraint_id, timeslice, rhs, date_from SWQLD1, qld_peak_demand, 3000, - SWQLD1, qld_peak_demand, 2500, """) - with pytest.raises(ValueError, match=r"Duplicate custom constraint RHS.*SWQLD1"): + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraint RHS rows dropped (no LHS terms in that period): " + "[('SWQLD1', 2026)]" + ) in caplog.text + expected_lhs, expected_rhs = _one_sided_period_expected_outputs(csv_str_to_df) + _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) + + +def test_no_one_sided_period_log_when_every_period_has_both_sides( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + with caplog.at_level("INFO"): _translate_custom_constraints_from_network_tables( ispypsa_tables, _links(csv_str_to_df), sample_model_config ) + assert "dropped (no LHS terms in that period)" not in caplog.text + assert "dropped (no RHS row in that period)" not in caplog.text + def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): """At coarser granularities the custom-constraint tables are header-only; From 4cecee8852e2c3b87c9ce4f993cb53a3c1519048 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:48:33 +1000 Subject: [PATCH 07/20] Rely on the costs schema's coverage rule for per-period relaxation generators Every option has a cost in every investment period (the costs schema's coverage rule), so joining options to costs gives exactly one generator per option and period. The test fixture violated that rule -- SWQLD1 had a 2026 cost only -- which hid the second per-period generator from every relaxation expectation; it now carries schema-valid costs and the tests exercise both SWQLD1_exp_2026 and SWQLD1_exp_2028. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 15 ++++++++----- tests/test_translator/test_constraints.py | 26 +++++++++++++++++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 8d3024d9..166575cf 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -87,9 +87,11 @@ Input integrity is the table schemas' job, not this module's. The rules the pipeline relies on without re-checking — unique input rows, a direction for every constraint with RHS values, the LHS and RHS naming the same -constraints, and no constraint_relaxation option on an "=" constraint — are -declared in src/ispypsa/validation/schemas (custom_constraints*.yaml and -network_expansion_options.yaml). +constraints, no constraint_relaxation option on an "=" constraint, and a +cost for every expandable element in every investment period — are declared +in src/ispypsa/validation/schemas (custom_constraints*.yaml, +network_expansion_options.yaml and +network_transmission_path_expansion_costs.yaml). Reference detail: @@ -521,8 +523,11 @@ def _create_constraint_relaxation_generators( model (constraint_ids) — and a blank cost year means "every investment period", so a single blank-id row gives all constraints the same relaxation option or cost, and a blank-year cost row is a static cost - across the periods. If the config's rez_transmission_expansion flag is - off, no relaxation generators are built at all. + across the periods. Every constraint with an option has a cost in every + investment period (the costs schema's coverage rule), so joining the two + gives exactly one generator per option and period. If the config's + rez_transmission_expansion flag is off, no relaxation generators are + built at all. I/O Example (blank cells are wildcards): ispypsa_tables["network_expansion_options"]: diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 4d0c7e0c..7b0dcdb8 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -40,7 +40,9 @@ def _constraint_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" expansion_id, year, cost NSW-QLD, 2026, 500000 + NSW-QLD, 2028, 500000 SWQLD1, 2026, 100000 + SWQLD1, 2028, 80000 """) return tables @@ -96,8 +98,10 @@ def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 """) sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] pd.testing.assert_frame_equal( @@ -119,9 +123,11 @@ def test_translate_custom_constraints_relaxation_generators( expected_generators = csv_str_to_df(f""" name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + SWQLD1_exp_2028, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2028, inf, {80000 * _ANNUITY_PER_DOLLAR} """) + generators = result["custom_constraints_generators"] pd.testing.assert_frame_equal( - result["custom_constraints_generators"], + generators.sort_values("name").reset_index(drop=True), expected_generators, check_dtype=False, rtol=1e-5, @@ -252,8 +258,10 @@ def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( SWQLD1, 2028, KINGASF1, Generator, p, 0.14 SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, 1.0 SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, 1.0 NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 """) sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] pd.testing.assert_frame_equal( @@ -325,13 +333,16 @@ def test_constraint_with_no_lhs_terms_dropped_and_logged( def _one_sided_period_expected_outputs(csv_str_to_df): """SWQLD1 binding in 2028 only, with a single generator term: the outputs - both mid-horizon date_from cases below converge on.""" + both mid-horizon date_from cases below converge on. Both relaxation + generators are still built, but only enter the 2028 constraint.""" expected_lhs = csv_str_to_df(""" constraint_name, investment_period, variable_name, component, attribute, coefficient SWQLD1, 2028, KINGASF1, Generator, p, 0.14 SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 """) expected_rhs = csv_str_to_df(""" constraint_name, investment_period, timeslice, rhs, constraint_type @@ -584,11 +595,12 @@ def test_relaxation_option_for_constraint_not_in_model_is_dropped( SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 NQ1, constraint_relaxation, 300, NQ1 Option 1 """) + # Blank years: a static cost across the investment periods. ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" expansion_id, year, cost - NSW-QLD, 2026, 500000 - SWQLD1, 2026, 100000 - NQ1, 2026, 100000 + NSW-QLD, , 500000 + SWQLD1, , 100000 + NQ1, , 100000 """) result = _translate_custom_constraints_from_network_tables( @@ -598,9 +610,11 @@ def test_relaxation_option_for_constraint_not_in_model_is_dropped( expected_generators = csv_str_to_df(f""" name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + SWQLD1_exp_2028, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2028, inf, {100000 * _ANNUITY_PER_DOLLAR} """) + generators = result["custom_constraints_generators"] pd.testing.assert_frame_equal( - result["custom_constraints_generators"], + generators.sort_values("name").reset_index(drop=True), expected_generators, check_dtype=False, rtol=1e-5, From 84e2249e92ed2e88b4574f1110d3dd3e467d697c Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:48:55 +1000 Subject: [PATCH 08/20] Declare that LHS variable_names resolve against their component tables Terms naming components not in the model were described as skipped with a log line downstream -- a silent per-term drop. The custom_constraints_lhs schema now ties each term_type's variable_name to the table it must exist in, so the translator and pypsa_build take the names as given. The templater TODO points at the rule as the end state for its IASR ID lookup. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- .../custom_constraints_from_plexos.py | 9 +++++--- src/ispypsa/translator/constraints.py | 13 ++++++----- .../schemas/custom_constraints_lhs.yaml | 22 +++++++++++++------ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/ispypsa/templater/custom_constraints_from_plexos.py b/src/ispypsa/templater/custom_constraints_from_plexos.py index 68c35fbe..1d617615 100644 --- a/src/ispypsa/templater/custom_constraints_from_plexos.py +++ b/src/ispypsa/templater/custom_constraints_from_plexos.py @@ -156,9 +156,12 @@ date_from is retained so time-varying coefficients carry through to the output. -TODO: switch the IASR ID lookup to a templated generator-summary table once -one exists -- matching against the raw table currently also matches DER/CER -rows that ISPyPSA may not template. +TODO: switch the IASR ID lookup to the templated generator and storage +tables once the existing-unit ones exist -- matching against the raw summary +tables currently also matches DER/CER rows that ISPyPSA may not template. +The custom_constraints_lhs schema's variable_names_resolve_by_term_type rule +declares that end state; note the new-entrant templater renames units to +" ", which the lookup here will need to follow. TODO: emitted timeslices are region-prefixed (qld_peak_demand etc.) while the rest of the templater still emits the bare canonical names diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 166575cf..95d639d7 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -87,9 +87,10 @@ Input integrity is the table schemas' job, not this module's. The rules the pipeline relies on without re-checking — unique input rows, a direction for every constraint with RHS values, the LHS and RHS naming the same -constraints, no constraint_relaxation option on an "=" constraint, and a -cost for every expandable element in every investment period — are declared -in src/ispypsa/validation/schemas (custom_constraints*.yaml, +constraints, every LHS variable_name naming a component in the table its +term_type refers to, no constraint_relaxation option on an "=" constraint, +and a cost for every expandable element in every investment period — are +declared in src/ispypsa/validation/schemas (custom_constraints*.yaml, network_expansion_options.yaml and network_transmission_path_expansion_costs.yaml). @@ -200,9 +201,9 @@ def _translate_custom_constraints_from_network_tables( custom_constraints_rhs, network_expansion_options and network_transmission_path_expansion_costs tables, plus the PyPSA friendly links table from ispypsa.translator.network (existing plus expansion - links). Generator and battery terms pass through with their IASR IDs as - variable_names: pypsa_build skips (and logs) terms whose components are - not in the model. + links). Generator, storage and load terms pass through with their IASR + IDs as variable_names, unchecked: the custom_constraints_lhs schema ties + every variable_name to its component table. I/O Example (config: investment periods 2026 and 2028): ispypsa_tables["custom_constraints"]: diff --git a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml index fb375179..ea800976 100644 --- a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml +++ b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml @@ -11,11 +11,10 @@ description: > templater. Source notes: - variable_name uses IASR IDs for generators and batteries and path_ids for - links. Generator and battery terms may reference units that are not in the - model (e.g. before generator templating lands, or units outside a filtered - region); such terms are skipped with a log line when constraints are - applied. + variable_name uses IASR IDs for generators and batteries, path_ids for links + and sub-region codes for loads. The variable_names_resolve_by_term_type rule + ties each to its component table, so the translator and pypsa_build take the + names as given rather than checking them against the model. If absent: Constraints have no terms, so no custom constraints bind. @@ -28,6 +27,14 @@ custom_validation: value, or a limit value but no terms, cannot be applied. This does not require every custom_constraints row to have terms: a catalogued constraint with neither LHS nor RHS rows simply never binds. + - name: variable_names_resolve_by_term_type + description: > + Each row's variable_name must exist in the table its term_type refers to: + link_flow -> network_transmission_paths.path_id; generator_output and + generator_capacity -> generators_existing_planned.name or + generators_new_entrant.name; storage_output -> storage_existing_planned.name + or storage_new_entrant.name; load -> network_geography.geo_id. A term + naming a component that is not in those tables cannot be applied. columns: constraint_id: type: string @@ -49,8 +56,9 @@ columns: required: true description: > Name of the component the term's variable belongs to: an IASR ID for - generator_output and storage_output terms, a path_id for link_flow - terms. + generator_output, generator_capacity and storage_output terms, a path_id + for link_flow terms, a sub-region geo_id for load terms (see + custom_validation for the table each resolves against). coefficient: type: float required: true From afa4e71dfd5f3ad2d7d2127827ebd3384fa2f5d0 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:49:05 +1000 Subject: [PATCH 09/20] State the blank-timeslice fallback semantics in the module docstring A blank timeslice isn't "binds at every snapshot": it's the constraint's fallback row, binding only at the snapshots none of its named-timeslice rows cover, matching the path-limits convention. Resolving timeslices to snapshots stays pypsa_build's job; this module passes them through. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 95d639d7..7e556f00 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -50,9 +50,13 @@ name isp_name bus p_nom p_nom_extendable build_year capital_cost SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2026 annuitise(100000) -A blank investment_period means the row applies in every period; a blank -timeslice means the RHS binds at every snapshot (otherwise only at snapshots -inside the timeslice's windows, see ispypsa.translator.timeslices). +A blank investment_period means the row applies in every period. A named +timeslice scopes the RHS to the snapshots inside that timeslice's windows +(the timeslices table); a blank timeslice is the constraint's fallback, +applying at the snapshots none of its named-timeslice rows cover, so a +constraint with only named rows does not bind outside them. This module +passes timeslice through untouched — resolving it to snapshots is +pypsa_build's job when the constraints are applied. The pipeline runs as follows. The date_from column of the LHS and RHS tables is resolved into one row per investment period: for each period, each group From 3c1bee6b947a6df1c0bae767ad6195d046894dc8 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:49:30 +1000 Subject: [PATCH 10/20] Assert full LHS frames instead of probing for absent names Set-membership probes like "SWQLD1_exp_2026 not in variable_name" pass for many wrong outputs. The relaxation-disabled, late-date_from and missing-link tests now compare whole expected frames, and the negative log case covers every INFO drop line the module can emit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- tests/test_translator/test_constraints.py | 70 ++++++++++++++++++++--- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 7b0dcdb8..9a03c1a6 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -150,8 +150,29 @@ def test_translate_custom_constraints_rez_expansion_disabled( pd.testing.assert_frame_equal( result["custom_constraints_generators"], expected_generators, check_dtype=False ) - lhs = result["custom_constraints_lhs"] - assert "SWQLD1_exp_2026" not in set(lhs["variable_name"]) + # No relaxation terms and no SWQLD1_expansion_limit; the path's expansion + # limit is unaffected by the flag. + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + """) + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, qld_winter_reference, 3500, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_winter_reference, 3500, <= + NSW-QLD_expansion_limit, , , 1000, <= + """) + _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config): @@ -196,7 +217,23 @@ def test_date_from_after_all_periods_contributes_nothing( ispypsa_tables, _links(csv_str_to_df), sample_model_config ) - assert "LATEGEN" not in set(result["custom_constraints_lhs"]["variable_name"]) + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_config): @@ -289,7 +326,23 @@ def test_link_terms_not_in_model_dropped_and_logged( assert ( "Custom constraint link_flow terms dropped (paths not in model): ['TAS-SEV']" ) in caplog.text - assert "TAS-SEV" not in set(result["custom_constraints_lhs"]["variable_name"]) + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) def test_constraint_with_no_lhs_terms_dropped_and_logged( @@ -427,9 +480,11 @@ def test_lhs_starting_mid_horizon_drops_rhs_for_earlier_periods_and_logs( _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) -def test_no_one_sided_period_log_when_every_period_has_both_sides( +def test_no_drop_logs_when_every_term_and_period_is_in_the_model( csv_str_to_df, sample_model_config, caplog ): + """The base fixture's link is in the model and both sides cover both + periods, so none of the module's INFO drop lines fire.""" ispypsa_tables = _constraint_tables(csv_str_to_df) with caplog.at_level("INFO"): @@ -437,8 +492,9 @@ def test_no_one_sided_period_log_when_every_period_has_both_sides( ispypsa_tables, _links(csv_str_to_df), sample_model_config ) - assert "dropped (no LHS terms in that period)" not in caplog.text - assert "dropped (no RHS row in that period)" not in caplog.text + assert "link_flow terms dropped" not in caplog.text + assert "RHS rows dropped" not in caplog.text + assert "LHS terms dropped" not in caplog.text def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): From ace0978552732a064ce781bcfa8d2fecbb5d39f0 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 10:49:54 +1000 Subject: [PATCH 11/20] Hand allowed_expansion straight to the expansion-limit constraints allowed_expansion rode through the relaxation-generator pipeline without any step using it, only to be read off at the end. The orchestrator now passes the resolved options' caps directly to _create_expansion_limit_constraints, the rez_transmission_expansion gate moves into _resolve_relaxation_options where the decision is made, and _finalise_generators disappears. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 134 ++++++++++++-------------- 1 file changed, 61 insertions(+), 73 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 7e556f00..247a81fb 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -278,8 +278,15 @@ def _translate_custom_constraints_from_network_tables( lhs = _expand_link_flow_terms(lhs, links) lhs, rhs = _drop_one_sided_constraint_periods(lhs, rhs) + relaxations = _resolve_relaxation_options( + ispypsa_tables["network_expansion_options"], + sorted(set(rhs["constraint_id"])), + config, + ) relaxation_generators = _create_constraint_relaxation_generators( - ispypsa_tables, sorted(set(rhs["constraint_id"])), config + relaxations, + ispypsa_tables["network_transmission_path_expansion_costs"], + config, ) relaxation_generator_lhs = _relaxation_generator_lhs_terms( relaxation_generators, rhs @@ -287,8 +294,9 @@ def _translate_custom_constraints_from_network_tables( path_caps = _resolve_path_expansion_caps( ispypsa_tables["network_expansion_options"], links ) + relaxation_caps = relaxations.loc[:, ["expansion_id", "allowed_expansion"]] expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( - links, relaxation_generators, path_caps + links, relaxation_generators, path_caps, relaxation_caps ) lhs = _concat_non_empty( @@ -296,7 +304,6 @@ def _translate_custom_constraints_from_network_tables( ) rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) - relaxation_generators = _finalise_generators(relaxation_generators) return { "custom_constraints_lhs": lhs, "custom_constraints_rhs": rhs, @@ -510,8 +517,8 @@ def _log_one_sided_periods( def _create_constraint_relaxation_generators( - ispypsa_tables: dict[str, pd.DataFrame], - constraint_ids: list[str], + relaxations: pd.DataFrame, + expansion_costs: pd.DataFrame, config: ModelConfig, ) -> pd.DataFrame: """Builds one extendable dummy generator per relaxable constraint and @@ -521,72 +528,62 @@ def _create_constraint_relaxation_generators( by the constraint's direction (see _relaxation_generator_lhs_terms), so building them relaxes the constraint at the option's cost; total relaxation is capped at the option's allowed_expansion by the - expansion-limit constraints. + expansion-limit constraints, which take that cap from the same resolved + relaxations rather than from the generators. - Options and costs may use blank key cells as wildcards: a blank - expansion_id means "every constraint" — here, every constraint in the - model (constraint_ids) — and a blank cost year means "every investment - period", so a single blank-id row gives all constraints the same - relaxation option or cost, and a blank-year cost row is a static cost - across the periods. Every constraint with an option has a cost in every - investment period (the costs schema's coverage rule), so joining the two - gives exactly one generator per option and period. If the config's - rez_transmission_expansion flag is off, no relaxation generators are - built at all. + The costs table may use blank key cells as wildcards: a blank + expansion_id is a table-wide default cost and a blank year a static cost + across the investment periods (see _prepare_expansion_costs). Every + relaxable constraint has a cost in every investment period (the costs + schema's coverage rule), so there is exactly one generator per relaxation + and period. I/O Example (blank cells are wildcards): - ispypsa_tables["network_expansion_options"]: + relaxations (from _resolve_relaxation_options): expansion_id expansion_type allowed_expansion expansion_option SWQLD1 constraint_relaxation 500 Option 2 - constraint_relaxation 200 Default + NQ1 constraint_relaxation 200 Default - ispypsa_tables["network_transmission_path_expansion_costs"]: + expansion_costs: expansion_id year cost SWQLD1 2030 100000 80000 # every constraint, every period - constraint_ids = ["SWQLD1", "NQ1"] - - config: rez_transmission_expansion = True, investment_periods = [2030, 2040] + config: investment_periods = [2030, 2040] returns: - name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion - SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(100000) 500 - SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 500 - NQ1_exp_2030 NQ1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(80000) 200 - NQ1_exp_2040 NQ1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) 200 + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(100000) + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) + NQ1_exp_2030 NQ1 bus_for_custom_constraint_gens 0.0 True 2030 inf annuitise(80000) + NQ1_exp_2040 NQ1 bus_for_custom_constraint_gens 0.0 True 2040 inf annuitise(80000) """ - if not config.network.rez_transmission_expansion: - return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) - relaxations = _resolve_relaxation_options( - ispypsa_tables["network_expansion_options"], constraint_ids - ) costs = _prepare_expansion_costs( - ispypsa_tables["network_transmission_path_expansion_costs"], - constraint_ids, + expansion_costs, + sorted(set(relaxations["expansion_id"])), config.temporal.capacity_expansion.investment_periods, config.wacc, config.network.annuitisation_lifetime, ) - generators = costs.merge( - relaxations.loc[:, ["expansion_id", "allowed_expansion"]], on="expansion_id" - ) - return _format_relaxation_generators(generators) + return _format_relaxation_generators(costs) def _resolve_relaxation_options( - options: pd.DataFrame, constraint_ids: list[str] + options: pd.DataFrame, constraint_ids: list[str], config: ModelConfig ) -> pd.DataFrame: """Resolves the expansion-options wildcards to one constraint_relaxation - row per constraint in the model that has an option. + row per constraint in the model that has an option — or to no rows at all + when the config's rez_transmission_expansion flag is off. The physical forward/reverse rows are set aside first (they become expansion links in ispypsa.translator.network); a blank expansion_type covers constraint_relaxation too, so it is kept. Options for constraints not in the model are dropped, then _resolve_wildcards fans blank cells out - against the model's constraints, most specific row winning. + against the model's constraints, most specific row winning. The resolved + rows drive both the relaxation generators and, through allowed_expansion, + the expansion-limit caps. - I/O Example (blank cells are wildcards): + I/O Example (blank cells are wildcards; rez_transmission_expansion on): options: expansion_id expansion_type allowed_expansion expansion_option CQ-NQ forward 1000 BigLine # physical: set aside @@ -600,6 +597,8 @@ def _resolve_relaxation_options( SWQLD1 constraint_relaxation 400 Relax SWV1 constraint_relaxation 200 Default """ + if not config.network.rez_transmission_expansion: + return pd.DataFrame(columns=options.columns) expansion_type = options["expansion_type"] options = options[ expansion_type.isna() | (expansion_type == "constraint_relaxation") @@ -619,14 +618,14 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: I/O Example: generators: - expansion_id year capital_cost allowed_expansion - SWQLD1 2030 8140 500 - SWQLD1 2040 7900 500 + expansion_id year capital_cost + SWQLD1 2030 8140 + SWQLD1 2040 7900 returns: - name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost allowed_expansion - SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf 8140 500 - SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf 7900 500 + name isp_name bus p_nom p_nom_extendable build_year lifetime capital_cost + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2030 inf 8140 + SWQLD1_exp_2040 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2040 inf 7900 """ generators = generators.rename(columns={"expansion_id": "isp_name"}) generators["name"] = ( @@ -637,7 +636,7 @@ def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: generators["p_nom_extendable"] = True generators["build_year"] = generators["year"] generators["lifetime"] = np.inf - return generators.loc[:, _GENERATOR_COLUMNS + ["allowed_expansion"]] + return generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) def _relaxation_generator_lhs_terms( @@ -754,16 +753,17 @@ def _create_expansion_limit_constraints( links: pd.DataFrame, relaxation_generators: pd.DataFrame, path_caps: pd.DataFrame, + relaxation_caps: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Caps the total capacity built across each expandable element's per-period components at the selected option's capacity. - For physical paths the cap is path_caps' max(forward, reverse); for - constraint relaxations it is the option's allowed_expansion carried on the - relaxation generators. The constraints have no investment_period or - timeslice — they apply to the p_nom variables globally. Names get an - "_expansion_limit" suffix so a relaxation cap doesn't collide with the - constraint it relaxes. + The components are the paths' expansion links and the constraints' + relaxation generators; the caps are path_caps' max(forward, reverse) and + relaxation_caps' allowed_expansion, one row per element in each. The + constraints have no investment_period or timeslice — they apply to the + p_nom variables globally. Names get an "_expansion_limit" suffix so a + relaxation cap doesn't collide with the constraint it relaxes. I/O Example: links: @@ -773,12 +773,12 @@ def _create_expansion_limit_constraints( CQ-NQ CQ-NQ_exp_2040 True relaxation_generators: - name isp_name allowed_expansion - SWQLD1_exp_2030 SWQLD1 500 + name isp_name + SWQLD1_exp_2030 SWQLD1 - path_caps: - expansion_id allowed_expansion - CQ-NQ 1000 + path_caps: relaxation_caps: + expansion_id allowed_expansion expansion_id allowed_expansion + CQ-NQ 1000 SWQLD1 500 returns lhs: constraint_id variable_name component attribute coefficient investment_period @@ -798,9 +798,7 @@ def _create_expansion_limit_constraints( ], ignore_index=True, ) - relaxation_caps = relaxation_generators.loc[:, ["isp_name", "allowed_expansion"]] - relaxation_caps = relaxation_caps.rename(columns={"isp_name": "expansion_id"}) - caps = pd.concat([path_caps, relaxation_caps.drop_duplicates()], ignore_index=True) + caps = pd.concat([path_caps, relaxation_caps], ignore_index=True) rhs = _expansion_limit_rhs(caps) lhs["constraint_id"] = lhs["constraint_id"] + "_expansion_limit" rhs["constraint_id"] = rhs["constraint_id"] + "_expansion_limit" @@ -879,13 +877,3 @@ def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: f"Duplicate custom constraint RHS rows for: " f"{sorted(set(duplicates['constraint_name']))}" ) - - -def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: - """Drops the allowed_expansion working column carried for the - expansion-limit RHS, leaving the PyPSA generator columns. - - I/O Example: - columns [*_GENERATOR_COLUMNS, allowed_expansion] -> _GENERATOR_COLUMNS - """ - return relaxation_generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) From a942bec0ecf74def0f92686a6a9f3f01813f6e1e Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 12:15:06 +1000 Subject: [PATCH 12/20] Tell the constraints docstring story once, on the orchestrator The module docstring and the orchestrator's docstring had become two accounts of the same pipeline, with fixes having to land in both. The orchestrator docstring now carries the lot -- the translation steps, the relaxation and expansion-limit behaviour, the schema rules the pipeline trusts without re-checking, and the blank-timeslice fallback semantics -- next to the I/O example, and the module docstring is deleted. Accuracy fixes folded in along the way: the one-sided period drop names both of its causes (date_from coverage and model scope), the flow-path cap is stated as max(forward, reverse), link_flow expansion is described as replacing the path_id term with per-link terms, and the rez_transmission_expansion gate, per-option-per-period generators and annuitised costs are now stated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 193 ++++++++++---------------- 1 file changed, 70 insertions(+), 123 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 247a81fb..86c2b12e 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -1,119 +1,3 @@ -"""Translate the new-format custom-constraint tables into PyPSA friendly form. - -This module sits in the translator stage alongside ispypsa.translator.network. -It turns the templated custom-constraint tables (PLEXOS-derived group -constraints such as SWQLD1) into the LHS/RHS tables pypsa_build applies as -linopy constraints, and adds the endogenous expansion-limit constraints, with -their constraint-relaxation generators, that cap how much capacity the model -can build for each expandable network element. - -The inputs are the three custom-constraint tables — custom_constraints (one -row per constraint), custom_constraints_lhs (one row per term per date_from) -and custom_constraints_rhs (one row per timeslice per date_from): - - custom_constraints: custom_constraints_rhs: - constraint_id direction constraint_id timeslice rhs date_from - SWQLD1 <= SWQLD1 qld_peak_demand 3000 - SWQLD1 qld_peak_demand 2500 2027-07-01T00:00:00 - - custom_constraints_lhs: - constraint_id term_type variable_name coefficient date_from - SWQLD1 link_flow NSW-QLD 0.84 - SWQLD1 generator_output KINGASF1 0.14 - -plus network_expansion_options and network_transmission_path_expansion_costs -(the unified expansion tables, see ispypsa.translator.network) and the PyPSA -friendly links table (existing plus expansion links). - -The outputs are the PyPSA friendly custom_constraints_rhs (one row per -constraint, investment period and timeslice), custom_constraints_lhs (one row -per constraint, investment period and term) and custom_constraints_generators -(one row per relaxable constraint and investment period): - - custom_constraints_rhs: - constraint_name investment_period timeslice rhs constraint_type - SWQLD1 2026 qld_peak_demand 3000 <= - SWQLD1 2028 qld_peak_demand 2500 <= - NSW-QLD_expansion_limit 1000 <= - SWQLD1_expansion_limit 400 <= - - custom_constraints_lhs (2028 rows mirror 2026): - constraint_name investment_period variable_name component attribute coefficient - SWQLD1 2026 NSW-QLD_existing Link p 0.84 - SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 - SWQLD1 2026 KINGASF1 Generator p 0.14 - SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 - NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 - SWQLD1_expansion_limit SWQLD1_exp_2026 Generator p_nom 1.0 - - custom_constraints_generators (abridged): - name isp_name bus p_nom p_nom_extendable build_year capital_cost - SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 True 2026 annuitise(100000) - -A blank investment_period means the row applies in every period. A named -timeslice scopes the RHS to the snapshots inside that timeslice's windows -(the timeslices table); a blank timeslice is the constraint's fallback, -applying at the snapshots none of its named-timeslice rows cover, so a -constraint with only named rows does not bind outside them. This module -passes timeslice through untouched — resolving it to snapshots is -pypsa_build's job when the constraints are applied. - -The pipeline runs as follows. The date_from column of the LHS and RHS tables -is resolved into one row per investment period: for each period, each group -(a constraint's term, or a constraint's timeslice) keeps the row active at -the period's start — the latest date_from on or before it, with no-date_from -rows as the baseline. The RHS gains each constraint's sense from -custom_constraints as constraint_type. LHS term_types map to the PyPSA -component and attribute their variable belongs to, and each link_flow term -is expanded from its path_id to every link the model has on that path — the -existing link and each expansion link — so flow through new builds counts -towards the constraint too; terms for paths not in the model are dropped. -Because date resolution can leave a constraint with terms but no limit (or a -limit but no terms) in some periods, the two tables are then reconciled -period by period: a constraint is kept only in the periods where it has both -LHS terms and an RHS row, and the one-sided periods are dropped and logged. - -Constraint relaxation comes next, gated by the config's -rez_transmission_expansion flag. Each constraint that has a -constraint_relaxation expansion option gets one extendable dummy generator -per investment period at the option's annualised cost. The generator's p_nom -enters the parent constraint's LHS, in each period the constraint binds in, -with a sign chosen by the constraint's direction so that building it always -loosens the constraint: subtracted from a "<=" it raises the cap, added to a -">=" it lowers the floor. Finally the expansion-limit constraints cap the total -p_nom built across each expandable element's per-period components: for a -path the cap is max(forward, reverse) of its option, matching the per-unit -ratings ispypsa.translator.network gives its expansion links; for a -relaxation it is the option's allowed_expansion. Both tables are then -finalised — constraint_id becomes constraint_name and duplicate constraint -names are rejected. - -Input integrity is the table schemas' job, not this module's. The rules the -pipeline relies on without re-checking — unique input rows, a direction for -every constraint with RHS values, the LHS and RHS naming the same -constraints, every LHS variable_name naming a component in the table its -term_type refers to, no constraint_relaxation option on an "=" constraint, -and a cost for every expandable element in every investment period — are -declared in src/ispypsa/validation/schemas (custom_constraints*.yaml, -network_expansion_options.yaml and -network_transmission_path_expansion_costs.yaml). - -Reference detail: - -- direction to constraint_type: "<=" and ">=" pass through, "=" becomes "==". -- relaxation generator coefficient: -1.0 on a "<=" constraint, +1.0 on a ">=". -- term_type to component/attribute lives in ispypsa.translator.mappings - (_CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE and _..._ATTRIBUTE_TYPE). -- Expansion-limit constraints are named "_expansion_limit" so a - relaxation cap doesn't collide with the constraint it relaxes. -- Dropped rows: link_flow terms whose path is not in the model (logged); per - investment period, RHS rows of a constraint with no LHS terms in that - period and LHS terms of a constraint with no RHS row in that period (both - logged); relaxation options and costs for constraints not in the model, or - all of them when rez_transmission_expansion is off; date_from rows that - only start after every investment period. -""" - import logging import numpy as np @@ -201,13 +85,76 @@ def _translate_custom_constraints_from_network_tables( """Translates the custom-constraint tables and builds the endogenous expansion-limit constraints. - Consumes the custom_constraints, custom_constraints_lhs, - custom_constraints_rhs, network_expansion_options and - network_transmission_path_expansion_costs tables, plus the PyPSA friendly - links table from ispypsa.translator.network (existing plus expansion - links). Generator, storage and load terms pass through with their IASR - IDs as variable_names, unchecked: the custom_constraints_lhs schema ties - every variable_name to its component table. + Custom constraint tables are translated to by: + + - determining the LHS and RHS values active during each investment period. Values + with the most recent date_from date falling on or before the start of an + investment are taken as active for that period. Blank date_from row are treated as + the earliest values. + - term_type values are mapped to PyPSA components and attribute combinations. + - LHS link_flow terms are expanded from their path_id into one term per link + the model has on that path, the existing link and each expansion link, so + flow through new builds counts towards the constraint. + - LHS and RHS rows are dropped in investment periods where the constraint does + not have both LHS terms and an RHS value. This happens when date_from coverage + differs between the two sides (including a side whose earliest date_from falls + after a period's start), or when all of a constraint's LHS terms reference + components outside the configured model scope (e.g. flow path links when + regional_granularity is single_region). + + Dummy generators are added to the LHS of constraints with relaxation options and + costs: + + - Constraint relaxation is gated by the config's rez_transmission_expansion + flag: with it off, no dummy generators or relaxation expansion-limit + constraints are created. + - For rows in network_expansion_options with an expansion_type of + constraint_relaxation dummy generators capacity values are added to the LHS of the + constraint specified by the expansion_id column. + - One dummy generator is added per relaxation option per investment period. A + period's constraint LHS carries every generator built up to that period, so + the relaxation available accumulates across the horizon. + - The dummy generators allow PyPSA to invest in relaxing the constraint. Their + capital_cost is the option's cost from network_transmission_path_expansion_costs, + annuitised with the config's wacc and annuitisation_lifetime. + - The generator definitions are returned in PyPSA friendly format in the + table custom_constraints_generators. + - For <= constraints the generator LHS term is negative and for >= the LHS is + positive. + + For each set of expansion links for a given transmission flow path and constraint + relaxation dummy generators, an additional constraint is created limiting the total + transmission expansion or constraint relaxation built across the investment + periods. + + - The constraint RHS is derived from allowed_expansion in + network_expansion_options, with a constraint_type of <=. For a relaxation the + option's allowed_expansion is used directly; for a flow path the RHS is + max(forward, reverse). + - Each of the expansion links or dummy generators capacity (p_nom) values are + added to the constraint LHS. + - These constraints are returned appended to the custom_constraints_lhs and + custom_constraints_rhs tables. + + Input integrity is the table schemas' job, not this module's. The rules the + pipeline relies on without re-checking — unique input rows, a direction for + every constraint with RHS values, the LHS and RHS naming the same + constraints, every LHS variable_name matching an ID in one of the input + tables for its term_type (e.g. a generator_output term names a generator + in generators_existing_planned or generators_new_entrant), no + constraint_relaxation option on an "=" constraint, and a cost for every + expandable element in every investment period — are declared in + src/ispypsa/validation/schemas (custom_constraints*.yaml, + network_expansion_options.yaml and + network_transmission_path_expansion_costs.yaml). + + In the output tables, a blank investment_period means the row applies in + every period. A named timeslice scopes the RHS to the snapshots inside that + timeslice's windows (the timeslices table); a blank timeslice is the + constraint's fallback, applying at the snapshots none of its named-timeslice + rows cover, so a constraint with only named rows does not bind outside them. + Timeslice values pass through untouched — resolving them to snapshots is + pypsa_build's job when the constraints are applied. I/O Example (config: investment periods 2026 and 2028): ispypsa_tables["custom_constraints"]: From 087239bc97206a14ea0f76adaa671c157916c0b5 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 13:27:05 +1000 Subject: [PATCH 13/20] Lay each I/O example table out in its own block The side-by-side table pairs and the compressed links one-liner in the constraints I/O examples saved vertical space at the cost of scanning -- every input table now sits sequentially in the same tabular format as the rest of the examples. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 34 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 86c2b12e..6114673e 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -384,7 +384,10 @@ def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFr SWQLD1 NSW-QLD Link 0.84 SWQLD1 KINGASF1 Generator 0.14 - links (isp_name -> name): NSW-QLD -> NSW-QLD_existing, NSW-QLD_exp_2030 + links: + isp_name name + NSW-QLD NSW-QLD_existing + NSW-QLD NSW-QLD_exp_2030 returns: constraint_id variable_name component coefficient @@ -425,11 +428,17 @@ def _drop_one_sided_constraint_periods( the model. Either way the constraint can't be applied in that period. I/O Example: - lhs (abridged): rhs (abridged): - constraint_id investment_period constraint_id investment_period - SWQLD1 2028 SWQLD1 2026 - NQ1 2026 SWQLD1 2028 - NQ1 2028 NQ1 2026 + lhs (abridged): + constraint_id investment_period + SWQLD1 2028 + NQ1 2026 + NQ1 2028 + + rhs (abridged): + constraint_id investment_period + SWQLD1 2026 + SWQLD1 2028 + NQ1 2026 returns: lhs without its NQ1 2028 term (no RHS row that period; logged) @@ -475,8 +484,7 @@ def _create_constraint_relaxation_generators( by the constraint's direction (see _relaxation_generator_lhs_terms), so building them relaxes the constraint at the option's cost; total relaxation is capped at the option's allowed_expansion by the - expansion-limit constraints, which take that cap from the same resolved - relaxations rather than from the generators. + expansion-limit constraints. The costs table may use blank key cells as wildcards: a blank expansion_id is a table-wide default cost and a blank year a static cost @@ -723,9 +731,13 @@ def _create_expansion_limit_constraints( name isp_name SWQLD1_exp_2030 SWQLD1 - path_caps: relaxation_caps: - expansion_id allowed_expansion expansion_id allowed_expansion - CQ-NQ 1000 SWQLD1 500 + path_caps: + expansion_id allowed_expansion + CQ-NQ 1000 + + relaxation_caps: + expansion_id allowed_expansion + SWQLD1 500 returns lhs: constraint_id variable_name component attribute coefficient investment_period From b915e04860a84f7add286e290ac2bd30b50d138e Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 15:49:44 +1000 Subject: [PATCH 14/20] Halt on constraint terms the model can't honour, and split the orchestrator Dropping a term whose component isn't in the model left the constraint applied with a weakened LHS -- plausible-looking and silently wrong. The translator now raises when any LHS term fails to resolve to a model component (load terms, which pypsa_build doesn't implement, raise too), and resolves every term through an isp_name -> name mapping supplied by new generators and storage inputs, so new entrant units expand to their per-build-year components the same way link terms expand over a path's links. The generator/storage frame contract (isp_name + name, mirroring links) is assumed ahead of the translators that will produce it, for review by the team building that side. The orchestrator is renamed _translate_custom_constraints (the constraints aren't network-specific any more) and split into three block producers -- translate the user-authored tables, create the constraint relaxations, create the expansion limits -- assembled and finalised once, so future producers of constraint blocks (e.g. new entrant build limits) have a seam to append into before relaxation resolves. The reusable pieces' docstrings now state their source-agnostic contracts rather than relaxation-specific vocabulary. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 495 ++++++++++++++++------ tests/test_translator/test_constraints.py | 256 +++++++++-- 2 files changed, 569 insertions(+), 182 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 6114673e..fa864a2e 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -77,64 +77,31 @@ def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.Data return pd.concat(non_empty, ignore_index=True) -def _translate_custom_constraints_from_network_tables( +def _translate_custom_constraints( ispypsa_tables: dict[str, pd.DataFrame], links: pd.DataFrame, + generators: pd.DataFrame, + storage: pd.DataFrame, config: ModelConfig, ) -> dict[str, pd.DataFrame]: - """Translates the custom-constraint tables and builds the endogenous - expansion-limit constraints. - - Custom constraint tables are translated to by: - - - determining the LHS and RHS values active during each investment period. Values - with the most recent date_from date falling on or before the start of an - investment are taken as active for that period. Blank date_from row are treated as - the earliest values. - - term_type values are mapped to PyPSA components and attribute combinations. - - LHS link_flow terms are expanded from their path_id into one term per link - the model has on that path, the existing link and each expansion link, so - flow through new builds counts towards the constraint. - - LHS and RHS rows are dropped in investment periods where the constraint does - not have both LHS terms and an RHS value. This happens when date_from coverage - differs between the two sides (including a side whose earliest date_from falls - after a period's start), or when all of a constraint's LHS terms reference - components outside the configured model scope (e.g. flow path links when - regional_granularity is single_region). - - Dummy generators are added to the LHS of constraints with relaxation options and - costs: - - - Constraint relaxation is gated by the config's rez_transmission_expansion - flag: with it off, no dummy generators or relaxation expansion-limit - constraints are created. - - For rows in network_expansion_options with an expansion_type of - constraint_relaxation dummy generators capacity values are added to the LHS of the - constraint specified by the expansion_id column. - - One dummy generator is added per relaxation option per investment period. A - period's constraint LHS carries every generator built up to that period, so - the relaxation available accumulates across the horizon. - - The dummy generators allow PyPSA to invest in relaxing the constraint. Their - capital_cost is the option's cost from network_transmission_path_expansion_costs, - annuitised with the config's wacc and annuitisation_lifetime. - - The generator definitions are returned in PyPSA friendly format in the - table custom_constraints_generators. - - For <= constraints the generator LHS term is negative and for >= the LHS is - positive. - - For each set of expansion links for a given transmission flow path and constraint - relaxation dummy generators, an additional constraint is created limiting the total - transmission expansion or constraint relaxation built across the investment - periods. - - - The constraint RHS is derived from allowed_expansion in - network_expansion_options, with a constraint_type of <=. For a relaxation the - option's allowed_expansion is used directly; for a flow path the RHS is - max(forward, reverse). - - Each of the expansion links or dummy generators capacity (p_nom) values are - added to the constraint LHS. - - These constraints are returned appended to the custom_constraints_lhs and - custom_constraints_rhs tables. + """Translates the custom-constraint tables and appends the endogenous + expansion-limit constraints, returning them in PyPSA friendly form. + + Three constraint blocks are produced in sequence and assembled here. + _translate_constraint_tables turns the user-authored constraint tables + into per-investment-period LHS/RHS rows. _create_constraint_relaxations + adds extendable dummy generators that let the model buy relaxation of + those constraints, with the LHS terms wiring them in. + _create_expansion_limit_constraints caps the total capacity built for + each expandable element — expansion links and relaxation generators + alike. The blocks are concatenated and finalised once: constraint_id + becomes constraint_name, and duplicate (constraint, period, timeslice) + RHS rows across the assembled set raise a ValueError, since pypsa_build + would build two constraints with the same name from them. Relaxation + resolves against the assembled RHS, + so a future producer of constraint blocks (e.g. new entrant build limits) + should append before the relaxation step if its constraints are to be + relaxable. Input integrity is the table schemas' job, not this module's. The rules the pipeline relies on without re-checking — unique input rows, a direction for @@ -166,9 +133,11 @@ def _translate_custom_constraints_from_network_tables( SWQLD1 qld_peak_demand 3000 ispypsa_tables["custom_constraints_lhs"]: - constraint_id term_type variable_name coefficient date_from - SWQLD1 link_flow NSW-QLD 0.84 - SWQLD1 generator_output KINGASF1 0.14 + constraint_id term_type variable_name coefficient date_from + SWQLD1 link_flow NSW-QLD 0.84 + SWQLD1 generator_output KINGASF1 0.14 + SWQLD1 generator_output N2 Solar 0.5 + SWQLD1 storage_output Q8 Battery - 2h 0.43 ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option @@ -179,13 +148,28 @@ def _translate_custom_constraints_from_network_tables( ispypsa_tables["network_transmission_path_expansion_costs"]: expansion_id year cost NSW-QLD 2026 500000 + NSW-QLD 2028 500000 SWQLD1 2026 100000 + SWQLD1 2028 80000 links: isp_name name p_nom_extendable NSW-QLD NSW-QLD_existing False NSW-QLD NSW-QLD_exp_2026 True + generators (isp_name = name for existing units; a new entrant's ID + maps to each of its per-build-year components): + isp_name name + KINGASF1 KINGASF1 + N2 Solar N2 Solar_2026 + N2 Solar N2 Solar_2028 + + storage: + isp_name name + Q8 Battery - 2h Q8 Battery - 2h + SQ BESS SQ BESS_2026 + SQ BESS SQ BESS_2028 + returns["custom_constraints_rhs"]: constraint_name investment_period timeslice rhs constraint_type SWQLD1 2026 qld_peak_demand 3000 <= @@ -198,13 +182,124 @@ def _translate_custom_constraints_from_network_tables( SWQLD1 2026 NSW-QLD_existing Link p 0.84 SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 SWQLD1 2026 KINGASF1 Generator p 0.14 + SWQLD1 2026 N2 Solar_2026 Generator p 0.5 + SWQLD1 2026 N2 Solar_2028 Generator p 0.5 + SWQLD1 2026 Q8 Battery - 2h Storage p 0.43 SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 + SWQLD1 2028 SWQLD1_exp_2028 Generator p_nom -1.0 # relaxation accumulates NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 SWQLD1_expansion_limit SWQLD1_exp_2026 Generator p_nom 1.0 + SWQLD1_expansion_limit SWQLD1_exp_2028 Generator p_nom 1.0 returns["custom_constraints_generators"] (abridged): name isp_name bus p_nom build_year capital_cost SWQLD1_exp_2026 SWQLD1 bus_for_custom_constraint_gens 0.0 2026 annuitise(100000) + SWQLD1_exp_2028 SWQLD1 bus_for_custom_constraint_gens 0.0 2028 annuitise(80000) + """ + lhs, rhs = _translate_constraint_tables( + ispypsa_tables, links, generators, storage, config + ) + relaxation_generators, relaxation_lhs, relaxation_caps = ( + _create_constraint_relaxations( + ispypsa_tables["network_expansion_options"], + ispypsa_tables["network_transmission_path_expansion_costs"], + rhs, + config, + ) + ) + expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( + ispypsa_tables["network_expansion_options"], + links, + relaxation_generators, + relaxation_caps, + ) + lhs = _concat_non_empty( + [lhs, relaxation_lhs, expansion_limit_lhs], _INTERNAL_LHS_COLUMNS + ) + rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) + lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + return { + "custom_constraints_lhs": lhs, + "custom_constraints_rhs": rhs, + "custom_constraints_generators": relaxation_generators, + } + + +def _translate_constraint_tables( + ispypsa_tables: dict[str, pd.DataFrame], + links: pd.DataFrame, + generators: pd.DataFrame, + storage: pd.DataFrame, + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Translates the user-authored custom-constraint tables into one LHS term + and one RHS row per constraint, investment period and (RHS only) + timeslice, still keyed by constraint_id. + + The translation steps: + + - the LHS and RHS values active during each investment period are + determined: each group keeps the value with the most recent date_from + falling on or before the period's start, blank date_from rows acting + as the earliest values. + - term_type values are mapped to PyPSA component and attribute + combinations. + - every LHS term must resolve to a component in the model. A term naming a + component the configured model doesn't contain (e.g. a link_flow term when + regional_granularity is single_region builds no links) raises, since applying + the constraint without the term would silently weaken it. load terms aren't + implemented in pypsa_build and also raise. + - LHS terms are expanded from their input IDs into one term per matching + model component (the links, generators and storage tables' isp_name to name + mapping): a link_flow term covers its path's existing link and each expansion + link, and a term on a new entrant generator or storage unit covers each of + its per-build-year components. + - LHS and RHS rows are dropped in investment periods where the constraint does + not have both LHS terms and an RHS value. This happens when date_from coverage + differs between the two sides (including a side whose earliest date_from falls + after a period's start). + + I/O Example (config: investment periods 2026 and 2028): + ispypsa_tables["custom_constraints"]: + constraint_id direction + SWQLD1 <= + + ispypsa_tables["custom_constraints_rhs"]: + constraint_id timeslice rhs date_from + SWQLD1 qld_peak_demand 3000 + + ispypsa_tables["custom_constraints_lhs"]: + constraint_id term_type variable_name coefficient date_from + SWQLD1 link_flow NSW-QLD 0.84 + SWQLD1 generator_output KINGASF1 0.14 + + links: + isp_name name p_nom_extendable + NSW-QLD NSW-QLD_existing False + NSW-QLD NSW-QLD_exp_2026 True + + generators: + isp_name name + KINGASF1 KINGASF1 + N2 Solar N2 Solar_2026 + N2 Solar N2 Solar_2028 + + storage: + isp_name name + Q8 Battery - 2h Q8 Battery - 2h + SQ BESS SQ BESS_2026 + SQ BESS SQ BESS_2028 + + returns lhs (2028 rows mirror 2026): + constraint_id investment_period variable_name component attribute coefficient + SWQLD1 2026 NSW-QLD_existing Link p 0.84 + SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 + SWQLD1 2026 KINGASF1 Generator p 0.14 + + returns rhs: + constraint_id investment_period timeslice rhs constraint_type + SWQLD1 2026 qld_peak_demand 3000 <= + SWQLD1 2028 qld_peak_demand 3000 <= """ period_starts = _investment_period_start_dates( config.temporal.capacity_expansion.investment_periods, @@ -222,40 +317,11 @@ def _translate_custom_constraints_from_network_tables( period_starts, ) lhs = _add_component_and_attribute(lhs) - lhs = _expand_link_flow_terms(lhs, links) - lhs, rhs = _drop_one_sided_constraint_periods(lhs, rhs) - - relaxations = _resolve_relaxation_options( - ispypsa_tables["network_expansion_options"], - sorted(set(rhs["constraint_id"])), - config, - ) - relaxation_generators = _create_constraint_relaxation_generators( - relaxations, - ispypsa_tables["network_transmission_path_expansion_costs"], - config, - ) - relaxation_generator_lhs = _relaxation_generator_lhs_terms( - relaxation_generators, rhs - ) - path_caps = _resolve_path_expansion_caps( - ispypsa_tables["network_expansion_options"], links - ) - relaxation_caps = relaxations.loc[:, ["expansion_id", "allowed_expansion"]] - expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( - links, relaxation_generators, path_caps, relaxation_caps - ) - - lhs = _concat_non_empty( - [lhs, relaxation_generator_lhs, expansion_limit_lhs], _INTERNAL_LHS_COLUMNS - ) - rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) - lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) - return { - "custom_constraints_lhs": lhs, - "custom_constraints_rhs": rhs, - "custom_constraints_generators": relaxation_generators, - } + model_components = _model_component_names(links, generators, storage) + _raise_on_load_terms(lhs) + _raise_on_terms_not_in_model(lhs, model_components) + lhs = _expand_terms_to_model_components(lhs, model_components) + return _drop_one_sided_constraint_periods(lhs, rhs) def _investment_period_start_dates( @@ -373,10 +439,101 @@ def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: ) -def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFrame: - """Replaces each link term's path_id with the model's link names — the - existing link plus each expansion link — one term per link. Terms for - paths not in the model are dropped and logged. +def _model_component_names( + links: pd.DataFrame, generators: pd.DataFrame, storage: pd.DataFrame +) -> pd.DataFrame: + """One row per model component an LHS term can resolve to: the component + type, the ID the constraint tables refer to it by (isp_name) and the + model component's name. + + I/O Example: + links: + isp_name name + NSW-QLD NSW-QLD_existing + NSW-QLD NSW-QLD_exp_2030 + + generators: + isp_name name + KINGASF1 KINGASF1 + N2 Solar N2 Solar_2030 + N2 Solar N2 Solar_2040 + + storage: + isp_name name + Q8 Battery - 2h Q8 Battery - 2h + SQ BESS SQ BESS_2030 + SQ BESS SQ BESS_2040 + + returns: + isp_name name component + NSW-QLD NSW-QLD_existing Link + NSW-QLD NSW-QLD_exp_2030 Link + KINGASF1 KINGASF1 Generator + N2 Solar N2 Solar_2030 Generator + N2 Solar N2 Solar_2040 Generator + Q8 Battery - 2h Q8 Battery - 2h Storage + SQ BESS SQ BESS_2030 Storage + SQ BESS SQ BESS_2040 Storage + """ + frames = [ + links.loc[:, ["isp_name", "name"]].assign(component="Link"), + generators.loc[:, ["isp_name", "name"]].assign(component="Generator"), + storage.loc[:, ["isp_name", "name"]].assign(component="Storage"), + ] + return _concat_non_empty(frames, ["isp_name", "name", "component"]) + + +def _raise_on_load_terms(lhs: pd.DataFrame) -> None: + """Raises for load terms — load variables aren't implemented in + pypsa_build, so a constraint carrying one can't be applied as specified.""" + load_terms = lhs[lhs["component"] == "Load"] + if not load_terms.empty: + raise ValueError( + "Custom constraint load terms are not supported; constraints " + f"with load terms: {sorted(set(load_terms['constraint_id']))}" + ) + + +def _raise_on_terms_not_in_model( + lhs: pd.DataFrame, model_components: pd.DataFrame +) -> None: + """Raises when a term references a component with no match in the model — + e.g. a link_flow term when regional_granularity is single_region builds no + links. Applying the constraint without the term would silently weaken it, + so the run halts instead. + + I/O Example: + lhs: + constraint_id variable_name component + SWQLD1 NSW-QLD Link + SWQLD1 KINGASF1 Generator + + model_components with only ("NSW-QLD", Link) raises: + "... components not in the model: [('SWQLD1', 'KINGASF1')]" + """ + ids = model_components.loc[:, ["component", "isp_name"]].drop_duplicates() + matched = lhs.merge( + ids, + how="left", + left_on=["component", "variable_name"], + right_on=["component", "isp_name"], + ) + missing = matched[matched["isp_name"].isna()] + if not missing.empty: + pairs = sorted(set(zip(missing["constraint_id"], missing["variable_name"]))) + raise ValueError( + f"Custom constraint LHS terms reference components not in the " + f"model: {pairs}" + ) + + +def _expand_terms_to_model_components( + lhs: pd.DataFrame, model_components: pd.DataFrame +) -> pd.DataFrame: + """Replaces each term's input ID with the model components it covers, one + term per component: a link_flow term covers its path's existing and + expansion links, and a term on a new entrant generator or storage unit + covers each of its per-build-year components. I/O Example: lhs: @@ -384,37 +541,25 @@ def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFr SWQLD1 NSW-QLD Link 0.84 SWQLD1 KINGASF1 Generator 0.14 - links: - isp_name name - NSW-QLD NSW-QLD_existing - NSW-QLD NSW-QLD_exp_2030 + model_components: + isp_name name component + NSW-QLD NSW-QLD_existing Link + NSW-QLD NSW-QLD_exp_2030 Link + KINGASF1 KINGASF1 Generator returns: constraint_id variable_name component coefficient - SWQLD1 KINGASF1 Generator 0.14 SWQLD1 NSW-QLD_existing Link 0.84 SWQLD1 NSW-QLD_exp_2030 Link 0.84 + SWQLD1 KINGASF1 Generator 0.14 """ - link_terms = lhs[lhs["component"] == "Link"] - other_terms = lhs[lhs["component"] != "Link"] - _log_link_terms_not_in_model(link_terms, links) - expanded = link_terms.merge( - links.loc[:, ["isp_name", "name"]], left_on="variable_name", right_on="isp_name" + expanded = lhs.merge( + model_components, + left_on=["component", "variable_name"], + right_on=["component", "isp_name"], ) expanded = expanded.drop(columns=["variable_name", "isp_name"]) - expanded = expanded.rename(columns={"name": "variable_name"}) - return pd.concat([other_terms, expanded], ignore_index=True) - - -def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) -> None: - """Logs the link_flow terms whose path has no link in the model (they are - dropped by the merge in _expand_link_flow_terms).""" - missing = set(link_terms["variable_name"]) - set(links["isp_name"]) - if missing: - logger.info( - f"Custom constraint link_flow terms dropped (paths not in model): " - f"{sorted(missing)}" - ) + return expanded.rename(columns={"name": "variable_name"}) def _drop_one_sided_constraint_periods( @@ -423,9 +568,9 @@ def _drop_one_sided_constraint_periods( """Keeps each constraint only in the investment periods where it has both LHS terms and an RHS row, dropping (and logging) the one-sided periods. - A period is one-sided when one side's date_from starts later than the - other's, or when every LHS term was dropped because its path is not in - the model. Either way the constraint can't be applied in that period. + A period is one-sided when one side's date_from coverage starts later + than the other's, leaving the constraint with terms but no limit (or a + limit but no terms) in the earlier periods, where it can't be applied. I/O Example: lhs (abridged): @@ -472,6 +617,74 @@ def _log_one_sided_periods( ) +def _create_constraint_relaxations( + options: pd.DataFrame, + expansion_costs: pd.DataFrame, + rhs: pd.DataFrame, + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """Creates the dummy generators that let PyPSA invest in relaxing + constraints with a constraint_relaxation expansion option, the LHS terms + wiring them into their constraints, and the caps on how much can be built. + + - Constraint relaxation is gated by the config's rez_transmission_expansion + flag: with it off, no dummy generators or relaxation expansion-limit + constraints are created. + - For rows in network_expansion_options with an expansion_type of + constraint_relaxation, dummy generator capacity is added to the LHS of the + constraint named by the expansion_id column. + - One dummy generator is created per relaxation option per investment period. + A period's constraint LHS carries every generator built up to that period, so + the relaxation available accumulates across the horizon. + - The generators' capital_cost is the option's cost from + network_transmission_path_expansion_costs, annuitised with the config's wacc + and annuitisation_lifetime. + - For <= constraints the generator LHS terms are negative and for >= they + are positive, so building capacity always loosens the constraint. + - Each option's allowed_expansion is returned as its cap, for + _create_expansion_limit_constraints to bound the total relaxation built. + + I/O Example (investment periods 2026 and 2028; rez_transmission_expansion on): + options: + expansion_id expansion_type allowed_expansion expansion_option + SWQLD1 constraint_relaxation 400 Option 2 + + expansion_costs: + expansion_id year cost + SWQLD1 2026 100000 + SWQLD1 2028 80000 + + rhs (abridged): + constraint_id investment_period constraint_type + SWQLD1 2026 <= + SWQLD1 2028 <= + + returns generators (abridged): + name isp_name build_year capital_cost + SWQLD1_exp_2026 SWQLD1 2026 annuitise(100000) + SWQLD1_exp_2028 SWQLD1 2028 annuitise(80000) + + returns lhs terms: + constraint_id investment_period variable_name component attribute coefficient + SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 + SWQLD1 2028 SWQLD1_exp_2026 Generator p_nom -1.0 + SWQLD1 2028 SWQLD1_exp_2028 Generator p_nom -1.0 + + returns caps: + expansion_id allowed_expansion + SWQLD1 400 + """ + relaxations = _resolve_relaxation_options( + options, sorted(set(rhs["constraint_id"])), config + ) + relaxation_generators = _create_constraint_relaxation_generators( + relaxations, expansion_costs, config + ) + relaxation_lhs = _relaxation_generator_lhs_terms(relaxation_generators, rhs) + caps = relaxations.loc[:, ["expansion_id", "allowed_expansion"]] + return relaxation_generators, relaxation_lhs, caps + + def _create_constraint_relaxation_generators( relaxations: pd.DataFrame, expansion_costs: pd.DataFrame, @@ -605,7 +818,10 @@ def _relaxation_generator_lhs_terms( a later period can't relax an earlier period's constraint. Each term's sign follows the parent constraint's direction (see _relaxation_coefficients) so that building capacity always loosens the - constraint. + constraint. The function works from any generators frame carrying name, + isp_name (the parent constraint) and build_year, against any RHS carrying + constraint_id, investment_period and constraint_type — it isn't specific + to where either came from. I/O Example: relaxation_generators: @@ -705,22 +921,33 @@ def _resolve_path_expansion_caps( def _create_expansion_limit_constraints( + options: pd.DataFrame, links: pd.DataFrame, relaxation_generators: pd.DataFrame, - path_caps: pd.DataFrame, relaxation_caps: pd.DataFrame, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Caps the total capacity built across each expandable element's per-period components at the selected option's capacity. The components are the paths' expansion links and the constraints' - relaxation generators; the caps are path_caps' max(forward, reverse) and - relaxation_caps' allowed_expansion, one row per element in each. The - constraints have no investment_period or timeslice — they apply to the - p_nom variables globally. Names get an "_expansion_limit" suffix so a - relaxation cap doesn't collide with the constraint it relaxes. + relaxation generators. A path's cap is max(forward, reverse) of its + resolved expansion option (see _resolve_path_expansion_caps); a + relaxation's cap is its option's allowed_expansion, passed in as + relaxation_caps. Each cap becomes an RHS row with constraint_type "<=" + and no investment_period or timeslice — it applies to the p_nom variables + globally — and each component contributes a coefficient-1.0 p_nom LHS + term. Names get an "_expansion_limit" suffix so a relaxation cap doesn't + collide with the constraint it relaxes. The underlying builders + (_expansion_limit_lhs and _expansion_limit_rhs) take any (element, + component) and (element, cap) rows — they aren't specific to paths or + relaxations. I/O Example: + options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 800 BigLine + CQ-NQ reverse 1000 BigLine + links: isp_name name p_nom_extendable CQ-NQ CQ-NQ_existing False @@ -731,10 +958,6 @@ def _create_expansion_limit_constraints( name isp_name SWQLD1_exp_2030 SWQLD1 - path_caps: - expansion_id allowed_expansion - CQ-NQ 1000 - relaxation_caps: expansion_id allowed_expansion SWQLD1 500 @@ -745,11 +968,12 @@ def _create_expansion_limit_constraints( CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN SWQLD1_expansion_limit SWQLD1_exp_2030 Generator p_nom 1.0 NaN - and rhs: + and rhs (CQ-NQ capped at max(forward, reverse)): constraint_id rhs constraint_type investment_period timeslice CQ-NQ_expansion_limit 1000 <= NaN NaN SWQLD1_expansion_limit 500 <= NaN NaN """ + path_caps = _resolve_path_expansion_caps(options, links) lhs = pd.concat( [ _expansion_limit_lhs(links[links["p_nom_extendable"]], "Link"), @@ -809,8 +1033,9 @@ def _expansion_limit_rhs(caps: pd.DataFrame) -> pd.DataFrame: def _finalise_lhs_and_rhs( lhs: pd.DataFrame, rhs: pd.DataFrame ) -> tuple[pd.DataFrame, pd.DataFrame]: - """Renames constraint_id to constraint_name, rejects duplicate constraint - names, and sets the final PyPSA friendly column orders. + """Renames constraint_id to constraint_name, raises on duplicate + (constraint, period, timeslice) RHS rows, and sets the final PyPSA + friendly column orders. I/O Example: lhs: constraint_id=SWQLD1, ... rhs: constraint_id=SWQLD1, ... diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 9a03c1a6..bb3aa676 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -2,7 +2,7 @@ import pytest from ispypsa.translator.constraints import ( - _translate_custom_constraints_from_network_tables, + _translate_custom_constraints, ) from ispypsa.translator.helpers import _annuitised_investment_costs @@ -55,11 +55,32 @@ def _links(csv_str_to_df) -> pd.DataFrame: """) +def _generators(csv_str_to_df) -> pd.DataFrame: + """Existing units carry their own name as isp_name. LATEGEN backs the + date_from-after-all-periods test.""" + return csv_str_to_df(""" + isp_name, name + KINGASF1, KINGASF1 + LATEGEN, LATEGEN + """) + + +def _storage(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + isp_name, name + Q8 Battery - 2h, Q8 Battery - 2h + """) + + def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): ispypsa_tables = _constraint_tables(csv_str_to_df) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_rhs = csv_str_to_df(""" @@ -82,8 +103,12 @@ def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): ispypsa_tables = _constraint_tables(csv_str_to_df) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_lhs = csv_str_to_df(""" @@ -116,8 +141,12 @@ def test_translate_custom_constraints_relaxation_generators( ): ispypsa_tables = _constraint_tables(csv_str_to_df) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_generators = csv_str_to_df(f""" @@ -140,8 +169,12 @@ def test_translate_custom_constraints_rez_expansion_disabled( ispypsa_tables = _constraint_tables(csv_str_to_df) sample_model_config.network.rez_transmission_expansion = False - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_generators = csv_str_to_df(""" @@ -185,8 +218,12 @@ def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config) SWQLD1, qld_peak_demand, 2500, 2026-12-01T00:00:00 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) rhs = result["custom_constraints_rhs"] @@ -213,8 +250,12 @@ def test_date_from_after_all_periods_contributes_nothing( SWQLD1, generator_output, LATEGEN, 0.5, 2040-01-01T00:00:00 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_lhs = csv_str_to_df(""" @@ -250,8 +291,12 @@ def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_co NSW-QLD, reverse, 900, NSW-QLD Option 1 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_rhs = csv_str_to_df(""" @@ -285,8 +330,12 @@ def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( SWQLD1, generator_output, KINGASF1, 0.14, """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_lhs = csv_str_to_df(""" @@ -308,9 +357,9 @@ def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( ) -def test_link_terms_not_in_model_dropped_and_logged( - csv_str_to_df, sample_model_config, caplog -): +def test_link_term_not_in_model_raises(csv_str_to_df, sample_model_config): + """TAS-SEV has no links in the model, so the constraint can't be applied + as written — dropping the term would silently weaken it, so raise.""" ispypsa_tables = _constraint_tables(csv_str_to_df) ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" constraint_id, term_type, variable_name, coefficient, date_from @@ -318,18 +367,100 @@ def test_link_terms_not_in_model_dropped_and_logged( SWQLD1, generator_output, KINGASF1, 0.14, """) - with caplog.at_level("INFO"): - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + with pytest.raises(ValueError) as excinfo: + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) assert ( - "Custom constraint link_flow terms dropped (paths not in model): ['TAS-SEV']" - ) in caplog.text + "Custom constraint LHS terms reference components not in the model: " + "[('SWQLD1', 'TAS-SEV')]" + ) in str(excinfo.value) + + +def test_generator_and_storage_terms_not_in_model_raise( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, UNKNOWNGEN, 0.14, + SWQLD1, storage_output, Big Battery, 0.43, + """) + + with pytest.raises(ValueError) as excinfo: + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, + ) + + assert ( + "Custom constraint LHS terms reference components not in the model: " + "[('SWQLD1', 'Big Battery'), ('SWQLD1', 'UNKNOWNGEN')]" + ) in str(excinfo.value) + + +def test_load_terms_raise(csv_str_to_df, sample_model_config): + """Load variables aren't implemented in pypsa_build, so a constraint with + a load term can't be applied as written.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, load, SQ, 0.3, + """) + + with pytest.raises(ValueError) as excinfo: + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, + ) + + assert ( + "Custom constraint load terms are not supported; constraints with " + "load terms: ['SWQLD1']" + ) in str(excinfo.value) + + +def test_new_entrant_terms_expand_to_per_build_year_components( + csv_str_to_df, sample_model_config +): + """A term naming a new entrant generator's isp_name covers each of its + per-build-year components, mirroring link_flow expansion.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, N2 Solar, 0.5, + """) + generators = csv_str_to_df(""" + isp_name, name + N2 Solar, N2 Solar_2026 + N2 Solar, N2 Solar_2028 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + generators, + _storage(csv_str_to_df), + sample_model_config, + ) + expected_lhs = csv_str_to_df(""" constraint_name, investment_period, variable_name, component, attribute, coefficient - SWQLD1, 2026, KINGASF1, Generator, p, 0.14 - SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, N2 Solar_2026, Generator, p, 0.5 + SWQLD1, 2026, N2 Solar_2028, Generator, p, 0.5 + SWQLD1, 2028, N2 Solar_2026, Generator, p, 0.5 + SWQLD1, 2028, N2 Solar_2028, Generator, p, 0.5 SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 @@ -361,8 +492,12 @@ def test_constraint_with_no_lhs_terms_dropped_and_logged( """) with caplog.at_level("INFO"): - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) assert ( @@ -439,8 +574,12 @@ def test_rhs_starting_mid_horizon_drops_lhs_for_earlier_periods_and_logs( """) with caplog.at_level("INFO"): - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) assert ( @@ -468,8 +607,12 @@ def test_lhs_starting_mid_horizon_drops_rhs_for_earlier_periods_and_logs( """) with caplog.at_level("INFO"): - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) assert ( @@ -480,19 +623,22 @@ def test_lhs_starting_mid_horizon_drops_rhs_for_earlier_periods_and_logs( _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) -def test_no_drop_logs_when_every_term_and_period_is_in_the_model( +def test_no_one_sided_drop_logs_when_both_sides_cover_both_periods( csv_str_to_df, sample_model_config, caplog ): - """The base fixture's link is in the model and both sides cover both - periods, so none of the module's INFO drop lines fire.""" + """Both sides of the base fixture's constraint cover both periods, so + neither one-sided INFO drop line fires.""" ispypsa_tables = _constraint_tables(csv_str_to_df) with caplog.at_level("INFO"): - _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) - assert "link_flow terms dropped" not in caplog.text assert "RHS rows dropped" not in caplog.text assert "LHS terms dropped" not in caplog.text @@ -522,8 +668,12 @@ def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): NSW-QLD, reverse, 900, Option 1 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + pd.DataFrame(columns=["isp_name", "name"]), + pd.DataFrame(columns=["isp_name", "name"]), + sample_model_config, ) expected_lhs = csv_str_to_df(""" @@ -556,8 +706,12 @@ def test_path_expansion_limit_is_max_of_forward_and_reverse( SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) rhs = result["custom_constraints_rhs"] @@ -607,8 +761,12 @@ def test_wildcard_relaxation_option_and_cost_apply_to_every_constraint( , , 100000 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_generators = csv_str_to_df(f""" @@ -659,8 +817,12 @@ def test_relaxation_option_for_constraint_not_in_model_is_dropped( NQ1, , 100000 """) - result = _translate_custom_constraints_from_network_tables( - ispypsa_tables, _links(csv_str_to_df), sample_model_config + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + sample_model_config, ) expected_generators = csv_str_to_df(f""" From 1589397e19b7c3a3a68d4258bad1d8b0c1874e17 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Tue, 25 Aug 2026 16:40:12 +1000 Subject: [PATCH 15/20] Support load terms as data terms on the demand nodes' Load components The blanket raise on load terms made the real constraint set untranslatable -- the baseline CNSW-SNW South GPG constraint carries date_from-varying load terms. Demand is exogenous (p_set on the Load components pypsa_build attaches per demand node), so a load term is a data term, not a variable term: it now resolves through the standard isp_name -> name mapping to the "load_" component with attribute p_set, ready for pypsa_build to fold coefficient x demand into the constraint. pypsa_build owes that implementation and should raise, not skip, until it lands. Validation is against a new demand_nodes input rather than all buses, because REZ buses carry no demand and which buses do follows the regional granularity -- a load term for a sub-region the granularity aggregates away raises through the same not-in-model check as link terms. Also labels the missing-component error's tuple structure as (constraint_id, variable_name). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QXjsJKX7ANSJxVyJbszLYW --- src/ispypsa/translator/constraints.py | 70 +++++++++++------ src/ispypsa/translator/mappings.py | 9 ++- tests/test_translator/test_constraints.py | 91 ++++++++++++++++++++--- 3 files changed, 133 insertions(+), 37 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index fa864a2e..89c5cf45 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -82,6 +82,7 @@ def _translate_custom_constraints( links: pd.DataFrame, generators: pd.DataFrame, storage: pd.DataFrame, + demand_nodes: pd.DataFrame, config: ModelConfig, ) -> dict[str, pd.DataFrame]: """Translates the custom-constraint tables and appends the endogenous @@ -138,6 +139,7 @@ def _translate_custom_constraints( SWQLD1 generator_output KINGASF1 0.14 SWQLD1 generator_output N2 Solar 0.5 SWQLD1 storage_output Q8 Battery - 2h 0.43 + SWQLD1 load SQ -0.33 ispypsa_tables["network_expansion_options"]: expansion_id expansion_type allowed_expansion expansion_option @@ -170,6 +172,10 @@ def _translate_custom_constraints( SQ BESS SQ BESS_2026 SQ BESS SQ BESS_2028 + demand_nodes: + name + SQ + returns["custom_constraints_rhs"]: constraint_name investment_period timeslice rhs constraint_type SWQLD1 2026 qld_peak_demand 3000 <= @@ -185,6 +191,7 @@ def _translate_custom_constraints( SWQLD1 2026 N2 Solar_2026 Generator p 0.5 SWQLD1 2026 N2 Solar_2028 Generator p 0.5 SWQLD1 2026 Q8 Battery - 2h Storage p 0.43 + SWQLD1 2026 load_SQ Load p_set -0.33 SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 SWQLD1 2028 SWQLD1_exp_2028 Generator p_nom -1.0 # relaxation accumulates NSW-QLD_expansion_limit NSW-QLD_exp_2026 Link p_nom 1.0 @@ -197,7 +204,7 @@ def _translate_custom_constraints( SWQLD1_exp_2028 SWQLD1 bus_for_custom_constraint_gens 0.0 2028 annuitise(80000) """ lhs, rhs = _translate_constraint_tables( - ispypsa_tables, links, generators, storage, config + ispypsa_tables, links, generators, storage, demand_nodes, config ) relaxation_generators, relaxation_lhs, relaxation_caps = ( _create_constraint_relaxations( @@ -230,6 +237,7 @@ def _translate_constraint_tables( links: pd.DataFrame, generators: pd.DataFrame, storage: pd.DataFrame, + demand_nodes: pd.DataFrame, config: ModelConfig, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Translates the user-authored custom-constraint tables into one LHS term @@ -246,14 +254,17 @@ def _translate_constraint_tables( combinations. - every LHS term must resolve to a component in the model. A term naming a component the configured model doesn't contain (e.g. a link_flow term when - regional_granularity is single_region builds no links) raises, since applying - the constraint without the term would silently weaken it. load terms aren't - implemented in pypsa_build and also raise. + regional_granularity is single_region builds no links, or a load term when + the granularity doesn't make its sub-region a demand node) raises, since + applying the constraint without the term would silently alter it. - LHS terms are expanded from their input IDs into one term per matching - model component (the links, generators and storage tables' isp_name to name - mapping): a link_flow term covers its path's existing link and each expansion - link, and a term on a new entrant generator or storage unit covers each of - its per-build-year components. + model component (the links, generators, storage and demand_nodes tables' + isp_name to name mapping): a link_flow term covers its path's existing link + and each expansion link, a term on a new entrant generator or storage unit + covers each of its per-build-year components, and a load term maps to + the "load_" Load component at its demand node — a data term whose + p_set attribute pypsa_build resolves from the demand trace, not an + optimisation variable. - LHS and RHS rows are dropped in investment periods where the constraint does not have both LHS terms and an RHS value. This happens when date_from coverage differs between the two sides (including a side whose earliest date_from falls @@ -290,6 +301,10 @@ def _translate_constraint_tables( SQ BESS SQ BESS_2026 SQ BESS SQ BESS_2028 + demand_nodes: + name + SQ + returns lhs (2028 rows mirror 2026): constraint_id investment_period variable_name component attribute coefficient SWQLD1 2026 NSW-QLD_existing Link p 0.84 @@ -317,8 +332,7 @@ def _translate_constraint_tables( period_starts, ) lhs = _add_component_and_attribute(lhs) - model_components = _model_component_names(links, generators, storage) - _raise_on_load_terms(lhs) + model_components = _model_component_names(links, generators, storage, demand_nodes) _raise_on_terms_not_in_model(lhs, model_components) lhs = _expand_terms_to_model_components(lhs, model_components) return _drop_one_sided_constraint_periods(lhs, rhs) @@ -440,12 +454,22 @@ def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: def _model_component_names( - links: pd.DataFrame, generators: pd.DataFrame, storage: pd.DataFrame + links: pd.DataFrame, + generators: pd.DataFrame, + storage: pd.DataFrame, + demand_nodes: pd.DataFrame, ) -> pd.DataFrame: """One row per model component an LHS term can resolve to: the component type, the ID the constraint tables refer to it by (isp_name) and the model component's name. + Demand nodes — the buses with demand attached, not all buses — appear as + Load rows: a load term resolves to the Load component pypsa_build attaches + to its node's demand trace, named "load_" (see + ispypsa.pypsa_build.buses). Which buses carry demand follows the regional + granularity (sub-region buses, region buses, or the single NEM bus); REZ + buses never do (see ispypsa.translator.buses). + I/O Example: links: isp_name name @@ -464,6 +488,10 @@ def _model_component_names( SQ BESS SQ BESS_2030 SQ BESS SQ BESS_2040 + demand_nodes: + name + SQ + returns: isp_name name component NSW-QLD NSW-QLD_existing Link @@ -474,26 +502,19 @@ def _model_component_names( Q8 Battery - 2h Q8 Battery - 2h Storage SQ BESS SQ BESS_2030 Storage SQ BESS SQ BESS_2040 Storage + SQ load_SQ Load """ frames = [ links.loc[:, ["isp_name", "name"]].assign(component="Link"), generators.loc[:, ["isp_name", "name"]].assign(component="Generator"), storage.loc[:, ["isp_name", "name"]].assign(component="Storage"), + demand_nodes.loc[:, ["name"]] + .rename(columns={"name": "isp_name"}) + .assign(name="load_" + demand_nodes["name"], component="Load"), ] return _concat_non_empty(frames, ["isp_name", "name", "component"]) -def _raise_on_load_terms(lhs: pd.DataFrame) -> None: - """Raises for load terms — load variables aren't implemented in - pypsa_build, so a constraint carrying one can't be applied as specified.""" - load_terms = lhs[lhs["component"] == "Load"] - if not load_terms.empty: - raise ValueError( - "Custom constraint load terms are not supported; constraints " - f"with load terms: {sorted(set(load_terms['constraint_id']))}" - ) - - def _raise_on_terms_not_in_model( lhs: pd.DataFrame, model_components: pd.DataFrame ) -> None: @@ -509,7 +530,8 @@ def _raise_on_terms_not_in_model( SWQLD1 KINGASF1 Generator model_components with only ("NSW-QLD", Link) raises: - "... components not in the model: [('SWQLD1', 'KINGASF1')]" + "... components not in the model, as (constraint_id, + variable_name): [('SWQLD1', 'KINGASF1')]" """ ids = model_components.loc[:, ["component", "isp_name"]].drop_duplicates() matched = lhs.merge( @@ -523,7 +545,7 @@ def _raise_on_terms_not_in_model( pairs = sorted(set(zip(missing["constraint_id"], missing["variable_name"]))) raise ValueError( f"Custom constraint LHS terms reference components not in the " - f"model: {pairs}" + f"model, as (constraint_id, variable_name): {pairs}" ) diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index 615203b5..3ff2deb6 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -153,8 +153,9 @@ "generator_output": "Generator", "load_consumption": "Load", # "load" is the new-format vocabulary for "load_consumption" (PLEXOS Node - # Load Coefficient terms). Load variables aren't implemented in - # pypsa_build, which skips these terms with a log line. + # Load Coefficient terms). A load term is a data term — the demand (p_set) + # at the term's demand node, scaled by the coefficient — rather than a + # term on an optimisation variable. "load": "Load", "storage_output": "Storage", } @@ -163,8 +164,8 @@ "link_flow": "p", "generator_capacity": "p_nom", "generator_output": "p", - "load_consumption": "p", - "load": "p", + "load_consumption": "p_set", + "load": "p_set", "storage_output": "p", } diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index bb3aa676..62321abe 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -72,6 +72,15 @@ def _storage(csv_str_to_df) -> pd.DataFrame: """) +def _demand_nodes(csv_str_to_df) -> pd.DataFrame: + """The buses with demand attached — at the fixture's sub_regions + granularity, the sub-region buses.""" + return csv_str_to_df(""" + name + SQ + """) + + def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): ispypsa_tables = _constraint_tables(csv_str_to_df) @@ -80,6 +89,7 @@ def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -108,6 +118,7 @@ def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -146,6 +157,7 @@ def test_translate_custom_constraints_relaxation_generators( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -174,6 +186,7 @@ def test_translate_custom_constraints_rez_expansion_disabled( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -223,6 +236,7 @@ def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config) _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -255,6 +269,7 @@ def test_date_from_after_all_periods_contributes_nothing( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -296,6 +311,7 @@ def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_co _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -335,6 +351,7 @@ def test_relaxation_on_greater_equal_constraint_adds_capacity_to_lhs( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -373,12 +390,13 @@ def test_link_term_not_in_model_raises(csv_str_to_df, sample_model_config): _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) assert ( - "Custom constraint LHS terms reference components not in the model: " - "[('SWQLD1', 'TAS-SEV')]" + "Custom constraint LHS terms reference components not in the model, " + "as (constraint_id, variable_name): [('SWQLD1', 'TAS-SEV')]" ) in str(excinfo.value) @@ -398,22 +416,67 @@ def test_generator_and_storage_terms_not_in_model_raise( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) assert ( - "Custom constraint LHS terms reference components not in the model: " + "Custom constraint LHS terms reference components not in the model, " + "as (constraint_id, variable_name): " "[('SWQLD1', 'Big Battery'), ('SWQLD1', 'UNKNOWNGEN')]" ) in str(excinfo.value) -def test_load_terms_raise(csv_str_to_df, sample_model_config): - """Load variables aren't implemented in pypsa_build, so a constraint with - a load term can't be applied as written.""" +def test_load_term_resolves_to_the_demand_nodes_load_component( + csv_str_to_df, sample_model_config +): + """A load term is a data term on the sub-region's demand: it maps to the + load_ Load component at its demand node, whose p_set pypsa_build + resolves from the demand trace.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, load, SQ, -0.33, + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, load_SQ, Load, p_set, -0.33 + SWQLD1, 2028, load_SQ, Load, p_set, -0.33 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_load_term_for_sub_region_without_demand_node_raises( + csv_str_to_df, sample_model_config +): + """A sub-region that isn't a demand node (it's outside the model, or the + granularity aggregates it away) has no demand for a load term to + reference.""" ispypsa_tables = _constraint_tables(csv_str_to_df) ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" constraint_id, term_type, variable_name, coefficient, date_from - SWQLD1, load, SQ, 0.3, + SWQLD1, load, CNSW, -0.33, """) with pytest.raises(ValueError) as excinfo: @@ -422,12 +485,13 @@ def test_load_terms_raise(csv_str_to_df, sample_model_config): _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) assert ( - "Custom constraint load terms are not supported; constraints with " - "load terms: ['SWQLD1']" + "Custom constraint LHS terms reference components not in the model, " + "as (constraint_id, variable_name): [('SWQLD1', 'CNSW')]" ) in str(excinfo.value) @@ -452,6 +516,7 @@ def test_new_entrant_terms_expand_to_per_build_year_components( _links(csv_str_to_df), generators, _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -497,6 +562,7 @@ def test_constraint_with_no_lhs_terms_dropped_and_logged( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -579,6 +645,7 @@ def test_rhs_starting_mid_horizon_drops_lhs_for_earlier_periods_and_logs( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -612,6 +679,7 @@ def test_lhs_starting_mid_horizon_drops_rhs_for_earlier_periods_and_logs( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -636,6 +704,7 @@ def test_no_one_sided_drop_logs_when_both_sides_cover_both_periods( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -673,6 +742,7 @@ def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): _links(csv_str_to_df), pd.DataFrame(columns=["isp_name", "name"]), pd.DataFrame(columns=["isp_name", "name"]), + pd.DataFrame(columns=["name"]), sample_model_config, ) @@ -711,6 +781,7 @@ def test_path_expansion_limit_is_max_of_forward_and_reverse( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -766,6 +837,7 @@ def test_wildcard_relaxation_option_and_cost_apply_to_every_constraint( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) @@ -822,6 +894,7 @@ def test_relaxation_option_for_constraint_not_in_model_is_dropped( _links(csv_str_to_df), _generators(csv_str_to_df), _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), sample_model_config, ) From 6f027f400ae8eeda100eea2e7ec94b29fb360b92 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 26 Aug 2026 16:36:38 +1000 Subject: [PATCH 16/20] Recast the LHS name mapping as constraint_variable_name to pypsa_model_name The mapping's inherited isp_name/name columns said where the data came from, not what the mapping means, so the constraints module now names both sides for their role. The custom_constraints_lhs schema also sets out the convention those names imply: a variable_name refers to a whole element - a new entrant across all its build years, a path across its existing and expansion links - never a per-build-year component. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0182yKiTKCwGoVPs5hToavKn --- src/ispypsa/translator/constraints.py | 122 +++++++++++------- .../schemas/custom_constraints_lhs.yaml | 11 +- 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 89c5cf45..5696aed4 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -258,8 +258,9 @@ def _translate_constraint_tables( the granularity doesn't make its sub-region a demand node) raises, since applying the constraint without the term would silently alter it. - LHS terms are expanded from their input IDs into one term per matching - model component (the links, generators, storage and demand_nodes tables' - isp_name to name mapping): a link_flow term covers its path's existing link + model component (the constraint-variable-name to PyPSA-model-name mapping + built from the links, generators, storage and demand_nodes tables): a + link_flow term covers its path's existing link and each expansion link, a term on a new entrant generator or storage unit covers each of its per-build-year components, and a load term maps to the "load_" Load component at its demand node — a data term whose @@ -332,9 +333,11 @@ def _translate_constraint_tables( period_starts, ) lhs = _add_component_and_attribute(lhs) - model_components = _model_component_names(links, generators, storage, demand_nodes) - _raise_on_terms_not_in_model(lhs, model_components) - lhs = _expand_terms_to_model_components(lhs, model_components) + variable_name_mapping = _map_constraint_variables_to_pypsa_names( + links, generators, storage, demand_nodes + ) + _raise_on_terms_not_in_model(lhs, variable_name_mapping) + lhs = _expand_terms_to_model_components(lhs, variable_name_mapping) return _drop_one_sided_constraint_periods(lhs, rhs) @@ -453,15 +456,25 @@ def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: ) -def _model_component_names( +def _map_constraint_variables_to_pypsa_names( links: pd.DataFrame, generators: pd.DataFrame, storage: pd.DataFrame, demand_nodes: pd.DataFrame, ) -> pd.DataFrame: - """One row per model component an LHS term can resolve to: the component - type, the ID the constraint tables refer to it by (isp_name) and the - model component's name. + """Builds the mapping from the names the custom-constraint tables refer to + model elements by (constraint_variable_name) to the names of the PyPSA + components built for them (pypsa_model_name), one row per component. + + The links, generators and storage frames already carry both sides as their + isp_name (the element's un-suffixed ISP-level ID) and name columns; this + function relabels them into the constraints module's vocabulary. A + constraint variable name identifies a whole element, so one name can map + to several components: a path's name (e.g. NSW-QLD, with no _existing or + _exp_ suffix) covers the path's existing link and each of its + expansion links, and a new entrant generator or storage unit's ID covers + each of its per-build-year components. An existing unit's name maps to + itself. Demand nodes — the buses with demand attached, not all buses — appear as Load rows: a load term resolves to the Load component pypsa_build attaches @@ -493,30 +506,43 @@ def _model_component_names( SQ returns: - isp_name name component - NSW-QLD NSW-QLD_existing Link - NSW-QLD NSW-QLD_exp_2030 Link - KINGASF1 KINGASF1 Generator - N2 Solar N2 Solar_2030 Generator - N2 Solar N2 Solar_2040 Generator - Q8 Battery - 2h Q8 Battery - 2h Storage - SQ BESS SQ BESS_2030 Storage - SQ BESS SQ BESS_2040 Storage - SQ load_SQ Load + constraint_variable_name pypsa_model_name component + NSW-QLD NSW-QLD_existing Link + NSW-QLD NSW-QLD_exp_2030 Link + KINGASF1 KINGASF1 Generator + N2 Solar N2 Solar_2030 Generator + N2 Solar N2 Solar_2040 Generator + Q8 Battery - 2h Q8 Battery - 2h Storage + SQ BESS SQ BESS_2030 Storage + SQ BESS SQ BESS_2040 Storage + SQ load_SQ Load """ + to_mapping = {"isp_name": "constraint_variable_name", "name": "pypsa_model_name"} frames = [ - links.loc[:, ["isp_name", "name"]].assign(component="Link"), - generators.loc[:, ["isp_name", "name"]].assign(component="Generator"), - storage.loc[:, ["isp_name", "name"]].assign(component="Storage"), - demand_nodes.loc[:, ["name"]] - .rename(columns={"name": "isp_name"}) - .assign(name="load_" + demand_nodes["name"], component="Load"), + links.loc[:, ["isp_name", "name"]] + .rename(columns=to_mapping) + .assign(component="Link"), + generators.loc[:, ["isp_name", "name"]] + .rename(columns=to_mapping) + .assign(component="Generator"), + storage.loc[:, ["isp_name", "name"]] + .rename(columns=to_mapping) + .assign(component="Storage"), + pd.DataFrame( + { + "constraint_variable_name": demand_nodes["name"], + "pypsa_model_name": "load_" + demand_nodes["name"], + "component": "Load", + } + ), ] - return _concat_non_empty(frames, ["isp_name", "name", "component"]) + return _concat_non_empty( + frames, ["constraint_variable_name", "pypsa_model_name", "component"] + ) def _raise_on_terms_not_in_model( - lhs: pd.DataFrame, model_components: pd.DataFrame + lhs: pd.DataFrame, variable_name_mapping: pd.DataFrame ) -> None: """Raises when a term references a component with no match in the model — e.g. a link_flow term when regional_granularity is single_region builds no @@ -529,18 +555,24 @@ def _raise_on_terms_not_in_model( SWQLD1 NSW-QLD Link SWQLD1 KINGASF1 Generator - model_components with only ("NSW-QLD", Link) raises: + variable_name_mapping: + constraint_variable_name pypsa_model_name component + NSW-QLD NSW-QLD_existing Link + + raises: "... components not in the model, as (constraint_id, variable_name): [('SWQLD1', 'KINGASF1')]" """ - ids = model_components.loc[:, ["component", "isp_name"]].drop_duplicates() + ids = variable_name_mapping.loc[ + :, ["component", "constraint_variable_name"] + ].drop_duplicates() matched = lhs.merge( ids, how="left", left_on=["component", "variable_name"], - right_on=["component", "isp_name"], + right_on=["component", "constraint_variable_name"], ) - missing = matched[matched["isp_name"].isna()] + missing = matched[matched["constraint_variable_name"].isna()] if not missing.empty: pairs = sorted(set(zip(missing["constraint_id"], missing["variable_name"]))) raise ValueError( @@ -550,12 +582,12 @@ def _raise_on_terms_not_in_model( def _expand_terms_to_model_components( - lhs: pd.DataFrame, model_components: pd.DataFrame + lhs: pd.DataFrame, variable_name_mapping: pd.DataFrame ) -> pd.DataFrame: - """Replaces each term's input ID with the model components it covers, one - term per component: a link_flow term covers its path's existing and - expansion links, and a term on a new entrant generator or storage unit - covers each of its per-build-year components. + """Replaces each term's constraint variable name (in the variable_name column) with + the PyPSA model component names it covers, one term per component: a link_flow term + covers its path's existing and expansion links, and a term on a new entrant + generator or storage unit covers each of its per-build-year components. I/O Example: lhs: @@ -563,11 +595,11 @@ def _expand_terms_to_model_components( SWQLD1 NSW-QLD Link 0.84 SWQLD1 KINGASF1 Generator 0.14 - model_components: - isp_name name component - NSW-QLD NSW-QLD_existing Link - NSW-QLD NSW-QLD_exp_2030 Link - KINGASF1 KINGASF1 Generator + variable_name_mapping: + constraint_variable_name pypsa_model_name component + NSW-QLD NSW-QLD_existing Link + NSW-QLD NSW-QLD_exp_2030 Link + KINGASF1 KINGASF1 Generator returns: constraint_id variable_name component coefficient @@ -576,12 +608,12 @@ def _expand_terms_to_model_components( SWQLD1 KINGASF1 Generator 0.14 """ expanded = lhs.merge( - model_components, + variable_name_mapping, left_on=["component", "variable_name"], - right_on=["component", "isp_name"], + right_on=["component", "constraint_variable_name"], ) - expanded = expanded.drop(columns=["variable_name", "isp_name"]) - return expanded.rename(columns={"name": "variable_name"}) + expanded = expanded.drop(columns=["variable_name", "constraint_variable_name"]) + return expanded.rename(columns={"pypsa_model_name": "variable_name"}) def _drop_one_sided_constraint_periods( diff --git a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml index ea800976..bfac2f34 100644 --- a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml +++ b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml @@ -55,10 +55,19 @@ columns: type: string required: true description: > - Name of the component the term's variable belongs to: an IASR ID for + Name of the model element the term's variable belongs to: an IASR ID for generator_output, generator_capacity and storage_output terms, a path_id for link_flow terms, a sub-region geo_id for load terms (see custom_validation for the table each resolves against). + + Names refer to whole elements, never to the per-build-year components + the translator builds from them, and one term covers every component of + its element. A new entrant generator or storage unit is referred to by + its name in generators_new_entrant / storage_new_entrant (its location + and technology, e.g. "N2 Solar"), and the term applies to the unit in + every build year. A path is referred to by its path_id (e.g. "NSW-QLD"), + and the term applies to the path's existing link and each of its + expansion links. coefficient: type: float required: true From f18ba2a5dc17b906daab819e888f303073260a98 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 26 Aug 2026 16:50:22 +1000 Subject: [PATCH 17/20] Lay every I/O example out as a full table, and polish constraint docstrings Condensed column=value examples made the reader reconstruct the table shape in their head, so the I/O Example convention now requires full tables (with abridged column/row sets for otherwise large ones) and the constraints module's remaining condensed examples follow it. The expansion example also gains a new entrant term, showing the per-build-year fan-out next to the link and existing-unit cases. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0182yKiTKCwGoVPs5hToavKn --- CLAUDE.md | 7 ++-- src/ispypsa/translator/constraints.py | 50 ++++++++++++++++++++------- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 533dba2f..3dbe051a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,8 +72,11 @@ Conventions: - Use a plain CSV-like table format for DataFrame inputs and outputs — no need to wrap in runnable `csv_str_to_df` calls, since this is illustrative, not a doctest. -- Abbreviate long column names when they would otherwise overflow the line; point at - the relevant constants for the real names. +- Always lay DataFrame examples out as full tables — a header row plus example rows, + each table in its own block — even when a single row would do. Never condense a + table into `column=value` prose. Abbreviated column sets, row sets and column + names are fine when the full table would otherwise be large; point at the + relevant constants for the real names. - Cover representative edge cases in the same example, with trailing `# comment` notes on the rows that demonstrate each case. - For trivial utility functions, one-line input → output cases are enough. diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index 5696aed4..cbdf16fe 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -240,7 +240,7 @@ def _translate_constraint_tables( demand_nodes: pd.DataFrame, config: ModelConfig, ) -> tuple[pd.DataFrame, pd.DataFrame]: - """Translates the user-authored custom-constraint tables into one LHS term + """Translates the ISPyPSA custom-constraint tables into one LHS term and one RHS row per constraint, investment period and (RHS only) timeslice, still keyed by constraint_id. @@ -594,18 +594,23 @@ def _expand_terms_to_model_components( constraint_id variable_name component coefficient SWQLD1 NSW-QLD Link 0.84 SWQLD1 KINGASF1 Generator 0.14 + SWQLD1 N2 Solar Generator 0.5 variable_name_mapping: constraint_variable_name pypsa_model_name component NSW-QLD NSW-QLD_existing Link NSW-QLD NSW-QLD_exp_2030 Link KINGASF1 KINGASF1 Generator + N2 Solar N2 Solar_2030 Generator + N2 Solar N2 Solar_2040 Generator returns: constraint_id variable_name component coefficient SWQLD1 NSW-QLD_existing Link 0.84 SWQLD1 NSW-QLD_exp_2030 Link 0.84 - SWQLD1 KINGASF1 Generator 0.14 + SWQLD1 KINGASF1 Generator 0.14 # existing unit: unchanged + SWQLD1 N2 Solar_2030 Generator 0.5 # new entrant: one term + SWQLD1 N2 Solar_2040 Generator 0.5 # per build year """ expanded = lhs.merge( variable_name_mapping, @@ -989,7 +994,7 @@ def _create_expansion_limit_constraints( relaxation's cap is its option's allowed_expansion, passed in as relaxation_caps. Each cap becomes an RHS row with constraint_type "<=" and no investment_period or timeslice — it applies to the p_nom variables - globally — and each component contributes a coefficient-1.0 p_nom LHS + globally — and each component contributes a coefficient 1.0 p_nom LHS term. Names get an "_expansion_limit" suffix so a relaxation cap doesn't collide with the constraint it relaxes. The underlying builders (_expansion_limit_lhs and _expansion_limit_rhs) take any (element, @@ -1046,10 +1051,16 @@ def _expansion_limit_lhs(components: pd.DataFrame, component_type: str) -> pd.Da """One LHS term per expandable component, summing p_nom across the investment periods of its parent element. - I/O Example: - components: name=CQ-NQ_exp_2030, isp_name=CQ-NQ; component_type="Link" - -> constraint_id=CQ-NQ, variable_name=CQ-NQ_exp_2030, component=Link, - attribute=p_nom, coefficient=1.0, investment_period=NaN + I/O Example (component_type="Link"): + components: + isp_name name + CQ-NQ CQ-NQ_exp_2030 + CQ-NQ CQ-NQ_exp_2040 + + returns: + constraint_id variable_name component attribute coefficient investment_period + CQ-NQ CQ-NQ_exp_2030 Link p_nom 1.0 NaN + CQ-NQ CQ-NQ_exp_2040 Link p_nom 1.0 NaN """ lhs = components.loc[:, ["isp_name", "name"]].copy() lhs = lhs.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) @@ -1091,10 +1102,22 @@ def _finalise_lhs_and_rhs( (constraint, period, timeslice) RHS rows, and sets the final PyPSA friendly column orders. - I/O Example: - lhs: constraint_id=SWQLD1, ... rhs: constraint_id=SWQLD1, ... - -> lhs.columns == _LHS_COLUMNS, rhs.columns == _RHS_COLUMNS, both - keyed by constraint_name=SWQLD1 + I/O Example (non-key columns abridged — see _LHS_COLUMNS and _RHS_COLUMNS): + lhs: + constraint_id variable_name coefficient + SWQLD1 NSW-QLD_existing 0.84 + + rhs: + constraint_id timeslice rhs + SWQLD1 qld_peak_demand 3000 + + returns lhs: + constraint_name variable_name coefficient + SWQLD1 NSW-QLD_existing 0.84 + + returns rhs: + constraint_name timeslice rhs + SWQLD1 qld_peak_demand 3000 """ lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) @@ -1106,8 +1129,9 @@ def _finalise_lhs_and_rhs( def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: - """Raise on duplicate (constraint, period, timeslice) RHS rows — pypsa_build - would create two constraints with the same name.""" + """Raise on duplicate (constraint, period, timeslice) RHS rows. A final check to make + sure the constraint translation process hasn't created constraints with overlapping + names.""" keys = ["constraint_name", "investment_period", "timeslice"] duplicates = rhs[rhs.duplicated(subset=keys, keep=False)] if not duplicates.empty: From 69f7238bb5401b30790c7d265d7d69ad52711d92 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 26 Aug 2026 17:02:54 +1000 Subject: [PATCH 18/20] Cover the constraint translator's guard raises and less-travelled branches Coverage showed four untested paths: the all-empty assembly branch, calendar year period starts, the unmapped-term_type drift guard, and the duplicate RHS name check. The last needed the one realistic collision - a relaxable constraint sharing its ID with an expandable path, giving both the same expansion-limit cap row - which the new test pins down. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0182yKiTKCwGoVPs5hToavKn --- tests/test_translator/test_constraints.py | 167 ++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 62321abe..9ede61a2 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -254,6 +254,40 @@ def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config) ) +def test_calendar_year_periods_start_in_january(csv_str_to_df, sample_model_config): + """Under calendar years the 2026 period starts 2026-01-01, so a value + dated 2025-12-01 is already active in the first period — under fy it + would miss the 2025-07-01 period start and the 2026 rows would drop.""" + sample_model_config.temporal.year_type = "calendar" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 2500, 2025-12-01T00:00:00 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + rhs = result["custom_constraints_rhs"] + rhs = rhs[rhs["constraint_name"] == "SWQLD1"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 2500, <= + SWQLD1, 2028, qld_peak_demand, 2500, <= + """) + pd.testing.assert_frame_equal( + rhs.sort_values("investment_period").reset_index(drop=True), + expected, + check_dtype=False, + ) + + def test_date_from_after_all_periods_contributes_nothing( csv_str_to_df, sample_model_config ): @@ -427,6 +461,32 @@ def test_generator_and_storage_terms_not_in_model_raise( ) in str(excinfo.value) +def test_term_type_without_component_mapping_raises(csv_str_to_df, sample_model_config): + """A term_type with no entry in the term-type-to-component mappings (the + schema's allowed term_types and the translator's mappings drifting apart) + halts the run rather than silently dropping the term.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, storage_capacity, Q8 Battery - 2h, 0.43, + """) + + with pytest.raises(ValueError) as excinfo: + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + assert ( + "Custom constraint LHS term_types with no component mapping: " + "['storage_capacity']" + ) in str(excinfo.value) + + def test_load_term_resolves_to_the_demand_nodes_load_component( csv_str_to_df, sample_model_config ): @@ -763,6 +823,72 @@ def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): ) +def test_no_constraints_and_no_expansion_yields_header_only_tables( + csv_str_to_df, sample_model_config +): + """With no custom constraints, no expansion options and no extendable + links, no constraint block produces rows and every output is + all-columns-no-rows.""" + ispypsa_tables = { + "custom_constraints": pd.DataFrame(columns=["constraint_id", "direction"]), + "custom_constraints_lhs": pd.DataFrame( + columns=[ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + ] + ), + "custom_constraints_rhs": pd.DataFrame( + columns=["constraint_id", "timeslice", "rhs", "date_from"] + ), + "network_expansion_options": pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ), + "network_transmission_path_expansion_costs": pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ), + } + links = csv_str_to_df(""" + isp_name, name, p_nom_extendable + NSW-QLD, NSW-QLD_existing, False + """) + + result = _translate_custom_constraints( + ispypsa_tables, + links, + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + """) + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"], expected_lhs, check_dtype=False + ) + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + """) + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"], expected_rhs, check_dtype=False + ) + expected_generators = csv_str_to_df(""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], expected_generators, check_dtype=False + ) + + def test_path_expansion_limit_is_max_of_forward_and_reverse( csv_str_to_df, sample_model_config ): @@ -910,3 +1036,44 @@ def test_relaxation_option_for_constraint_not_in_model_is_dropped( check_dtype=False, rtol=1e-5, ) + + +def test_constraint_named_after_a_path_raises_on_colliding_expansion_limits( + csv_str_to_df, sample_model_config +): + """A relaxable constraint sharing its ID with an expandable path gives + both the same "_expansion_limit" RHS row — pypsa_build would build + two constraints with the same name, so the final duplicate check halts.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + NSW-QLD, <= + """) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + NSW-QLD, generator_output, KINGASF1, 0.14, + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + NSW-QLD, qld_peak_demand, 3000, + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + NSW-QLD, constraint_relaxation, 400, NSW-QLD Option 2 + """) + + with pytest.raises(ValueError) as excinfo: + _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + assert ( + "Duplicate custom constraint RHS rows for: ['NSW-QLD_expansion_limit']" + ) in str(excinfo.value) From 2254b1faf7184d62e3bdc4e8914a5b8952cfe2a8 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 27 Aug 2026 09:40:15 +1000 Subject: [PATCH 19/20] Scope constraint terms to each component's in-service periods A component's p_nom is a single horizon-wide variable, so a term on a not-yet-built or already-retired component would let capacity from outside a period alter that period's constraint (dispatch terms were harmless, since PyPSA zeroes inactive dispatch, but made the output misleading). Terms are now kept for exactly the periods PyPSA activates the component - build_year <= period < build_year + lifetime, with the retirement year derived from the lifetime the translated tables already carry - so the constraints can't disagree with the model's own activity mask. Also pins the schema-legal edge cases the suite didn't cover: blank-timeslice RHS fallbacks, values dated exactly on a period start, mid-horizon coefficient supersession, generator_capacity terms, new-entrant storage expansion, wildcard relaxation options, and populated constraints alongside empty expansion tables. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DBECPsKWwUCxESCcU2Ykxk --- src/ispypsa/translator/constraints.py | 263 ++++++++++---- tests/test_translator/test_constraints.py | 408 +++++++++++++++++++++- 2 files changed, 581 insertions(+), 90 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index cbdf16fe..cfbe2619 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -116,6 +116,18 @@ def _translate_custom_constraints( network_expansion_options.yaml and network_transmission_path_expansion_costs.yaml). + Everything time-varying is quantised forward onto the investment-period + sequence, mirroring how PyPSA's multi-investment-period optimisation treats + the components themselves. A dated LHS or RHS value takes effect at the + first period whose start falls on or after its date_from (the boundary + is inclusive), so a change landing mid-period defers to the next period + rather than reaching back into the one it landed in. A term is kept in + exactly the periods PyPSA activates its component — build_year <= period + < build_year + lifetime, compared against the period labels — so a + component built between two periods joins the constraint at the next + label, one retiring between two periods leaves at the next label, and + the constraint never counts capacity or dispatch the model doesn't have. + In the output tables, a blank investment_period means the row applies in every period. A named timeslice scopes the RHS to the snapshots inside that timeslice's windows (the timeslices table); a blank timeslice is the @@ -155,22 +167,22 @@ def _translate_custom_constraints( SWQLD1 2028 80000 links: - isp_name name p_nom_extendable - NSW-QLD NSW-QLD_existing False - NSW-QLD NSW-QLD_exp_2026 True + isp_name name p_nom_extendable build_year lifetime + NSW-QLD NSW-QLD_existing False 2025 inf + NSW-QLD NSW-QLD_exp_2026 True 2026 inf generators (isp_name = name for existing units; a new entrant's ID maps to each of its per-build-year components): - isp_name name - KINGASF1 KINGASF1 - N2 Solar N2 Solar_2026 - N2 Solar N2 Solar_2028 + isp_name name build_year lifetime + KINGASF1 KINGASF1 2025 inf # inf: never retires + N2 Solar N2 Solar_2026 2026 30 + N2 Solar N2 Solar_2028 2028 30 storage: - isp_name name - Q8 Battery - 2h Q8 Battery - 2h - SQ BESS SQ BESS_2026 - SQ BESS SQ BESS_2028 + isp_name name build_year lifetime + Q8 Battery - 2h Q8 Battery - 2h 2025 inf + SQ BESS SQ BESS_2026 2026 20 + SQ BESS SQ BESS_2028 2028 20 demand_nodes: name @@ -183,13 +195,13 @@ def _translate_custom_constraints( NSW-QLD_expansion_limit 1000 <= SWQLD1_expansion_limit 400 <= - returns["custom_constraints_lhs"] (2028 rows mirror 2026): + returns["custom_constraints_lhs"] (2028 rows mirror 2026, and also + pick up N2 Solar_2028 once it is built): constraint_name investment_period variable_name component attribute coefficient SWQLD1 2026 NSW-QLD_existing Link p 0.84 SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 SWQLD1 2026 KINGASF1 Generator p 0.14 - SWQLD1 2026 N2 Solar_2026 Generator p 0.5 - SWQLD1 2026 N2 Solar_2028 Generator p 0.5 + SWQLD1 2026 N2 Solar_2026 Generator p 0.5 # N2 Solar_2028 enters in 2028 only SWQLD1 2026 Q8 Battery - 2h Storage p 0.43 SWQLD1 2026 load_SQ Load p_set -0.33 SWQLD1 2026 SWQLD1_exp_2026 Generator p_nom -1.0 @@ -266,6 +278,16 @@ def _translate_constraint_tables( the "load_" Load component at its demand node — a data term whose p_set attribute pypsa_build resolves from the demand trace, not an optimisation variable. + - each term is then dropped in the investment periods before its + component's build year, so a per-build-year component only enters the + constraint from its build year onward. Existing components' build years + precede the horizon and load terms have no build year, so both apply in + every period. + - terms are likewise dropped from their component's retirement year + onward — build_year + lifetime, the exclusive bound at which PyPSA + deactivates the component — so a unit retiring in a period's label + year contributes nothing to that period. Components with an infinite + lifetime never retire. - LHS and RHS rows are dropped in investment periods where the constraint does not have both LHS terms and an RHS value. This happens when date_from coverage differs between the two sides (including a side whose earliest date_from falls @@ -286,31 +308,33 @@ def _translate_constraint_tables( SWQLD1 generator_output KINGASF1 0.14 links: - isp_name name p_nom_extendable - NSW-QLD NSW-QLD_existing False - NSW-QLD NSW-QLD_exp_2026 True + isp_name name p_nom_extendable build_year lifetime + NSW-QLD NSW-QLD_existing False 2025 inf + NSW-QLD NSW-QLD_exp_2026 True 2026 inf generators: - isp_name name - KINGASF1 KINGASF1 - N2 Solar N2 Solar_2026 - N2 Solar N2 Solar_2028 + isp_name name build_year lifetime + KINGASF1 KINGASF1 2025 3 # retires 2028: out of service from the 2028 period + N2 Solar N2 Solar_2026 2026 30 + N2 Solar N2 Solar_2028 2028 30 storage: - isp_name name - Q8 Battery - 2h Q8 Battery - 2h - SQ BESS SQ BESS_2026 - SQ BESS SQ BESS_2028 + isp_name name build_year lifetime + Q8 Battery - 2h Q8 Battery - 2h 2025 inf + SQ BESS SQ BESS_2026 2026 20 + SQ BESS SQ BESS_2028 2028 20 demand_nodes: name SQ - returns lhs (2028 rows mirror 2026): + returns lhs: constraint_id investment_period variable_name component attribute coefficient SWQLD1 2026 NSW-QLD_existing Link p 0.84 SWQLD1 2026 NSW-QLD_exp_2026 Link p 0.84 SWQLD1 2026 KINGASF1 Generator p 0.14 + SWQLD1 2028 NSW-QLD_existing Link p 0.84 + SWQLD1 2028 NSW-QLD_exp_2026 Link p 0.84 # KINGASF1 retired: no 2028 term returns rhs: constraint_id investment_period timeslice rhs constraint_type @@ -338,6 +362,8 @@ def _translate_constraint_tables( ) _raise_on_terms_not_in_model(lhs, variable_name_mapping) lhs = _expand_terms_to_model_components(lhs, variable_name_mapping) + lhs = _drop_terms_before_build_year(lhs) + lhs = _drop_terms_from_retirement_year(lhs) return _drop_one_sided_constraint_periods(lhs, rhs) @@ -474,7 +500,13 @@ def _map_constraint_variables_to_pypsa_names( _exp_ suffix) covers the path's existing link and each of its expansion links, and a new entrant generator or storage unit's ID covers each of its per-build-year components. An existing unit's name maps to - itself. + itself. Each row also carries its component's in-service window as + build_year and retirement_year — the latter computed as build_year + + lifetime, the year PyPSA deactivates the component, so a component with + an infinite lifetime never retires. _drop_terms_before_build_year and + _drop_terms_from_retirement_year use the pair to scope terms to the + periods the component is in service; Load rows have neither year — + demand is not built. Demand nodes — the buses with demand attached, not all buses — appear as Load rows: a load term resolves to the Load component pypsa_build attaches @@ -485,60 +517,88 @@ def _map_constraint_variables_to_pypsa_names( I/O Example: links: - isp_name name - NSW-QLD NSW-QLD_existing - NSW-QLD NSW-QLD_exp_2030 + isp_name name build_year lifetime + NSW-QLD NSW-QLD_existing 2029 inf + NSW-QLD NSW-QLD_exp_2030 2030 inf generators: - isp_name name - KINGASF1 KINGASF1 - N2 Solar N2 Solar_2030 - N2 Solar N2 Solar_2040 + isp_name name build_year lifetime + KINGASF1 KINGASF1 2029 11 # closes 2040 + N2 Solar N2 Solar_2030 2030 30 + N2 Solar N2 Solar_2040 2040 30 storage: - isp_name name - Q8 Battery - 2h Q8 Battery - 2h - SQ BESS SQ BESS_2030 - SQ BESS SQ BESS_2040 + isp_name name build_year lifetime + Q8 Battery - 2h Q8 Battery - 2h 2029 inf + SQ BESS SQ BESS_2030 2030 20 + SQ BESS SQ BESS_2040 2040 20 demand_nodes: name SQ returns: - constraint_variable_name pypsa_model_name component - NSW-QLD NSW-QLD_existing Link - NSW-QLD NSW-QLD_exp_2030 Link - KINGASF1 KINGASF1 Generator - N2 Solar N2 Solar_2030 Generator - N2 Solar N2 Solar_2040 Generator - Q8 Battery - 2h Q8 Battery - 2h Storage - SQ BESS SQ BESS_2030 Storage - SQ BESS SQ BESS_2040 Storage - SQ load_SQ Load + constraint_variable_name pypsa_model_name component build_year retirement_year + NSW-QLD NSW-QLD_existing Link 2029 inf + NSW-QLD NSW-QLD_exp_2030 Link 2030 inf + KINGASF1 KINGASF1 Generator 2029 2040 + N2 Solar N2 Solar_2030 Generator 2030 2060 + N2 Solar N2 Solar_2040 Generator 2040 2070 + Q8 Battery - 2h Q8 Battery - 2h Storage 2029 inf + SQ BESS SQ BESS_2030 Storage 2030 2050 + SQ BESS SQ BESS_2040 Storage 2040 2060 + SQ load_SQ Load # data term: never built, always present """ - to_mapping = {"isp_name": "constraint_variable_name", "name": "pypsa_model_name"} frames = [ - links.loc[:, ["isp_name", "name"]] - .rename(columns=to_mapping) - .assign(component="Link"), - generators.loc[:, ["isp_name", "name"]] - .rename(columns=to_mapping) - .assign(component="Generator"), - storage.loc[:, ["isp_name", "name"]] - .rename(columns=to_mapping) - .assign(component="Storage"), + _component_mapping_rows(links, "Link"), + _component_mapping_rows(generators, "Generator"), + _component_mapping_rows(storage, "Storage"), pd.DataFrame( { "constraint_variable_name": demand_nodes["name"], "pypsa_model_name": "load_" + demand_nodes["name"], "component": "Load", + "build_year": np.nan, + "retirement_year": np.nan, } ), ] return _concat_non_empty( - frames, ["constraint_variable_name", "pypsa_model_name", "component"] + frames, + [ + "constraint_variable_name", + "pypsa_model_name", + "component", + "build_year", + "retirement_year", + ], + ) + + +def _component_mapping_rows( + components: pd.DataFrame, component_type: str +) -> pd.DataFrame: + """One mapping row per component, carrying its in-service window as + build_year and retirement_year — build_year + lifetime, the year PyPSA + deactivates the component, so an infinite lifetime never retires. + + I/O Example (component_type="Generator"): + components: + isp_name name build_year lifetime + KINGASF1 KINGASF1 2029 11 + LOYYB LOYYB 2029 inf + + returns: + constraint_variable_name pypsa_model_name component build_year retirement_year + KINGASF1 KINGASF1 Generator 2029 2040 + LOYYB LOYYB Generator 2029 inf + """ + rows = components.loc[:, ["isp_name", "name", "build_year"]].rename( + columns={"isp_name": "constraint_variable_name", "name": "pypsa_model_name"} ) + rows["component"] = component_type + rows["retirement_year"] = components["build_year"] + components["lifetime"] + return rows def _raise_on_terms_not_in_model( @@ -597,20 +657,20 @@ def _expand_terms_to_model_components( SWQLD1 N2 Solar Generator 0.5 variable_name_mapping: - constraint_variable_name pypsa_model_name component - NSW-QLD NSW-QLD_existing Link - NSW-QLD NSW-QLD_exp_2030 Link - KINGASF1 KINGASF1 Generator - N2 Solar N2 Solar_2030 Generator - N2 Solar N2 Solar_2040 Generator + constraint_variable_name pypsa_model_name component build_year retirement_year + NSW-QLD NSW-QLD_existing Link 2029 inf + NSW-QLD NSW-QLD_exp_2030 Link 2030 inf + KINGASF1 KINGASF1 Generator 2029 2040 + N2 Solar N2 Solar_2030 Generator 2030 2060 + N2 Solar N2 Solar_2040 Generator 2040 2070 returns: - constraint_id variable_name component coefficient - SWQLD1 NSW-QLD_existing Link 0.84 - SWQLD1 NSW-QLD_exp_2030 Link 0.84 - SWQLD1 KINGASF1 Generator 0.14 # existing unit: unchanged - SWQLD1 N2 Solar_2030 Generator 0.5 # new entrant: one term - SWQLD1 N2 Solar_2040 Generator 0.5 # per build year + constraint_id variable_name component coefficient build_year retirement_year + SWQLD1 NSW-QLD_existing Link 0.84 2029 inf + SWQLD1 NSW-QLD_exp_2030 Link 0.84 2030 inf + SWQLD1 KINGASF1 Generator 0.14 2029 2040 # existing unit: unchanged + SWQLD1 N2 Solar_2030 Generator 0.5 2030 2060 # new entrant: one term + SWQLD1 N2 Solar_2040 Generator 0.5 2040 2070 # per build year """ expanded = lhs.merge( variable_name_mapping, @@ -621,6 +681,67 @@ def _expand_terms_to_model_components( return expanded.rename(columns={"pypsa_model_name": "variable_name"}) +def _drop_terms_before_build_year(lhs: pd.DataFrame) -> pd.DataFrame: + """Drops each term in the investment periods before its component's build + year, then drops the build_year column. + + A component contributes nothing to a period before it is built — PyPSA + fixes its dispatch (p) to zero there, and its capacity (p_nom) is a single + horizon-wide variable that would otherwise let capacity built for a later + period alter an earlier period's constraint. Terms with no build_year + (load data terms) apply in every period. + + I/O Example (columns abridged): + lhs: + constraint_id investment_period variable_name build_year + SWQLD1 2026 N2 Solar_2026 2026 + SWQLD1 2026 N2 Solar_2028 2028 # not built in 2026: dropped + SWQLD1 2028 N2 Solar_2028 2028 + SWQLD1 2026 load_SQ # no build year: kept + + returns: + constraint_id investment_period variable_name + SWQLD1 2026 N2 Solar_2026 + SWQLD1 2028 N2 Solar_2028 + SWQLD1 2026 load_SQ + """ + built = lhs["build_year"].isna() | (lhs["build_year"] <= lhs["investment_period"]) + return lhs[built].drop(columns="build_year") + + +def _drop_terms_from_retirement_year(lhs: pd.DataFrame) -> pd.DataFrame: + """Drops each term in the investment periods from its component's + retirement year onward, then drops the retirement_year column. The + retirement year is build_year + lifetime and the bound is exclusive, + matching when PyPSA deactivates the component (it is in service for + build_year <= period < build_year + lifetime). + + A retired component's dispatch (p) is zero, and its capacity (p_nom) is a + single horizon-wide variable that would otherwise keep counting capacity + after it has left the system. An infinite lifetime gives an inf + retirement year and load data terms have none — both apply in every + period. + + I/O Example (columns abridged): + lhs: + constraint_id investment_period variable_name retirement_year + SWQLD1 2026 KINGASF1 2028 + SWQLD1 2028 KINGASF1 2028 # retired: dropped + SWQLD1 2026 load_SQ # no retirement year: kept + SWQLD1 2028 load_SQ + + returns: + constraint_id investment_period variable_name + SWQLD1 2026 KINGASF1 + SWQLD1 2026 load_SQ + SWQLD1 2028 load_SQ + """ + in_service = lhs["retirement_year"].isna() | ( + lhs["investment_period"] < lhs["retirement_year"] + ) + return lhs[in_service].drop(columns="retirement_year") + + def _drop_one_sided_constraint_periods( lhs: pd.DataFrame, rhs: pd.DataFrame ) -> tuple[pd.DataFrame, pd.DataFrame]: diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py index 9ede61a2..49180962 100644 --- a/tests/test_translator/test_constraints.py +++ b/tests/test_translator/test_constraints.py @@ -49,26 +49,27 @@ def _constraint_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: def _links(csv_str_to_df) -> pd.DataFrame: return csv_str_to_df(""" - isp_name, name, p_nom_extendable - NSW-QLD, NSW-QLD_existing, False - NSW-QLD, NSW-QLD_exp_2026, True + isp_name, name, p_nom_extendable, build_year, lifetime + NSW-QLD, NSW-QLD_existing, False, 2025, inf + NSW-QLD, NSW-QLD_exp_2026, True, 2026, inf """) def _generators(csv_str_to_df) -> pd.DataFrame: - """Existing units carry their own name as isp_name. LATEGEN backs the - date_from-after-all-periods test.""" + """Existing units carry their own name as isp_name, a build year before + the first investment period, and an infinite lifetime (no scheduled + closure). LATEGEN backs the date_from-after-all-periods test.""" return csv_str_to_df(""" - isp_name, name - KINGASF1, KINGASF1 - LATEGEN, LATEGEN + isp_name, name, build_year, lifetime + KINGASF1, KINGASF1, 2025, inf + LATEGEN, LATEGEN, 2025, inf """) def _storage(csv_str_to_df) -> pd.DataFrame: return csv_str_to_df(""" - isp_name, name - Q8 Battery - 2h, Q8 Battery - 2h + isp_name, name, build_year, lifetime + Q8 Battery - 2h, Q8 Battery - 2h, 2025, inf """) @@ -110,6 +111,45 @@ def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): ) +def test_rhs_row_with_blank_timeslice_is_kept_as_fallback( + csv_str_to_df, sample_model_config +): + """A blank-timeslice RHS row is the constraint's fallback limit, applying + at the snapshots no named-timeslice row covers, and passes through + alongside the named rows rather than being dropped as a NaN group.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, , 2800, + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, , 2800, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, , 2800, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): ispypsa_tables = _constraint_tables(csv_str_to_df) @@ -254,6 +294,42 @@ def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config) ) +def test_date_from_exactly_on_period_start_applies_in_that_period( + csv_str_to_df, sample_model_config +): + """The value active at a period's start includes one dated exactly on it: + FY2028 starts 2027-07-01, so a 2027-07-01 value supersedes the baseline + for 2028 — the boundary is inclusive.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, 2027-07-01T00:00:00 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + rhs = result["custom_constraints_rhs"] + rhs = rhs[rhs["constraint_name"] == "SWQLD1"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_peak_demand, 2500, <= + """) + pd.testing.assert_frame_equal( + rhs.sort_values("investment_period").reset_index(drop=True), + expected, + check_dtype=False, + ) + + def test_calendar_year_periods_start_in_january(csv_str_to_df, sample_model_config): """Under calendar years the 2026 period starts 2026-01-01, so a value dated 2025-12-01 is already active in the first period — under fy it @@ -326,6 +402,44 @@ def test_date_from_after_all_periods_contributes_nothing( ) +def test_lhs_coefficient_superseded_mid_horizon(csv_str_to_df, sample_model_config): + """A dated LHS row supersedes the baseline coefficient in the periods + whose start it falls before, mirroring the RHS date_from resolution.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, generator_output, KINGASF1, 0.3, 2026-12-01T00:00:00 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.3 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_config): ispypsa_tables = _constraint_tables(csv_str_to_df) ispypsa_tables["custom_constraints"] = csv_str_to_df(""" @@ -559,16 +673,114 @@ def test_new_entrant_terms_expand_to_per_build_year_components( csv_str_to_df, sample_model_config ): """A term naming a new entrant generator's isp_name covers each of its - per-build-year components, mirroring link_flow expansion.""" + per-build-year components from their build years onward — the 2028 + component's dispatch is fixed at zero in 2026, so its term only enters + the constraint once built.""" ispypsa_tables = _constraint_tables(csv_str_to_df) ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" constraint_id, term_type, variable_name, coefficient, date_from SWQLD1, generator_output, N2 Solar, 0.5, """) generators = csv_str_to_df(""" - isp_name, name - N2 Solar, N2 Solar_2026 - N2 Solar, N2 Solar_2028 + isp_name, name, build_year, lifetime + N2 Solar, N2 Solar_2026, 2026, 30 + N2 Solar, N2 Solar_2028, 2028, 30 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + generators, + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, N2 Solar_2026, Generator, p, 0.5 + SWQLD1, 2028, N2 Solar_2026, Generator, p, 0.5 + SWQLD1, 2028, N2 Solar_2028, Generator, p, 0.5 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_new_entrant_storage_term_expands_to_per_build_year_components( + csv_str_to_df, sample_model_config +): + """A storage_output term naming a new entrant unit's isp_name covers each + of its per-build-year components from their build years onward, matching + the generator behaviour.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, storage_output, SQ BESS, 0.43, + """) + storage = csv_str_to_df(""" + isp_name, name, build_year, lifetime + SQ BESS, SQ BESS_2026, 2026, 30 + SQ BESS, SQ BESS_2028, 2028, 30 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + storage, + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, SQ BESS_2026, Storage, p, 0.43 + SWQLD1, 2028, SQ BESS_2026, Storage, p, 0.43 + SWQLD1, 2028, SQ BESS_2028, Storage, p, 0.43 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_generator_capacity_term_targets_p_nom_of_built_components( + csv_str_to_df, sample_model_config +): + """A generator_capacity term is a term on installed capacity (p_nom) + rather than dispatch, and on a new entrant it covers each per-build-year + component from its build year onward — a component's p_nom is a single + horizon-wide variable, so counting it earlier would let capacity built + for 2028 alter the 2026 constraint. Here alongside an output (p) term on + the same unit.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_capacity, N2 Solar, 1.0, + SWQLD1, generator_output, N2 Solar, 0.5, + """) + generators = csv_str_to_df(""" + isp_name, name, build_year, lifetime + N2 Solar, N2 Solar_2026, 2026, 30 + N2 Solar, N2 Solar_2028, 2028, 30 """) result = _translate_custom_constraints( @@ -582,8 +794,10 @@ def test_new_entrant_terms_expand_to_per_build_year_components( expected_lhs = csv_str_to_df(""" constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, N2 Solar_2026, Generator, p_nom, 1.0 + SWQLD1, 2028, N2 Solar_2026, Generator, p_nom, 1.0 + SWQLD1, 2028, N2 Solar_2028, Generator, p_nom, 1.0 SWQLD1, 2026, N2 Solar_2026, Generator, p, 0.5 - SWQLD1, 2026, N2 Solar_2028, Generator, p, 0.5 SWQLD1, 2028, N2 Solar_2026, Generator, p, 0.5 SWQLD1, 2028, N2 Solar_2028, Generator, p, 0.5 SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 @@ -601,6 +815,55 @@ def test_new_entrant_terms_expand_to_per_build_year_components( ) +def test_generator_terms_dropped_from_retirement_year( + csv_str_to_df, sample_model_config +): + """A component's terms leave the constraint from its retirement year + (build_year + lifetime) onward — the bound is exclusive, matching when + PyPSA deactivates the component, so a unit built in 2025 with a 3-year + lifetime contributes in 2026 but not in the 2028 period itself.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, NSW-QLD, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + generators = csv_str_to_df(""" + isp_name, name, build_year, lifetime + KINGASF1, KINGASF1, 2025, 3 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + generators, + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2028, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2028, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + def test_constraint_with_no_lhs_terms_dropped_and_logged( csv_str_to_df, sample_model_config, caplog ): @@ -800,8 +1063,8 @@ def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): result = _translate_custom_constraints( ispypsa_tables, _links(csv_str_to_df), - pd.DataFrame(columns=["isp_name", "name"]), - pd.DataFrame(columns=["isp_name", "name"]), + pd.DataFrame(columns=["isp_name", "name", "build_year", "lifetime"]), + pd.DataFrame(columns=["isp_name", "name", "build_year", "lifetime"]), pd.DataFrame(columns=["name"]), sample_model_config, ) @@ -856,8 +1119,8 @@ def test_no_constraints_and_no_expansion_yields_header_only_tables( ), } links = csv_str_to_df(""" - isp_name, name, p_nom_extendable - NSW-QLD, NSW-QLD_existing, False + isp_name, name, p_nom_extendable, build_year, lifetime + NSW-QLD, NSW-QLD_existing, False, 2025, inf """) result = _translate_custom_constraints( @@ -889,6 +1152,64 @@ def test_no_constraints_and_no_expansion_yields_header_only_tables( ) +def test_constraints_translate_without_any_expansion_options( + csv_str_to_df, sample_model_config +): + """The mirror of the empty-constraint-tables case: populated constraint + tables with an empty expansion options table (and so no extendable links) + translate as usual, while every expansion block — relaxation generators + and expansion limits — is empty.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + links = csv_str_to_df(""" + isp_name, name, p_nom_extendable, build_year, lifetime + NSW-QLD, NSW-QLD_existing, False, 2025, inf + """) + + result = _translate_custom_constraints( + ispypsa_tables, + links, + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 + """) + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, qld_winter_reference, 3500, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_winter_reference, 3500, <= + """) + _assert_lhs_and_rhs_equal(result, expected_lhs, expected_rhs) + expected_generators = csv_str_to_df(""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], expected_generators, check_dtype=False + ) + + def test_path_expansion_limit_is_max_of_forward_and_reverse( csv_str_to_df, sample_model_config ): @@ -996,6 +1317,55 @@ def test_wildcard_relaxation_option_and_cost_apply_to_every_constraint( ) +def test_blank_expansion_type_option_covers_constraint_relaxation( + csv_str_to_df, sample_model_config +): + """An option row with a blank expansion_type is a wildcard covering + forward, reverse and constraint_relaxation alike, so a row keyed only by + the constraint's expansion_id still yields its relaxation.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, , 400, SWQLD1 Option 2 + """) + + result = _translate_custom_constraints( + ispypsa_tables, + _links(csv_str_to_df), + _generators(csv_str_to_df), + _storage(csv_str_to_df), + _demand_nodes(csv_str_to_df), + sample_model_config, + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + SWQLD1_exp_2028, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2028, inf, {80000 * _ANNUITY_PER_DOLLAR} + """) + generators = result["custom_constraints_generators"] + pd.testing.assert_frame_equal( + generators.sort_values("name").reset_index(drop=True), + expected_generators, + check_dtype=False, + rtol=1e-5, + ) + rhs = result["custom_constraints_rhs"] + expected_limits = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + limits = rhs[rhs["constraint_name"].str.endswith("_expansion_limit")] + pd.testing.assert_frame_equal( + limits.sort_values("constraint_name").reset_index(drop=True), + expected_limits, + check_dtype=False, + ) + + def test_relaxation_option_for_constraint_not_in_model_is_dropped( csv_str_to_df, sample_model_config ): From 12d4503a13e08bc865935b86d8fe0cfbd5cd61f0 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 27 Aug 2026 10:05:53 +1000 Subject: [PATCH 20/20] Document the forward quantisation where it happens The forward-quantisation semantics - date_from, build_year and lifetime all rounding onto the investment-period sequence the way PyPSA's multi-investment-period activity mask does - were described on the top-level orchestrator, away from the function that implements them and duplicated across its step bullets. State the rule once on _translate_constraint_tables, collapse the bullets that restated it, and leave the orchestrator a pointer. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DBECPsKWwUCxESCcU2Ykxk --- src/ispypsa/translator/constraints.py | 47 +++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py index cfbe2619..5ebca656 100644 --- a/src/ispypsa/translator/constraints.py +++ b/src/ispypsa/translator/constraints.py @@ -116,17 +116,9 @@ def _translate_custom_constraints( network_expansion_options.yaml and network_transmission_path_expansion_costs.yaml). - Everything time-varying is quantised forward onto the investment-period - sequence, mirroring how PyPSA's multi-investment-period optimisation treats - the components themselves. A dated LHS or RHS value takes effect at the - first period whose start falls on or after its date_from (the boundary - is inclusive), so a change landing mid-period defers to the next period - rather than reaching back into the one it landed in. A term is kept in - exactly the periods PyPSA activates its component — build_year <= period - < build_year + lifetime, compared against the period labels — so a - component built between two periods joins the constraint at the next - label, one retiring between two periods leaves at the next label, and - the constraint never counts capacity or dispatch the model doesn't have. + How time-varying inputs land on the investment periods — the forward + quantisation of date_from, build_year and lifetime — is described on + _translate_constraint_tables. In the output tables, a blank investment_period means the row applies in every period. A named timeslice scopes the RHS to the snapshots inside that @@ -256,12 +248,24 @@ def _translate_constraint_tables( and one RHS row per constraint, investment period and (RHS only) timeslice, still keyed by constraint_id. + Everything time-varying is quantised forward onto the investment-period + sequence, mirroring how PyPSA's multi-investment-period optimisation + treats the components themselves. A dated LHS or RHS value takes effect + at the first period whose start falls on or after its date_from (the + boundary is inclusive), so a change landing mid-period defers to the + next period rather than reaching back into the one it landed in. A term + is kept in exactly the periods PyPSA activates its component — + build_year <= period < build_year + lifetime, compared against the + period labels — so a component built between two periods joins the + constraint at the next label, one retiring between two periods leaves at + the next label, and the constraint never counts capacity or dispatch the + model doesn't have. + The translation steps: - the LHS and RHS values active during each investment period are - determined: each group keeps the value with the most recent date_from - falling on or before the period's start, blank date_from rows acting - as the earliest values. + resolved per the date_from quantisation above, blank date_from rows + acting as the earliest values. - term_type values are mapped to PyPSA component and attribute combinations. - every LHS term must resolve to a component in the model. A term naming a @@ -278,16 +282,11 @@ def _translate_constraint_tables( the "load_" Load component at its demand node — a data term whose p_set attribute pypsa_build resolves from the demand trace, not an optimisation variable. - - each term is then dropped in the investment periods before its - component's build year, so a per-build-year component only enters the - constraint from its build year onward. Existing components' build years - precede the horizon and load terms have no build year, so both apply in - every period. - - terms are likewise dropped from their component's retirement year - onward — build_year + lifetime, the exclusive bound at which PyPSA - deactivates the component — so a unit retiring in a period's label - year contributes nothing to that period. Components with an infinite - lifetime never retire. + - each term is then dropped in the investment periods outside its + component's in-service window, per the activity quantisation above. + Existing components' build years precede the horizon, an infinite + lifetime never retires, and load terms carry neither year — all of + these apply in every period. - LHS and RHS rows are dropped in investment periods where the constraint does not have both LHS terms and an RHS value. This happens when date_from coverage differs between the two sides (including a side whose earliest date_from falls