diff --git a/example_output/6.0/flow_path_augmentation_options_NNSW-SQ.csv b/example_output/6.0/flow_path_augmentation_options_NNSW-SQ.csv index 3aa9ef7..ad68f75 100644 --- a/example_output/6.0/flow_path_augmentation_options_NNSW-SQ.csv +++ b/example_output/6.0/flow_path_augmentation_options_NNSW-SQ.csv @@ -1,6 +1,6 @@ Flow path,Development path,Development Driver,Option Name,Augmentation Description,Forward direction power flow,"Notional transfer level increase (MW) Note: Same increase applies to all transfer limit conditions (Peak demand, Summer typical and Winter reference)_Forward direction","Notional transfer level increase (MW) Note: Same increase applies to all transfer limit conditions (Peak demand, Summer typical and Winter reference)_Reverse direction","Indicative cost estimate ($2023, $ million) - Note 2",Cost estimate source,Cost estimate class,Easement length in km,Lead time or Earliest in Service Date & 4,Additional REZ hosting capacity provided -NNSW-SQ,Near Queensland-New South Wales Interconnector (QNI) path,"Increase thermal capacity and, voltage and transient stability limits of 330 kV and 275 kV lines between Armidale and Braemar.",NNSW–SQ Option 1,"Option 1: • A new 330 kV single-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactor at New England Hub 5, Dumaresq, Bulli Creek, and Braemar for the New England Hub 5 – Dumaresq - Bulli Creek - Braemar 330 kV circuits. (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).",NNSW to SQ,730,900,930,Powerlink & Transgrid preparatory activity,Class 5,460.5,Medium, -NNSW-SQ,Near Queensland-New South Wales Interconnector (QNI) path,"Increase thermal capacity and, voltage and transient stability limits of 330 kV and 275 kV lines between Armidale and Braemar.",NNSW–SQ Option 2,"• A new 330 kV double-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactors at New England Hub 5, Dumaresq, Bulli Creek, and Braemar, for the 330 kV lines between New England Hub 5 and Braemar (via Dumaresq and Bulli Creek). (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).","• A new 330 kV double-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactors at New England Hub 5, Dumaresq, Bulli Creek, and Braemar, for the 330 kV lines between New England Hub 5 and Braemar (via Dumaresq and Bulli Creek). (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).",1260,1700,1210,Powerlink & Transgrid preparatory activity,Class 5,460.5,Medium, +NNSW-SQ,Near Queensland-New South Wales Interconnector (QNI) path,"Increase thermal capacity and, voltage and transient stability limits of 330 kV and 275 kV lines between Armidale and Braemar.",NNSW–SQ Option 1,"Option 1: • A new 330 kV single-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactor at New England Hub 5, Dumaresq, Bulli Creek, and Braemar for the New England Hub 5 – Dumaresq - Bulli Creek - Braemar 330 kV circuits. (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).",NNSW to SQ,730,900,930 (NSW works) 964 (QLD works),Powerlink & Transgrid preparatory activity,Class 5,460.5,Medium, +NNSW-SQ,Near Queensland-New South Wales Interconnector (QNI) path,"Increase thermal capacity and, voltage and transient stability limits of 330 kV and 275 kV lines between Armidale and Braemar.",NNSW–SQ Option 2,"• A new 330 kV double-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactors at New England Hub 5, Dumaresq, Bulli Creek, and Braemar, for the 330 kV lines between New England Hub 5 and Braemar (via Dumaresq and Bulli Creek). (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).","• A new 330 kV double-circuit line from locality of New England Hub 5 to Dumaresq to Bulli Creek to Braemar. • A new 330/275 kV transformer at Braemar. • 330 kV Line shunt reactors at New England Hub 5, Dumaresq, Bulli Creek, and Braemar, for the 330 kV lines between New England Hub 5 and Braemar (via Dumaresq and Bulli Creek). (Pre-requisite: Cut-in both Tamworth-Armidale 330 kV lines to a new substation in locality of New England Hub 5).",1260,1700,1210 (NSW works) 1309 (QLD works),Powerlink & Transgrid preparatory activity,Class 5,460.5,Medium, NNSW-SQ,Near Queensland-New South Wales Interconnector (QNI) path,"Increase thermal capacity and, voltage and transient stability limits of 330 kV and 275 kV lines between Armidale and Braemar.",NNSW–SQ Option 3,• A Virtual Transmission Line option with a 200 MW energy storage system south of Armidale and north of Braemar.,• A Virtual Transmission Line option with a 200 MW energy storage system south of Armidale and north of Braemar.,200,200,Non-network option costs to be provided by interested parties,Non-network option costs to be provided by interested parties,Non-network option costs to be provided by interested parties,Non-network option costs to be provided by interested parties,Non-network option costs to be provided by interested parties, NNSW-SQ,West of Queensland-New South Wales Interconnector (QNI) path,West of Queensland-New South Wales Interconnector (QNI) path,NNSW–SQ Option 4,• A 2000 MW HVDC bi-pole overhead transmission between a new substation in North West New South Wales (NWNSW) REZ and Halys. • A new 2000 HVDC bipole converter station in North West New South Wales • A new 2000 HVDC bipole converter station in locality of Halys. • AC network connection between HVDC converter station and 275 kV substation in Halys. • AC network connection between HVDC converter station and ac network in in NWNSW REZ. • A new 330 kV line between NWNSW REZ and Tamworth.,• A 2000 MW HVDC bi-pole overhead transmission between a new substation in North West New South Wales (NWNSW) REZ and Halys. • A new 2000 HVDC bipole converter station in North West New South Wales • A new 2000 HVDC bipole converter station in locality of Halys. • AC network connection between HVDC converter station and 275 kV substation in Halys. • AC network connection between HVDC converter station and ac network in in NWNSW REZ. • A new 330 kV line between NWNSW REZ and Tamworth.,1800,2000,4279,AEMO (TCD),Class 5b,765,Long, -NNSW-SQ,West of Queensland-New South Wales Interconnector (QNI) path,West of Queensland-New South Wales Interconnector (QNI) path,NNSW–SQ Option 5,"• Establish a new substation in NNSW and to New England REZ Hub 5. • A new 500/275 kV transformer at Halys substation • A new 1x500 kV double-circuit line between Halys and new Dumaresq 500kV substation in NNSW. • A new 1x500 kV double-circuit line between Dumaresq and New England REZ Hub 5). • 2x 500/330 kV transformers connecting to 330kV Dumaresq Pre-requisite: CQ-SQ Option 5 (QEJP), CNSW-NNSW Option 1.","• Establish a new substation in NNSW and to New England REZ Hub 5. • A new 500/275 kV transformer at Halys substation • A new 1x500 kV double-circuit line between Halys and new Dumaresq 500kV substation in NNSW. • A new 1x500 kV double-circuit line between Dumaresq and New England REZ Hub 5). • 2x 500/330 kV transformers connecting to 330kV Dumaresq Pre-requisite: CQ-SQ Option 5 (QEJP), CNSW-NNSW Option 1.",3000,2250,2600,Powerlink & Transgrid preparatory activity,Class 5,616,Long,SWQLD1 : 700 +NNSW-SQ,West of Queensland-New South Wales Interconnector (QNI) path,West of Queensland-New South Wales Interconnector (QNI) path,NNSW–SQ Option 5,"• Establish a new substation in NNSW and to New England REZ Hub 5. • A new 500/275 kV transformer at Halys substation • A new 1x500 kV double-circuit line between Halys and new Dumaresq 500kV substation in NNSW. • A new 1x500 kV double-circuit line between Dumaresq and New England REZ Hub 5). • 2x 500/330 kV transformers connecting to 330kV Dumaresq Pre-requisite: CQ-SQ Option 5 (QEJP), CNSW-NNSW Option 1.","• Establish a new substation in NNSW and to New England REZ Hub 5. • A new 500/275 kV transformer at Halys substation • A new 1x500 kV double-circuit line between Halys and new Dumaresq 500kV substation in NNSW. • A new 1x500 kV double-circuit line between Dumaresq and New England REZ Hub 5). • 2x 500/330 kV transformers connecting to 330kV Dumaresq Pre-requisite: CQ-SQ Option 5 (QEJP), CNSW-NNSW Option 1.",3000,2250,2600 (NSW works) 2659 (QLD works),Powerlink & Transgrid preparatory activity,Class 5,616,Long,SWQLD1 : 700 diff --git a/example_output/6.0/flow_path_augmentation_options_TAS-VIC.csv b/example_output/6.0/flow_path_augmentation_options_TAS-VIC.csv index 221b38d..096069e 100644 --- a/example_output/6.0/flow_path_augmentation_options_TAS-VIC.csv +++ b/example_output/6.0/flow_path_augmentation_options_TAS-VIC.csv @@ -1,3 +1,3 @@ Flow path,Development path,Development Driver,Option Name,Augmentation Description,Forward direction power flow,"Notional transfer level increase (MW) Note: Same increase applies to all transfer limit conditions (Peak demand, Summer typical and Winter reference)_Forward direction","Notional transfer level increase (MW) Note: Same increase applies to all transfer limit conditions (Peak demand, Summer typical and Winter reference)_Reverse direction","Indicative cost estimate ($2023, $ million) - Note 2",Cost estimate source,Cost estimate class,Easement length in km,Lead time or Earliest in Service Date & 4,Additional REZ hosting capacity provided TAS-VIC,Proposed Marinus Link path (path between Burnie area and Latrobe Valley area),Access to high quality wind resources in Tasmania and deep storage capability of Tasmanian hydro generation.,TAS-VIC Option 1 (Project Marinus Stage 1),"•A 750 MW monopole HVDC link between Burnie area in Tasmania and Hazelwood area in Victoria. • A new 750 MW HVDC monopole converter station in Burnie area. • A new 750 MW HVDC monopole converter station in Hazelwood area. • A new 220 kV switching station at Heybridge adjacent to the converter station. • A new double-circuit 220 kV transmission line between Sheffield, Heybridge and Burnie • A new 220 kV double-circuit line from Palmerston to Sheffield with decommissioning of existing the single-circuit line. • A new 500 kV connection from converter station in Hazelwood area. • Decommission existing Sheffield – Burnie 220 kV line. Note: HVDC interconnector components are referred to as Marinus Link 1",TAS to VIC,750 Basslink and Marinus Link 1 combined transfer limit 1344,750 Basslink and Marinus Link 1 combined transfer limit 978,3840,TasNetworks Marinuslink,Class 4,~90 (underground cable) 0 (HVAC new easement),Dec-30 (See note 7),Dec-30 (See note 7) -TAS-VIC,Proposed Marinus Link path (path between Burnie area and Latrobe Valley area),Access to high quality wind resources in Tasmania and deep storage capability of Tasmanian hydro generation.,TAS-VIC Option 2 (Project Marinus Stage 2),• An additional 750 MW monopole HVDC link between Burnie area in Tasmania and Hazelwood area in Victoria. • An additional new 750 MW HVDC monopole converter station in Burnie area. • An additional new 750 MW HVDC monopole converter station in Hazelwood area. • A new 220 kV switching station at Staverton. • A new double-circuit 220 kV transmission line from Staverton to Burnie via Hampshire • Cut-in both Sheffield-Mersey Forth double-circuit 220 kV lines at Staverton. • Capacity increase of the four Sheffield–Staverton 220 kV transmission circuits. • A new 500 kV connection from converter station in Hazelwood area. Pre-requisite: TAS-VIC Option 1 (Project Marinus Stage 1) Note: HVDC interconnector components are referred to as Marinus Link 2,• An additional 750 MW monopole HVDC link between Burnie area in Tasmania and Hazelwood area in Victoria. • An additional new 750 MW HVDC monopole converter station in Burnie area. • An additional new 750 MW HVDC monopole converter station in Hazelwood area. • A new 220 kV switching station at Staverton. • A new double-circuit 220 kV transmission line from Staverton to Burnie via Hampshire • Cut-in both Sheffield-Mersey Forth double-circuit 220 kV lines at Staverton. • Capacity increase of the four Sheffield–Staverton 220 kV transmission circuits. • A new 500 kV connection from converter station in Hazelwood area. Pre-requisite: TAS-VIC Option 1 (Project Marinus Stage 1) Note: HVDC interconnector components are referred to as Marinus Link 2,750 Basslink and Marinus Link1&2 combined transfer limit 2094,750 Basslink and Marinus Link1&2 combined transfer limit 1728,2735,TasNetworks Marinuslink,Class 4,0,Dec-32 (See note 7),Dec-32 (See note 7) +TAS-VIC,Proposed Marinus Link path (path between Burnie area and Latrobe Valley area),Access to high quality wind resources in Tasmania and deep storage capability of Tasmanian hydro generation.,TAS-VIC Option 2 (Project Marinus Stage 2),• An additional 750 MW monopole HVDC link between Burnie area in Tasmania and Hazelwood area in Victoria. • An additional new 750 MW HVDC monopole converter station in Burnie area. • An additional new 750 MW HVDC monopole converter station in Hazelwood area. • A new 220 kV switching station at Staverton. • A new double-circuit 220 kV transmission line from Staverton to Burnie via Hampshire • Cut-in both Sheffield-Mersey Forth double-circuit 220 kV lines at Staverton. • Capacity increase of the four Sheffield–Staverton 220 kV transmission circuits. • A new 500 kV connection from converter station in Hazelwood area. Pre-requisite: TAS-VIC Option 1 (Project Marinus Stage 1) Note: HVDC interconnector components are referred to as Marinus Link 2,• An additional 750 MW monopole HVDC link between Burnie area in Tasmania and Hazelwood area in Victoria. • An additional new 750 MW HVDC monopole converter station in Burnie area. • An additional new 750 MW HVDC monopole converter station in Hazelwood area. • A new 220 kV switching station at Staverton. • A new double-circuit 220 kV transmission line from Staverton to Burnie via Hampshire • Cut-in both Sheffield-Mersey Forth double-circuit 220 kV lines at Staverton. • Capacity increase of the four Sheffield–Staverton 220 kV transmission circuits. • A new 500 kV connection from converter station in Hazelwood area. Pre-requisite: TAS-VIC Option 1 (Project Marinus Stage 1) Note: HVDC interconnector components are referred to as Marinus Link 2,750 Basslink and Marinus Link1&2 combined transfer limit 2094,750 Basslink and Marinus Link1&2 combined transfer limit 1728,2735,TasNetworks Marinuslink,Class 4,0 (underground cable) ~94 (HVAC new easement),Dec-32 (See note 7),Dec-32 (See note 7) diff --git a/example_output/6.0/rez_augmentation_options_TAS.csv b/example_output/6.0/rez_augmentation_options_TAS.csv index fd97abc..c642f50 100644 --- a/example_output/6.0/rez_augmentation_options_TAS.csv +++ b/example_output/6.0/rez_augmentation_options_TAS.csv @@ -4,7 +4,7 @@ T1,North East Tasmania,,"• Second 220 kV double-circuit line between the new s T2,North West Tasmania,Option 1,• Build a new 220 kV switching station at Staverton and cut-in all Sheffield-Mersey Forth 220 kV lines at Staverton. • Build a new double-circuit Staverton-Hampshire Hills 220 kV line. • Build new Hampshire Hills wind collector station.,800,304,AEMO TCD,Class 4,0.38,59,Short,Included as connection cost T2,North West Tasmania,Option 2,"• Build new “Farrell 2” wind collector station on west coast Tasmania (nearby existing Farrell substation) and establish new double-circuit Farrell2-Hampshire Hills 220 kV line. Pre-requisite: TAS-VIC Option 2, T2 Option 1.",500,258,AEMO TCD,Class 5a,0.516,65,Long, T2,North West Tasmania,Option 3,"• Build double-circuit West Montagu-Hampshire 220 kV line. • Build a second 220 kV double-circuit line from Hampshire Hills to Burnie to Heybridge. Pre-requisite: TAS-VIC Option 2, T2 Option 1.",800,534,AEMO TCD,Class 5a,0.6675,132,Medium, -T3,Central Highlands,Option 1,"• Build a Palmerston-Waddamana 220 kV double-circuit line. Prior to (without) MarinusLink: • If before Marinus Link 1, bring forward the rebuild of Palmerston-Sheffield 220 kV line as double-circuit and build 2 x power flow controllers on the 2 x 220 kV transmission lines from Palmerston-Sheffield. With MarinusLink: • If after Marinus Link 1, build 2 x power flow controllers on the 2 x 220 kV transmission lines from Palmerton-Sheffield.",690,201,AEMO TCD,Class 5a,0.6289855072463768,0,Short,Included as connection cost +T3,Central Highlands,Option 1,"• Build a Palmerston-Waddamana 220 kV double-circuit line. Prior to (without) MarinusLink: • If before Marinus Link 1, bring forward the rebuild of Palmerston-Sheffield 220 kV line as double-circuit and build 2 x power flow controllers on the 2 x 220 kV transmission lines from Palmerston-Sheffield. With MarinusLink: • If after Marinus Link 1, build 2 x power flow controllers on the 2 x 220 kV transmission lines from Palmerton-Sheffield.",690,201 (with Marinus Link) 434 (without Marinus Link),AEMO TCD,Class 5a,0.6289855072463768,0 (with Marinus Link) 0 (without Marinus Link),Short,Included as connection cost T3,Central Highlands,Option 2,"• Build a second Palmerston-Sheffield 220 kV double-circuit line. Pre-requisite: TAS-VIC Option 2, T3 Option 1.",675,274,AEMO TCD,Class 5a,0.4059259259259259,79,Long, T4,North Tasmania Coast,Option 1,• Build a new Burnie-Heybridge-Sheffield 220 kV double-circuit line. Pre-requisites: TAS-VIC Option 2.,1360,206,AEMO TCD,Class 5a,0.1514705882352941,53,Long,Included as connection cost T4,North Tasmania Coast,Option 2,• Build a new George Town-Sheffield 220 kV double-circuit line. • Build 2 x power flow controllers on the 2 x 220 kV double-circuit line between George Town-Hadspen. Pre-requisite: TAS-VIC Option 2.,900,307,AEMO TCD,Class 5a,0.3411111111111111,68,Long,Included as connection cost diff --git a/example_output/7.0/pumped_hydro_properties.csv b/example_output/7.0/pumped_hydro_properties.csv index 9034a52..31cdf69 100644 --- a/example_output/7.0/pumped_hydro_properties.csv +++ b/example_output/7.0/pumped_hydro_properties.csv @@ -1,9 +1,9 @@ Property,Installed capacity (MW),Storage capacity (hours),Pumping efficiency (%) -Snowy 2.0,2200.0,159.0,76.0 -Lower Tumut,1500.0,,78.0 -Wivenhoe,570.0,10.0,70.0 -Shoalhaven,240.0,64.0,70.0 -Kidston,250.0,6.0,80.0 -Cethana,750.0,20.0,76.0 -Borumba,1998.0,24.0,76.0 +Snowy 2.0,2200,159.0,76.0 +Lower Tumut,1500,,78.0 +Wivenhoe,570,10.0,70.0 +Shoalhaven,240,64.0,70.0 +Kidston,250 (generation) 325 (pump),6.0,80.0 +Cethana,750,20.0,76.0 +Borumba,1998,24.0,76.0 New Entrant PHES,,,76.0 diff --git a/example_output/7.3/gas_system_properties_pipelines.csv b/example_output/7.3/gas_system_properties_pipelines.csv index ee5b9b7..0b35862 100644 --- a/example_output/7.3/gas_system_properties_pipelines.csv +++ b/example_output/7.3/gas_system_properties_pipelines.csv @@ -18,7 +18,7 @@ Comet Ridge to Wallumbilla Pipeline,Complete and operational.,QLD,950 North 175 GLNG Pipeline,Online,QLD,1430,Data shown from 2025 GSOO Atlas Gas Pipeline,Commissioning,QLD,Commercial in confidence,Data shown from 2025 GSOO Darling Downs Pipeline (“DDP”),Online,QLD,DDP90 Southern 155 TJ/d DDP133 Western 502 TJ/d DDP133 Eastern 160 TJ/d DDP134 Western 365 TJ/d DDP134 Eastern 268 TJ/d,Data shown from 2025 GSOO -Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350,Data shown from 2025 GSOO +Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350 (Summer) 362 (Winter),Data shown from 2025 GSOO Northern Gas Pipeline (“NGP”),Online,NT / QLD,106,Data shown from 2025 GSOO Queensland Gas Pipeline (“QGP”),Online,QLD,145 TJ/d North 37.2 TJ/d South,Data shown from 2025 GSOO North Queensland Gas Pipeline (“NQGP”),Online,QLD,108,Data shown from 2025 GSOO diff --git a/example_output/7.5/gas_system_properties_pipelines.csv b/example_output/7.5/gas_system_properties_pipelines.csv index ee5b9b7..0b35862 100644 --- a/example_output/7.5/gas_system_properties_pipelines.csv +++ b/example_output/7.5/gas_system_properties_pipelines.csv @@ -18,7 +18,7 @@ Comet Ridge to Wallumbilla Pipeline,Complete and operational.,QLD,950 North 175 GLNG Pipeline,Online,QLD,1430,Data shown from 2025 GSOO Atlas Gas Pipeline,Commissioning,QLD,Commercial in confidence,Data shown from 2025 GSOO Darling Downs Pipeline (“DDP”),Online,QLD,DDP90 Southern 155 TJ/d DDP133 Western 502 TJ/d DDP133 Eastern 160 TJ/d DDP134 Western 365 TJ/d DDP134 Eastern 268 TJ/d,Data shown from 2025 GSOO -Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350,Data shown from 2025 GSOO +Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350 (Summer) 362 (Winter),Data shown from 2025 GSOO Northern Gas Pipeline (“NGP”),Online,NT / QLD,106,Data shown from 2025 GSOO Queensland Gas Pipeline (“QGP”),Online,QLD,145 TJ/d North 37.2 TJ/d South,Data shown from 2025 GSOO North Queensland Gas Pipeline (“NQGP”),Online,QLD,108,Data shown from 2025 GSOO diff --git a/example_output/7.8/gas_system_properties_pipelines.csv b/example_output/7.8/gas_system_properties_pipelines.csv index 01f7af1..58936c6 100644 --- a/example_output/7.8/gas_system_properties_pipelines.csv +++ b/example_output/7.8/gas_system_properties_pipelines.csv @@ -18,7 +18,7 @@ Moomba to Adelaide Pipeline System (“MAPS”),Online,SA,249 South 85 North,Dat GLNG Pipeline,Online,QLD,1430,Data shown from 2026 GSOO Atlas Gas Pipeline,Commissioning,QLD,Commercial in confidence,Data shown from 2026 GSOO Darling Downs Pipeline (“DDP”),Online,QLD,DDP90 Southern 155 TJ/d DDP133 Western 502 TJ/d DDP133 Eastern 160 TJ/d DDP134 Western 365 TJ/d DDP134 Eastern 268 TJ/d,Data shown from 2026 GSOO -Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350,Data shown from 2026 GSOO +Eastern Gas Pipeline (“EGP”),Online,VIC / NSW,350 (Summer) 362 (Winter),Data shown from 2026 GSOO Northern Gas Pipeline (“NGP”),Online,NT / QLD,106,Data shown from 2026 GSOO Queensland Gas Pipeline (“QGP”),Online,QLD,145 TJ/d North 37.2 TJ/d South,Data shown from 2026 GSOO North Queensland Gas Pipeline (“NQGP”),Online,QLD,108,Data shown from 2026 GSOO diff --git a/src/isp_workbook_parser/sanitisers.py b/src/isp_workbook_parser/sanitisers.py index bce3144..53abe28 100644 --- a/src/isp_workbook_parser/sanitisers.py +++ b/src/isp_workbook_parser/sanitisers.py @@ -149,12 +149,38 @@ def _remove_series_bracketed_footnotes( return series.str.replace(r"\[footnote\s*\d+\]", "", regex=True) +def _where_multiple_values_with_notes( + series: pd.Index | pd.Series, +) -> pd.Index | pd.Series: + """Flags cells that hold more than one value, each with its own bracketed note, + e.g. ``'930 (NSW works) 964 (QLD works)'``. + + There is no single numeric value that can stand in for such a cell, so + `_remove_series_notes_after_values` leaves them as text rather than silently + discarding every value but the first. + + A second value is only recognised where nothing but non-alphanumeric characters + (whitespace, a comma, an approximation symbol) separates it from the closing + parenthesis of the first note. This is what distinguishes a genuine second value + from a footnote reference, which is always preceded by words + (e.g. ``'400 (with VNI SIPS) - Note 8'``). + """ + return series.str.contains( + r"^[~<>=]?\s*[0-9\.]+\s+\([^()]*\)[^0-9A-Za-z]*[0-9]", regex=True, na=False + ) + + def _remove_series_notes_after_values( series: pd.Index | pd.Series, ) -> pd.Index | pd.Series: """Removes notes after numeric values in a `pandas.Series` or `pandas.Index` - This is done using three regular expression substitutions: + Cells that hold more than one value, each with its own note (see + `_where_multiple_values_with_notes`), are left as text rather than run through any + of the three substitutions below, since no single one of their values can stand in + for the cell. + + The substitutions are: 1. Capture a value (digits and decimal points) that is followed by whitespace and then an opening parenthesis. Retain the captured group and discard everything from the parenthesis onwards. @@ -165,13 +191,15 @@ def _remove_series_notes_after_values( 3. Replace any hyphen followed by one or more sequences of text preceded by a hyphen with an empty string. """ - series = series.str.replace(r"^([0-9\.]+)\s+\(.*$", r"\1", regex=True) - series = series.str.replace( + keep_full_text = _where_multiple_values_with_notes(series) + cleaned = series.str.replace(r"^([0-9\.]+)\s+\(.*$", r"\1", regex=True) + cleaned = cleaned.str.replace( r"^(?![0-9]{4}\-[0-9]{2,4})([0-9\.]+)\s?(?:(\-[\w\s\.\<\=\-\(\)]+)+)", r"\1", regex=True, ) - series = series.str.replace(r"^\-\s?(?:(\([\w\s\.\<\=\-\(\)]+)+)", "", regex=True) + cleaned = cleaned.str.replace(r"^\-\s?(?:(\([\w\s\.\<\=\-\(\)]+)+)", "", regex=True) + series = series.where(keep_full_text, cleaned) return series diff --git a/tests/test_sanitisers.py b/tests/test_sanitisers.py index faa7aa1..1764736 100644 --- a/tests/test_sanitisers.py +++ b/tests/test_sanitisers.py @@ -1,6 +1,7 @@ from pathlib import Path import pandas as pd +import pytest from isp_workbook_parser.sanitisers import ( _extract_numeric_value_millions, @@ -13,6 +14,7 @@ _replace_series_newlines_with_whitespace, _strip_series_whitespaces, _values_casting_and_sanitisation, + _where_multiple_values_with_notes, ) @@ -262,3 +264,105 @@ def test_remove_series_bracketed_footnotes(): result = _remove_series_bracketed_footnotes(unsanitised) expected = pd.Series(["750", "750", "1234", "750"]) pd.testing.assert_series_equal(result, expected) + + +# (cell, does `_where_multiple_values_with_notes` fire, result after +# `_remove_series_notes_after_values`). Cells reach these functions with newlines +# already replaced by whitespace, so the two-value cells are written that way here. +MULTIPLE_VALUE_CASES = [ + # -- Two values, each with a note: the whole cell is kept, as no single value can + # stand in for it. Four further cells in the workbooks share the shape of the + # first case ("201 (with Marinus Link) 434 (without Marinus Link)" and + # "0 (with Marinus Link) 0 (without Marinus Link)" in 6.0 REZ Augmentations + # Options, "250 (generation) 325 (pump)" in 6.0 Storage properties and + # "350 (Summer) 362 (Winter)" in 7.3 Gas System Properties). + ( # 6.0 Flow Path Augmentation options J34 + "930 (NSW works) 964 (QLD works)", + True, + "930 (NSW works) 964 (QLD works)", + ), + ( # 6.0 Flow Path Augmentation options M80, approximation symbol on the first value + "~90 (underground cable) 0 (HVAC new easement)", + True, + "~90 (underground cable) 0 (HVAC new easement)", + ), + ( # 6.0 Flow Path Augmentation options M81, and on the second value + "0 (underground cable) ~94 (HVAC new easement)", + True, + "0 (underground cable) ~94 (HVAC new easement)", + ), + # Punctuation other than whitespace may separate the two values. + ("930 (NSW works), 964 (QLD works)", True, "930 (NSW works), 964 (QLD works)"), + # The second value need not carry a note of its own. + ("250 (generation) 325", True, "250 (generation) 325"), + # -- One value plus a note: still cut down to the value, so the column casts to a + # numeric type. + ("685 (with QNI Minor)", False, "685"), # 6.0 Network Capability + ("0.16 (apply from 5,400 MW)", False, "0.16"), # 6.0 Build limits, decimal value + ( # 6.0 Network Capability. The digit after the note is a footnote reference, not + # a second value, so the separator between them may not contain letters. + "400 (with VNI SIPS) - Note 8 (Snowy 2.0 generation or pump load <= 660 " + "- Note 11)", + False, + "400", + ), + # A second bracketed note is not a second value either. + ("400 (with VNI SIPS) (Note 4)", False, "400"), # 6.0 Network Capability + # Currency symbols and thousands commas in the note (the case that motivated + # discarding the whole note rather than matching its characters). + ("5325 (based on provided cost of $5,035 in $2023)", False, "5325"), # 7.3 + # A note containing parentheses does not make the cell multi-value: the note is + # the first parenthesis-free bracketed run, so the "2023" below is part of the + # note rather than a second value. + ("5325 (provided cost ($5,035) 2023)", False, "5325"), + # -- Two values, but not in the shape substitution 1 matches, so the predicate has + # nothing to guard against: no note is stripped and the cell comes through whole. + ("930(NSW works) 964 (QLD works)", False, "930(NSW works) 964 (QLD works)"), + ( + "about 930 (NSW works) 964 (QLD works)", + False, + "about 930 (NSW works) 964 (QLD works)", + ), + # -- A known gap. Two values delimited by hyphens rather than bracketed notes are + # still cut down to the first, by substitution 2, which this predicate does not + # guard. Whether the workbooks hold cells of this shape has not been checked. + ("930 - NSW works 964 - QLD works", False, "930"), + # -- Nothing to flag. + ("930 (NSW works)", False, "930"), + ("930", False, "930"), + ("", False, ""), +] + + +def test_multiple_values_with_notes_detection_and_sanitisation(): + """Cells holding two values are detected and kept whole; cells holding one value + and a note are still cut down to that value.""" + cells, fires, sanitised = (list(field) for field in zip(*MULTIPLE_VALUE_CASES)) + unsanitised = pd.Series(cells) + pd.testing.assert_series_equal( + _where_multiple_values_with_notes(unsanitised), pd.Series(fires) + ) + pd.testing.assert_series_equal( + _remove_series_notes_after_values(unsanitised), pd.Series(sanitised) + ) + + +def test_where_multiple_values_with_notes_on_mixed_and_index_input(): + """Columns reaching the sanitisers hold a mix of strings, numbers and nulls, and + the sanitisers are also applied to a `pandas.Index` of column names.""" + series = pd.Series(["250 (generation) 325 (pump)", "250 (generation)", 42.0, None]) + pd.testing.assert_series_equal( + _where_multiple_values_with_notes(series), + pd.Series([True, False, False, False]), + ) + index = pd.Index(["250 (generation) 325 (pump)", "250 (generation)"]) + assert list(_where_multiple_values_with_notes(index)) == [True, False] + + +def test_values_casting_and_sanitisation_leaves_multi_value_column_as_text(): + """A column containing a multi-value cell cannot be cast to a numeric type, which + is the signal to consumers that the cell holds more than one value.""" + df = pd.DataFrame({"capacity": ["250 (generation) 325 (pump)", "500 (generation)"]}) + result = _values_casting_and_sanitisation(df) + assert result["capacity"].dtype == "object" + assert result["capacity"].tolist() == ["250 (generation) 325 (pump)", "500"]