From 43bb171b1e49653cf779962fa13ece0b44195341 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:10:53 +0100 Subject: [PATCH 01/87] fix(TUNES): record a leakage behaviour instead of refusing it FillRateBasedControlTUNES.send_leakage_behaviour read its fill-level scale from self._fill_level_scalefill_level_scale, an attribute name pasted in twice, which has been there since #158. The AttributeError that raised was caught by the handler's broad except as if the post had failed, so every FRBC.LeakageBehaviour was answered with PERMANENT_ERROR and none was ever recorded. mypy could not see it: the type-check task only looks at files that contain `from typing import`, and the FRBC base class is not one of them, so the attribute lookup went to Any. The next commit widens that. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_tunes.py | 2 +- tests/s2/test_frbc_tunes.py | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 3b6b956c..29a1e4f7 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -239,7 +239,7 @@ async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): leakage_behaviour_to_storage_efficiency( message=leakage, resolution=timedelta(minutes=15), - fill_level_scale=self._fill_level_scalefill_level_scale, + fill_level_scale=self._fill_level_scale, ) ], unit=PERCENTAGE, diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 645692c9..cd43fb9b 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -288,3 +288,44 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert third_call["values"][0] == frbc._nes_fill_rate_sensor_id await cem.close() + + +@pytest.mark.asyncio +async def test_a_leakage_behaviour_is_recorded_rather_than_refused( + cem_in_frbc_control_type, +): + """A leakage behaviour is posted as a storage efficiency, and not answered as a permanent error. + + The handler read its fill-level scale from an attribute whose name was pasted in twice, + and the AttributeError that raised was caught as if the post had failed, + so every leakage behaviour was refused and none was recorded. + """ + from s2python.common import NumberRange + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc.send_message = AsyncMock() + + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + await frbc.send_leakage_behaviour(leakage) + + sent = [call.args[0] for call in frbc.send_message.await_args_list] + assert not any( + getattr(message, "status", None) == ReceptionStatusValues.PERMANENT_ERROR + for message in sent + ), sent + posted_to = [ + call.args[0] if call.args else call.kwargs.get("sensor_id") + for call in fm_client.post_sensor_data.await_args_list + ] + assert 15 in posted_to, posted_to From 30dc81e8f7e1001d3bdd6a1bedeae62cbfecffbb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:12:22 +0100 Subject: [PATCH 02/87] chore(types): check every source file, including untyped function bodies The type-check task fed mypy only the files that contain `from typing import`, which were seven of eighteen, and left out frbc_simple.py, both control-type __init__.py files and the example scripts. With --follow-imports skip, anything those files defined then looked like Any to the files that were checked, which is how the previous commit's misspelt attribute went unnoticed since #158. It now checks all of src, and with --check-untyped-defs, so the bodies of functions without annotations are checked too. What that surfaced, besides the attribute fixed in the previous commit: - get_access_token stored a token whose type could be None and then used it as a header value; it now returns the token, so the caller holds a str. - get_access_token and get_versions indexed a response typed as a dict or a list; each now says so plainly when the server answers with something other than an object, instead of failing on the lookup. - The S2 translations annotated their resolution as a string while frbc_simple passes a timedelta, which pandas accepts as readily; the annotation now says so. - FillRateBasedControlTUNES overrode trigger_schedule without the arguments the base class declares. It calls its own override without them and ignores them, so it now accepts and ignores them. - The example server indexed an account that can be None and spread a dict of mixed values into add_asset; it now says so when the demo user has no organisation, and types the dict. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- pyproject.toml | 7 +-- src/flexmeasures_client/client.py | 23 +++++++--- .../s2/control_types/FRBC/frbc_tunes.py | 6 ++- .../s2/control_types/translations.py | 6 +-- .../s2/script/websockets_server.py | 7 ++- tests/client/test_init.py | 46 +++++++++++++++++++ 6 files changed, 79 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3407ad3d..33c29c8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,11 +89,8 @@ help = "Run S2 tests." cmd = "pytest tests/s2" [tool.poe.tasks.type-check] -help = "Run mypy on files with type hints." -shell = """ - files=$(find src -name \\*.py | xargs grep -l "from typing import") - mypy --follow-imports skip --ignore-missing-imports $files -""" +help = "Run mypy on every source file, including the bodies of functions without type hints." +cmd = "mypy --follow-imports skip --ignore-missing-imports --check-untyped-defs src" [tool.poe.tasks.lint] help = "Run pre-commit hooks on all files." diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c344d900..14de2cc4 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -563,9 +563,10 @@ async def get_headers(self, include_auth: bool) -> dict: headers = dict(CONTENT_TYPE_HEADERS) | {CLIENT_VERSION_HEADER: client_version} if include_auth: - if self.access_token is None: - await self.get_access_token() - headers |= {"Authorization": self.access_token} + access_token = self.access_token + if access_token is None: + access_token = await self.get_access_token() + headers |= {"Authorization": access_token} return headers def build_url(self, uri: str, path: str = path) -> URL: @@ -577,8 +578,8 @@ def build_url(self, uri: str, path: str = path) -> URL: ) return url - async def get_access_token(self): - """Get access token and store it on the FlexMeasuresClient.""" + async def get_access_token(self) -> str: + """Get access token, store it on the FlexMeasuresClient, and return it.""" response, _status = await self.request( uri="requestAuthToken", path="/api/", @@ -588,7 +589,13 @@ async def get_access_token(self): }, include_auth=False, ) - self.access_token = response["auth_token"] + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from requestAuthToken, but got a {type(response).__name__}." + ) + access_token: str = response["auth_token"] + self.access_token = access_token + return access_token async def get_versions(self): """Get the FlexMeasures version and the supported API versions of the FlexMeasures server. @@ -601,6 +608,10 @@ async def get_versions(self): method="GET", include_auth=False, ) + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from the API root, but got a {type(response).__name__}." + ) # Check whether the API version used by the client is supported by the server server_api_versions = response["versions"] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 29a1e4f7..45245d89 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -310,7 +310,11 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - async def trigger_schedule(self): + async def trigger_schedule( + self, + system_description_id: str | None = None, + request_number: int | None = None, + ): """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager """ diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a51a14be..a5ccea7c 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -75,7 +75,7 @@ def unevenly_ts_to_evenly( start: pd.Timestamp, values: list[float], durations: list[pd.Timedelta], - target_resolution: str, + target_resolution: str | timedelta, strategy="mean", ) -> pd.Series: """ @@ -132,7 +132,7 @@ def unevenly_ts_to_evenly( def translate_usage_forecast_to_fm( usage_forecast: FRBCUsageForecast, - resolution: str = "1h", + resolution: str | timedelta = "1h", strategy: str = "mean", fill_level_scale: float = 1, ) -> pd.Series: @@ -166,7 +166,7 @@ def translate_usage_forecast_to_fm( def translate_fill_level_target_profile( fill_level_target_profile: FRBCFillLevelTargetProfile, - resolution: str = "1h", + resolution: str | timedelta = "1h", fill_level_scale: float = 1, ) -> tuple[pd.Series, pd.Series]: """ diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 354adb18..108a475b 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime, timedelta +from typing import Any from urllib.parse import urlparse from zoneinfo import ZoneInfo @@ -133,6 +134,10 @@ async def configure_site( ) -> tuple[dict, dict, dict, dict]: """Create or update the site asset and its sensors, and seed it with demo prices.""" account = await fm_client.get_account() + if account is None: + raise ValueError( + "The demo user belongs to no organisation, so there is nowhere to create the site." + ) assets = await fm_client.get_assets(parse_json_fields=True) site_asset = None @@ -141,7 +146,7 @@ async def configure_site( site_asset = asset break - site_asset_specs = dict( + site_asset_specs: dict[str, Any] = dict( latitude=0, longitude=0, generic_asset_type_id=6, # Building asset type diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 0647b16e..138f9809 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -957,3 +957,49 @@ async def test_server_version_at_least_looks_the_version_up() -> None: assert await flexmeasures_client.server_version_at_least("0.27.0") is True assert await flexmeasures_client.server_version_at_least("1.1.0") is False await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_an_auth_token_answer_that_is_not_an_object_is_refused() -> None: + """A list where the token should be says what came back, instead of failing on the lookup.""" + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload=["auth_token"], + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + with pytest.raises(ValueError, match="requestAuthToken, but got a list"): + await flexmeasures_client.get_access_token() + assert flexmeasures_client.access_token is None + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_the_access_token_is_returned_as_well_as_stored() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + assert await flexmeasures_client.get_access_token() == "a-token" + assert flexmeasures_client.access_token == "a-token" + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_version_answer_that_is_not_an_object_is_refused() -> None: + with aioresponses() as m: + m.get("http://localhost:5000/api/", status=200, payload=["v3_0"]) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ValueError, match="API root, but got a list"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() From ff1304c95985da8089010e94f4d62396bfe2128d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:12:32 +0100 Subject: [PATCH 03/87] ci: measure the coverage of the S2 tests None of the S2 integration had a coverage number, for two reasons that each hid the other. The S2 tests run in a CI job of their own, since they need the s2 extra, and that job never asked for coverage. And .coveragerc omitted src/flexmeasures_client/s2/* outright, since #183, so asking for it would have reported nothing about S2 while looking as though it did. The omission is gone, and the S2 job now reports and uploads its own partial report, which Coveralls merges with the other job's in the finalize step, as it already does across the test matrix. The poe task that runs the S2 tests locally collects it as well. The job that skips tests/s2 will now list the S2 files as unexecuted in its own report; the merged figure is the one that counts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .coveragerc | 3 --- .github/workflows/ci.yml | 14 +++++++++++++- pyproject.toml | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5d068fcb..9cda9250 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,9 +2,6 @@ [run] branch = True source = flexmeasures_client -omit = - src/flexmeasures_client/s2/* - */site-packages/flexmeasures_client/s2/* [paths] source = diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb624446..188fa1bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,19 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra s2 --group test - name: Run S2 tests - run: uv run pytest tests/s2 -rFEx --durations 10 --color yes + run: uv run pytest tests/s2 -rFEx --durations 10 --color yes --cov + - name: Show coverage report + run: uv run coverage report + - name: Generate coverage report + run: uv run coverage lcov -o coverage.lcov + - name: Upload partial coverage report + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + path-to-lcov: coverage.lcov + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: s2 - py${{ matrix.python }} + parallel: true finalize: needs: [ test, test-s2] diff --git a/pyproject.toml b/pyproject.toml index 33c29c8b..8f1552e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,8 @@ help = "Run the test suite (excluding S2) and generate coverage report." cmd = "pytest --ignore=tests/s2 --cov --cov-report=" [tool.poe.tasks.test-s2] -help = "Run S2 tests." -cmd = "pytest tests/s2" +help = "Run S2 tests and generate coverage report." +cmd = "pytest tests/s2 --cov --cov-report=" [tool.poe.tasks.type-check] help = "Run mypy on every source file, including the bodies of functions without type hints." From 372a39a827bc5249237b01ebf4b273818d3e37d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:13:53 +0100 Subject: [PATCH 04/87] chore(lint): no print() in the S2 code that runs unattended A CEM's handlers run for days with nobody watching standard output, so anything they print is lost to a deployment, which keeps the log instead. FRBCSimple gave up on a schedule that way when no storage status had arrived, which is the very failure, a storage left without an instruction, that the handler otherwise goes to some length to say out loud. It now logs a warning. A pygrep hook keeps it that way for s2/control_types and s2/cem.py. The example scripts print on purpose, as do the client's interactive delete confirmations, which answer an input() prompt on the console, so the hook leaves those alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 8 +++++++ .../s2/control_types/FRBC/frbc_simple.py | 4 +++- tests/s2/test_frbc_simple_planning.py | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09dd0ef1..536c9366 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,3 +66,11 @@ repos: Put screenshots in the FlexMeasures/screenshots repo, or leave them untracked and attach them to the PR/issue manually. files: '\.(png|jpe?g|gif|bmp|tiff?|webp|ico|psd)$' + - id: no-print-in-unattended-code + name: No print() in code that runs unattended + # A CEM's handlers run for days without anyone watching standard output, + # so what they have to say goes to the log, where a deployment keeps it. + # The example scripts and the client's interactive confirmations print on purpose, and are left alone. + language: pygrep + entry: '^\s*print\(' + files: '^src/flexmeasures_client/s2/(control_types/|cem\.py$)' diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 351f8f97..720854e3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -726,7 +726,9 @@ async def trigger_schedule( -1 ].present_fill_level else: - print("Can't trigger schedule without knowing the status of the storage...") + self._logger.warning( + "Cannot ask for a schedule: no FRBC.StorageStatus has arrived yet, so there is no state of charge to plan from." + ) return # Scaled at the FlexMeasures edge, like every bound and level below it. diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index dab9231c..5d5e5c77 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -195,3 +195,25 @@ async def test_the_schedule_is_asked_for_from_an_aligned_start(): start = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs["start"] assert start == datetime(2022, 12, 3, 9, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_planning_without_a_storage_status_is_logged_not_printed(capsys, caplog): + """A handler runs unattended, so giving up on a schedule goes to the log, where it can be found. + + It used to go to standard output, which a deployment does not keep. + """ + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + system_description_id = list(frbc._system_description_history)[-1] + + with caplog.at_level("WARNING", logger="FRBCSimple"): + await frbc.trigger_schedule(system_description_id) + + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + assert capsys.readouterr().out == "" + assert any( + record.levelname == "WARNING" and "FRBC.StorageStatus" in record.getMessage() + for record in caplog.records + ), caplog.records From 9ae5a5cd7e7a053dbaaff9300828ebf694630466 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:18:24 +0100 Subject: [PATCH 05/87] docs(changelog): FillRateBasedControlTUNES records a leakage behaviour Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 502eea76..1bcc43a4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,6 +41,7 @@ Changed Fixed ----- +* ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] * ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] From f791bdbb8f2fb5673e9a50826a21c0842e0b4455 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:38:47 +0100 Subject: [PATCH 06/87] chore: close the gaps an adversarial review found in the instruments - The print ban now matches a print anywhere on a line and with a space before its parenthesis, plus pprint and writes to stdout or stderr, and covers all of s2/ but the example scripts. It only caught an indented `print(` at the start of a line before, in two directories, and missed s2/__init__.py, utils.py and wrapper.py, which run inside the CEM as much as the handlers do. - The example scripts get an __init__.py. Without one, coverage never listed websockets_server.py at all, since no test imports it, so the S2 report left it out instead of showing it at 0%. Running them by path is unaffected. - The TUNES leakage test now pins what is posted, its value and its unit, rather than only that something reached the sensor. - A client without a token now has a test that it authorises with the one it fetches. - FillRateBasedControlTUNES.trigger_schedule is back to taking no arguments, with the override mismatch marked for mypy instead. Accepting the base class's arguments and ignoring them would have turned a caller's mistake, such as expecting its superseded-request protection, into a silent one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 9 ++++++--- .../s2/control_types/FRBC/frbc_tunes.py | 9 ++++----- src/flexmeasures_client/s2/script/__init__.py | 1 + tests/client/test_init.py | 16 ++++++++++++++++ tests/s2/test_frbc_tunes.py | 19 ++++++++++++++++--- 5 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/flexmeasures_client/s2/script/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 536c9366..a53e82e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -70,7 +70,10 @@ repos: name: No print() in code that runs unattended # A CEM's handlers run for days without anyone watching standard output, # so what they have to say goes to the log, where a deployment keeps it. - # The example scripts and the client's interactive confirmations print on purpose, and are left alone. + # The example scripts print on purpose, and so do the client's interactive confirmations, which live outside s2/. + # Matched anywhere on a line, not only at its start, and with a space before the parenthesis too, + # so that `if x: print(...)`, `return print(...)`, `print (...)`, pprint and writes to stdout or stderr are caught. language: pygrep - entry: '^\s*print\(' - files: '^src/flexmeasures_client/s2/(control_types/|cem\.py$)' + entry: '\b(print|pprint)\s*\(|sys\.std(out|err)\.write' + files: '^src/flexmeasures_client/s2/.*\.py$' + exclude: '^src/flexmeasures_client/s2/script/' diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 45245d89..4dae13ca 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -310,11 +310,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - async def trigger_schedule( - self, - system_description_id: str | None = None, - request_number: int | None = None, - ): + # It takes neither argument the base class declares: TUNES plans from its latest description, + # and does not take part in dropping superseded requests. + # Accepting them and ignoring them would turn a caller's mistake into a silent one, so the mismatch stays. + async def trigger_schedule(self): # type: ignore[override] """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager """ diff --git a/src/flexmeasures_client/s2/script/__init__.py b/src/flexmeasures_client/s2/script/__init__.py new file mode 100644 index 00000000..c777fdcf --- /dev/null +++ b/src/flexmeasures_client/s2/script/__init__.py @@ -0,0 +1 @@ +"""Example scripts: a CEM server and a resource manager that talk S2 to each other, run by path as the CEM documentation shows.""" diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 138f9809..52a02af7 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -1003,3 +1003,19 @@ async def test_a_version_answer_that_is_not_an_object_is_refused() -> None: with pytest.raises(ValueError, match="API root, but got a list"): await flexmeasures_client.get_versions() await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_client_without_a_token_authorises_with_the_one_it_fetches() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + headers = await flexmeasures_client.get_headers(include_auth=True) + assert headers["Authorization"] == "a-token" + await flexmeasures_client.close() diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index cd43fb9b..4086b25b 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -324,8 +324,21 @@ async def test_a_leakage_behaviour_is_recorded_rather_than_refused( getattr(message, "status", None) == ReceptionStatusValues.PERMANENT_ERROR for message in sent ), sent - posted_to = [ - call.args[0] if call.args else call.kwargs.get("sensor_id") + posts = [ + call for call in fm_client.post_sensor_data.await_args_list + if (call.args[0] if call.args else call.kwargs.get("sensor_id")) == 15 ] - assert 15 in posted_to, posted_to + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + # What the translation makes of this message on a quarter-hour grid, which is what TUNES says it posts. + # The fill-level scale is not exercised here: it multiplies both the leakage and the range it is a share of, and cancels. + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + assert posts[0].kwargs["values"] == [ + leakage_behaviour_to_storage_efficiency( + message=leakage, resolution=timedelta(minutes=15), fill_level_scale=1 + ) + ] + assert posts[0].kwargs["unit"] == "%" From 5ff631deafb2457b6c7a1ff51aeb04271056af8f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:32:45 +0100 Subject: [PATCH 07/87] fix(FRBC): answer by the element that applies, whatever order S2 lists them in S2 lets the elements of an operation mode, and of a leakage behaviour, cover the fill-level range in pieces, and s2python checks that they are contiguous on a sorted copy, leaving the list in whatever order it was sent. Four places took an element by its position instead, and answered three different ways: - the measured power of an actuator status took elements[0], so a heat pump that derates as its buffer warms was recorded at its cold power when warm: 2000 W where it draws at most 1200 W; - the charging efficiency took elements[-1] as though it were the top of the range, so a mode listed high element first planned against the wrong ratio; - the storage efficiency took the leakage rate of elements[-1], on the same assumption, though the leakiest element need not be the top one; - the schedule translation selected by fill level, but settled a shared boundary by list order. element_for_fill_level now answers wherever a fill level is known: the element whose range holds it, the upper of two at a shared boundary, and past either end of the range the element that applies at that end, so the answer does not jump there. The measured power and the schedule translation use it. Before any storage status there is no fill level, so the measured power takes the bottom of the range and says in the log that it guessed. Where FlexMeasures takes one number for a whole schedule, it does not know which fill levels the schedule will pass through, so the most cautious element answers, which is the rule #253 already applies across modes: the lowest conversion efficiency, and the leakiest element. That is also independent of order. The efficiency now also skips an element that draws power while the storage does not fill, such as a standby or circulation mode. Before, such a mode gave a ratio of zero or below, which then won the minimum across modes, went onto the charging-efficiency sensor, and told the scheduler that no power could fill the storage at all; with the fill-level band priced rather than hard, never running the heat pump was then a feasible plan. Two guards were dropped rather than tested, because S2 makes them unreachable: every operation mode and leakage behaviour has at least one element, and every element at least one power range. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 21 +- .../s2/control_types/FRBC/utils.py | 100 +++-- .../s2/control_types/translations.py | 8 +- tests/s2/test_frbc_element_selection.py | 415 ++++++++++++++++++ 4 files changed, 506 insertions(+), 38 deletions(-) create mode 100644 tests/s2/test_frbc_element_selection.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 720854e3..a62f3c8e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -30,8 +30,10 @@ from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( clip_fill_level_target_profile, + element_for_fill_level, fm_schedule_to_instructions, get_soc_min_max, + om_label, operation_mode_conversion_efficiency, ) from flexmeasures_client.s2.control_types.translations import ( @@ -426,7 +428,24 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): status.actuator_id, ) operation_mode = actuator.operation_modes[0] - element = operation_mode.elements[0] + + # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, + # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. + if self._storage_status_history: + fill_level = list(self._storage_status_history.values())[ + -1 + ].present_fill_level + element = element_for_fill_level(operation_mode.elements, fill_level) + else: + # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. + element = element_for_fill_level(operation_mode.elements, float("-inf")) + if len(operation_mode.elements) > 1: + self._logger.warning( + "FRBC.ActuatorStatus for operation mode %s arrived with no FRBC.StorageStatus yet, " + "so its power is recorded as the element at the bottom of the fill-level range states it, " + "which need not be the element the actuator runs on.", + om_label(operation_mode), + ) # Interpolate the element's POWER range, not its fill rate. A fill rate is the thermal # output a heat pump delivers into its buffer; the power range is what it draws diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 4acd8d8c..d6f78386 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,7 +1,7 @@ import logging from datetime import timedelta from math import isclose -from typing import List +from typing import List, Sequence, TypeVar import numpy as np import pandas as pd @@ -11,6 +11,7 @@ from s2python.frbc import ( FRBCInstruction, FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCSystemDescription, ) @@ -327,32 +328,38 @@ def operation_mode_conversion_efficiency( Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, and for a thermal actuator that ratio is well above one. - The ratio is taken at the top of the last element's range, rather than as the slope over that range. + Each element's ratio is taken at the top of its range, rather than as the slope over that range. An actuator may well declare degenerate ranges, where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. - Where an efficiency varies over a mode's range, this is the one it reaches at full tilt. + + A mode whose elements convert differently, as a heat pump does when its buffer warms, states the lowest of them. + FlexMeasures takes one charging efficiency for a whole schedule, and does not know which fill levels it will pass through, + so the most cautious element answers, for the same reason the lowest mode answers across modes: + a plan that converts less than the actuator really does reaches its target early, where one that converts more reaches it late. + That also makes the answer independent of the order the elements were listed in, which S2 does not fix. :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :returns: the efficiency, or None where the mode states no power to convert. - An idle mode draws nothing, and a mode that only feeds power back is discharging, - which FlexMeasures takes as a separate discharging efficiency that this does not state. + :returns: the efficiency, or None where no element of the mode converts drawn power into stored energy. + An element converts nothing where it draws no power, as an idle mode does, + where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, + or where it draws power while the storage does not fill, as a standby or circulation mode does: + no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. """ - if not operation_mode.elements: - return None - element = operation_mode.elements[-1] - if not element.power_ranges: - return None - # todo: an element states one power range per commodity quantity, - # and no distinction is made between them here, - # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). - power = element.power_ranges[0].end_of_range - if power <= 0: - return None - fill_rate = element.fill_rate.end_of_range * fill_level_scale - return fill_rate_in_watts(fill_rate, energy_unit) / power + efficiencies = [] + # S2 gives every operation mode at least one element, and every element at least one power range. + for element in operation_mode.elements: + # todo: an element states one power range per commodity quantity, + # and no distinction is made between them here, + # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). + power = element.power_ranges[0].end_of_range + fill_rate = element.fill_rate.end_of_range * fill_level_scale + if power <= 0 or fill_rate <= 0: + continue + efficiencies.append(fill_rate_in_watts(fill_rate, energy_unit) / power) + return min(efficiencies) if efficiencies else None def clip_fill_level_target_profile( @@ -408,22 +415,12 @@ def power_to_fill_rate_with_metrics( efficiency: float element: FRBCOperationModeElement that produced fill_rate """ - elements = operation_mode.elements - - # --- 1. Select element with minimal fill-level penalty --- - best_element, fill_penalty = min( - ( - ( - e, - clamp_distance( - fill_level, - e.fill_level_range.start_of_range, - e.fill_level_range.end_of_range, - ), - ) - for e in elements - ), - key=lambda x: x[1], + # --- 1. Select the element that applies at this fill level --- + best_element = element_for_fill_level(operation_mode.elements, fill_level) + fill_penalty = clamp_distance( + fill_level, + best_element.fill_level_range.start_of_range, + best_element.fill_level_range.end_of_range, ) # --- 2. Select power range with minimal power penalty --- @@ -465,6 +462,39 @@ def clamp_distance(value: float, start: float, end: float) -> float: return 0.0 +ElementWithFillLevelRange = TypeVar( + "ElementWithFillLevelRange", FRBCOperationModeElement, FRBCLeakageBehaviourElement +) + + +def element_for_fill_level( + elements: Sequence[ElementWithFillLevelRange], fill_level: float +) -> ElementWithFillLevelRange: + """The element that applies at a fill level, whatever order the elements were listed in. + + S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, + and s2python checks that they are contiguous on a sorted copy, so the list itself may arrive in any order. + Two neighbouring elements share a boundary, which belongs to the upper of the two: + of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. + That includes an element whose range is a single point at the top, which no half-open rule could ever select. + A fill level outside the range the elements cover is answered as the nearest end of that range is, + since a storage may be outside its declared range, and the answer should not jump as it crosses that end. + Elements with a gap between them, which S2 rules out, answer a fill level in the gap with the nearer, and on a tie the lower. + """ + def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: + return e.fill_level_range.start_of_range, e.fill_level_range.end_of_range + + covered_from = min(bounds(e)[0] for e in elements) + covered_to = max(bounds(e)[1] for e in elements) + clamped = min(max(fill_level, covered_from), covered_to) + holding = [e for e in elements if bounds(e)[0] <= clamped <= bounds(e)[1]] + if holding: + return max(holding, key=bounds) + return min( + elements, key=lambda e: (clamp_distance(clamped, *bounds(e)), *bounds(e)) + ) + + def om_label(operation_mode: FRBCOperationMode) -> str: label = getattr(operation_mode, "diagnostic_label", None) return label if isinstance(label, str) else operation_mode.id diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a5ccea7c..8b02b15e 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -56,11 +56,15 @@ def leakage_behaviour_to_storage_efficiency( """ - last_element = message.elements[-1] + # FlexMeasures takes one storage efficiency here, and does not know which fill levels a schedule will pass through, + # so the leakiest element answers, as the most cautious: a plan that expects to lose more reaches its target early. + # It is looked for rather than assumed to come last, since S2 does not fix the order the elements are listed in, + # and the leakiest is not always the top one. + leakiest = max(message.elements, key=lambda e: e.leakage_rate) max_fill_level = max( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) - leakage = last_element.leakage_rate * fill_level_scale + leakage = leakiest.leakage_rate * fill_level_scale base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) # Discuss conversions diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py new file mode 100644 index 00000000..426b7d10 --- /dev/null +++ b/tests/s2/test_frbc_element_selection.py @@ -0,0 +1,415 @@ +"""Which element of an operation mode, or of a leakage behaviour, answers a question. + +S2 lets those elements cover the fill-level range in pieces, and lets them arrive in any order: +s2python's own contiguity check sorts a copy of them and leaves the list as it was sent. +So no answer may depend on the order they are listed in. + +Where a fill level is known, the element that applies at it answers, +and a boundary two elements share belongs to the upper one. +Where FlexMeasures takes one number for a whole schedule, the most cautious element answers: +the one converting drawn power into stored energy least, or the one leaking most. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest +from s2python.common import Commodity, CommodityQuantity, NumberRange, PowerRange +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, +) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + element_for_fill_level, + operation_mode_conversion_efficiency, + power_to_fill_rate_with_metrics, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +POWER_SENSOR = 1 +CHARGING_EFFICIENCY_SENSOR = 7 +RANGE_TOP = 80.0 +KWH_PER_SECOND_IN_WATTS = 3.6e6 + + +def _element( + lowest: float, + highest: float, + power: float, + heat_in_watts: float, +) -> FRBCOperationModeElement: + """A stretch of fill level over which a mode runs from nothing up to a power, delivering up to some heat.""" + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=0, end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS + ), + power_ranges=[ + PowerRange( + start_of_range=0, + end_of_range=power, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +#: A heat pump that derates as its buffer warms: 2000 W at a COP of 4 below 40, 1200 W at a COP of 3 above. +COLD = _element(0, 40, power=2000.0, heat_in_watts=8000.0) +WARM = _element(40, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + + +def _mode(*elements: FRBCOperationModeElement, label: str = "heat") -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _range(element) -> tuple[float, float]: + return ( + element.fill_level_range.start_of_range, + element.fill_level_range.end_of_range, + ) + + +# --------------------------------------------------------------------------- +# The selector itself +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fill_level", [-5.0, 0.0, 20.0, 40.0, 60.0, 80.0, 95.0]) +def test_the_element_for_a_fill_level_does_not_depend_on_the_order_listed(fill_level): + assert _range(element_for_fill_level([COLD, WARM], fill_level)) == _range( + element_for_fill_level([WARM, COLD], fill_level) + ) + + +@pytest.mark.parametrize( + "fill_level, expected", + [ + (0.0, (0, 40)), + (39.9, (0, 40)), + # A boundary two elements share belongs to the upper one, + (40.0, (40, RANGE_TOP)), + # except at the very top, where there is no upper one. + (RANGE_TOP, (40, RANGE_TOP)), + # Outside the range, the nearest element answers: a storage may be outside its declared range. + (-5.0, (0, 40)), + (95.0, (40, RANGE_TOP)), + ], +) +def test_the_element_that_applies_at_a_fill_level(fill_level, expected): + for listed in ([COLD, WARM], [WARM, COLD]): + assert _range(element_for_fill_level(listed, fill_level)) == expected + + +def test_an_element_that_is_a_single_point_applies_at_that_point(): + """S2 accepts an element whose range is one fill level, as long as the elements stay contiguous. + + At the top of a buffer, a heat pump may run at its minimum only, and say so with such an element. + It is the upper of the two at their shared boundary, so it applies there. + """ + below = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ([below, at_the_top], [at_the_top, below]): + assert element_for_fill_level(listed, RANGE_TOP) is at_the_top + assert element_for_fill_level(listed, RANGE_TOP - 1) is below + + +def test_past_either_end_the_answer_is_that_at_the_end(): + """A storage outside its declared range is answered as at the nearest end, so the answer does not jump there.""" + at_the_bottom = _element(0, 0, power=0.0, heat_in_watts=0.0) + middle = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ( + [at_the_bottom, middle, at_the_top], + [at_the_top, middle, at_the_bottom], + ): + assert element_for_fill_level(listed, RANGE_TOP + 15) is at_the_top + assert element_for_fill_level(listed, -5.0) is element_for_fill_level( + listed, 0.0 + ) + # At the bottom, the point element shares its only fill level with the element above it, which is upper. + assert element_for_fill_level(listed, 0.0) is middle + + +def test_a_fill_level_in_a_gap_is_answered_by_the_nearer_element_and_on_a_tie_the_lower(): + """S2 rules gaps out, but the selector takes raw elements and should not depend on their order there either.""" + low = _element(0, 30, power=2000.0, heat_in_watts=8000.0) + high = _element(50, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + for listed in ([low, high], [high, low]): + assert element_for_fill_level(listed, 35.0) is low + assert element_for_fill_level(listed, 45.0) is high + assert element_for_fill_level(listed, 40.0) is low + + +def test_leakage_elements_are_selected_the_same_way(): + low = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=1e-5 + ) + high = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=RANGE_TOP), + leakage_rate=3e-5, + ) + assert element_for_fill_level([high, low], 10.0) is low + assert element_for_fill_level([low, high], 40.0) is high + + +# --------------------------------------------------------------------------- +# Where a fill level is known: the measured power, and the schedule translation +# --------------------------------------------------------------------------- + + +def _system_description(*modes: FRBCOperationMode) -> FRBCSystemDescription: + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=list(modes), + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=False, + provides_fill_level_target_profile=False, + provides_usage_forecast=False, + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + ), + ) + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=POWER_SENSOR, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + # What the power sensor means by a sign is not the question here. + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +async def _measured_power(frbc: FRBCSimple, mode: FRBCOperationMode, factor: float): + actuator = list(frbc._system_description_history.values())[-1].actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode.id, + operation_mode_factor=factor, + ) + ) + posts = [ + call + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] == POWER_SENSOR + ] + assert len(posts) == 1, posts + return posts[0].kwargs["values"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("listed", [(COLD, WARM), (WARM, COLD)]) +async def test_the_measured_power_is_that_of_the_element_at_the_present_fill_level( + listed, +): + """A warm buffer runs on the derated element, so full tilt there is 1200 W, not 2000 W.""" + mode = _mode(*listed) + frbc = _frbc(_system_description(mode)) + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=70.0 + ) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_without_a_fill_level_the_measured_power_is_a_guess_said_out_loud(caplog): + """Before any storage status, the bottom element answers, and the log says it was a guess.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any( + "no FRBC.StorageStatus" in record.getMessage() for record in caplog.records + ), caplog.records + + +@pytest.mark.parametrize("fill_level", [20.0, 40.0, 60.0]) +def test_a_schedule_is_translated_the_same_whichever_order_the_elements_come_in( + fill_level, +): + forward = power_to_fill_rate_with_metrics(_mode(COLD, WARM), 1000.0, fill_level) + backward = power_to_fill_rate_with_metrics(_mode(WARM, COLD), 1000.0, fill_level) + assert forward[:4] == pytest.approx(backward[:4]) + assert _range(forward[4]) == _range(backward[4]) + + +# --------------------------------------------------------------------------- +# Where one number covers a whole schedule: the charging and the storage efficiency +# --------------------------------------------------------------------------- + + +def _efficiency(mode: FRBCOperationMode) -> float | None: + return operation_mode_conversion_efficiency( + mode, fill_level_scale=1, energy_unit="kWh" + ) + + +def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(): + assert _efficiency(_mode(COLD, WARM)) == pytest.approx( + _efficiency(_mode(WARM, COLD)) + ) + + +def test_a_mode_whose_elements_convert_differently_states_the_lowest(): + """A COP of 4 when cold and 3 when warm plans as 3: a plan that converts less reaches its target early.""" + assert _efficiency(_mode(COLD, WARM)) == pytest.approx(3.0) + + +@pytest.mark.parametrize( + "heat_in_watts", + [0.0, -50.0], + ids=["standby, drawing power and delivering nothing", "circulation, drawing power and losing heat"], +) +def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_watts): + """Drawing power while the storage does not fill is not a conversion one ratio can state.""" + standby = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange( + start_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + ), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + assert _efficiency(_mode(standby)) is None + + +@pytest.mark.asyncio +async def test_a_standby_mode_does_not_drag_the_recorded_efficiency_to_zero(): + """Off, standby at 5 W, and on at a COP of 4.3: the scheduler is told 4.3, not 0. + + At 0, the stock could not rise at any power, so the cheapest plan would be never to run the heat pump, + and with the fill-level band priced rather than hard, that plan would be feasible rather than refused. + """ + off = _mode(_element(0, RANGE_TOP, power=0.0, heat_in_watts=0.0), label="off") + standby_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange(start_of_range=0, end_of_range=0), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + standby = _mode(standby_element, label="standby") + on = _mode(_element(0, RANGE_TOP, power=883.7, heat_in_watts=3800.0), label="on") + system_description = _system_description(off, standby, on) + frbc = _frbc( + system_description, charging_efficiency_sensor_id=CHARGING_EFFICIENCY_SENSOR + ) + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + assert await frbc._record_charging_efficiency(system_description, start) is True + recorded = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] + == CHARGING_EFFICIENCY_SENSOR + ] + assert recorded == [[pytest.approx(3800.0 / 883.7)]] + + +def _leakage(*elements: FRBCLeakageBehaviourElement) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=list(elements), + ) + + +LEAKS_LITTLE = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=3e-5 +) +LEAKS_MOST = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=60), leakage_rate=9e-5 +) +LEAKS_SOME = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=60, end_of_range=RANGE_TOP), + leakage_rate=6e-5, +) + + +# The storage efficiencies below are compared exactly, not approximately. +# They are the same arithmetic on the same numbers, so equal means equal, +# and a leakage rate per second becomes an efficiency so close to 100% that pytest.approx's default tolerance +# would call two different leakage rates the same, and pass with the defect these tests are about. + + +def test_the_storage_efficiency_does_not_depend_on_the_order_listed(): + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), resolution=timedelta(minutes=15) + ) + assert efficiency == leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_SOME, LEAKS_LITTLE, LEAKS_MOST), + resolution=timedelta(minutes=15), + ) + + +def test_the_storage_efficiency_states_the_leakiest_element(): + """The leakiest element is not always the top one, so it is looked for rather than assumed to come last.""" + leakiest = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_MOST), resolution=timedelta(minutes=15), fill_level_scale=1 + ) + # Alone, the leakiest element is measured against its own top; together, against the range's top. + together = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), + resolution=timedelta(minutes=15), + ) + top_only = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_MOST.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert together == top_only + assert leakiest != top_only From d0b1c1dc33612a8af650298900da1259af0050c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:44:03 +0100 Subject: [PATCH 08/87] docs(changelog): the element that applies, and what a standby mode states Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 1bcc43a4..4c06c478 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -19,7 +19,7 @@ Added * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] * ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged [`#253 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes, or the elements of one mode, state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] @@ -41,6 +41,7 @@ Changed Fixed ----- +* A leakage behaviour is recorded as a storage efficiency at the rate of its leakiest element, where it took the last one listed, which S2 does not require to be the leakiest or even the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] @@ -64,7 +65,7 @@ Fixed * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] * The instruction-status history is keyed by a string, as its type declares, instead of by the ``UUID`` that a no-op ``cast`` let through, which made every lookup miss [`#237 `__] -* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the mode's power range, and posts it to the power sensor [`#236 `__] +* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the power range of the element that applies at the latest fill level, and posts it to the power sensor [`#236 `__, `#256 `__] * A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] Added From b572a86f0c6a806a2c8661b1adee781c53d6ee0a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:07:49 +0100 Subject: [PATCH 09/87] fix(FRBC): plan with the element where the schedule starts, not the lowest An adversarial review showed that the lowest efficiency over a mode's elements is not the cautious choice the previous commit called it. In FlexMeasures' linear program a charging efficiency scales what the actuator adds to the stock, so understating it protects the lower band only: the plan overshoots the upper band, buys more than it uses, and against a usage forecast the plan cannot relax, it can turn a problem the actuator can meet into one the scheduler finds infeasible. A heat pump at a COP of 4 cold and 2 warm, planned at 2, empties its planned buffer in a few hours while the real one keeps up. So within a mode, the element that applies at the fill level the schedule starts from answers. That is also the part of a schedule that runs before the next storage status asks for a new one, and it is what the actuator really does there. The storage efficiency likewise takes the leakage element at the latest fill level, and the leakiest only where no fill level is known yet. Across modes, the lowest still goes out, as #253 decided; the same argument applies to it, and that is left as a question rather than changed here. Also from the review: - A fill level that is not a number, which s2python lets through, counts as no fill level, and the selector refuses one, where it used to pick the bottom element silently. - The selector's docstring now says what is true: s2python checks contiguity for an operation mode only, never for a leakage behaviour; a single-point range, which the standard rules out, applies only at the top; and of two elements with the same range, the first listed answers. - Tests now cover the latest of several storage statuses, a fill-level scale other than 1 where scaling would pick another element, no warning for a mode with one element, and a schedule starting in the upper element. Each was a production mutation that passed the suite before. Formatted with black as well, which the previous commit was not: the pre-commit hooks were never installed in this repository, so they did not run on commit, and CI's check caught it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- .../s2/control_types/FRBC/frbc_simple.py | 30 +++- .../s2/control_types/FRBC/utils.py | 68 +++++--- .../s2/control_types/translations.py | 18 +- tests/s2/test_frbc_charging_efficiency.py | 49 +++++- tests/s2/test_frbc_element_selection.py | 154 ++++++++++++++++-- 6 files changed, 262 insertions(+), 61 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 4c06c478..c90d1871 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -19,7 +19,7 @@ Added * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] * ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes, or the elements of one mode, state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] @@ -41,7 +41,7 @@ Changed Fixed ----- -* A leakage behaviour is recorded as a storage efficiency at the rate of its leakiest element, where it took the last one listed, which S2 does not require to be the leakiest or even the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] +* A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a62f3c8e..c29ad86e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -4,6 +4,7 @@ """ import asyncio +import math from datetime import datetime, timedelta from zoneinfo import ZoneInfo @@ -395,6 +396,13 @@ def _in_the_power_unit(self, power: float) -> float: return power return convert_units([power], S2_POWER_UNIT, self.power_unit)[0] + def _latest_fill_level(self) -> float | None: + """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" + if not self._storage_status_history: + return None + fill_level = list(self._storage_status_history.values())[-1].present_fill_level + return None if math.isnan(fill_level) else fill_level + async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor sd: FRBCSystemDescription = list(self._system_description_history.values())[-1] @@ -431,17 +439,15 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. - if self._storage_status_history: - fill_level = list(self._storage_status_history.values())[ - -1 - ].present_fill_level + fill_level = self._latest_fill_level() + if fill_level is not None: element = element_for_fill_level(operation_mode.elements, fill_level) else: # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. element = element_for_fill_level(operation_mode.elements, float("-inf")) if len(operation_mode.elements) > 1: self._logger.warning( - "FRBC.ActuatorStatus for operation mode %s arrived with no FRBC.StorageStatus yet, " + "FRBC.ActuatorStatus for operation mode %s arrived with no usable FRBC.StorageStatus yet, " "so its power is recorded as the element at the bottom of the fill-level range states it, " "which need not be the element the actuator runs on.", om_label(operation_mode), @@ -534,7 +540,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): ) async def _record_charging_efficiency( - self, system_description: FRBCSystemDescription, start: datetime + self, + system_description: FRBCSystemDescription, + start: datetime, + fill_level: float, ) -> bool: """Record what the actuator converts drawn power into stored energy at, over the window a schedule covers. @@ -544,6 +553,8 @@ async def _record_charging_efficiency( a schedule planned against a sensor holding nothing for its window is planned as if the actuator converted power one for one. :param start: where the schedule starts, which is where the value is recorded from. + :param fill_level: the fill level the schedule starts from, in the resource manager's own units, + which picks the element of each mode that answers. :returns: whether that schedule may be planned against the sensor, which it may not where nothing is recorded on it for this window. """ @@ -556,7 +567,7 @@ async def _record_charging_efficiency( try: for operation_mode in actuator.operation_modes: efficiency = operation_mode_conversion_efficiency( - operation_mode, self._fill_level_scale, self.energy_unit + operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) if efficiency is not None: efficiencies[operation_mode.id] = efficiency @@ -829,7 +840,9 @@ async def trigger_schedule( # What the operation modes say about converting drawn power into stored energy. # Recorded first, since FlexMeasures reads it from the sensor named here and not from the flex model, # and named only where it was recorded. - if await self._record_charging_efficiency(system_description, start): + if await self._record_charging_efficiency( + system_description, start, fill_level_at_start + ): flex_model["charging-efficiency"] = { "sensor": self._charging_efficiency_sensor_id } @@ -1007,6 +1020,7 @@ async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): message=leakage_behaviour, resolution=self._resolution, fill_level_scale=self._fill_level_scale, + fill_level=self._latest_fill_level(), ) now = self.now() diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index d6f78386..646b36bc 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,4 +1,5 @@ import logging +import math from datetime import timedelta from math import isclose from typing import List, Sequence, TypeVar @@ -318,9 +319,12 @@ def fill_rate_in_watts(fill_rate: float, energy_unit: str) -> float: def operation_mode_conversion_efficiency( - operation_mode: FRBCOperationMode, fill_level_scale: float, energy_unit: str + operation_mode: FRBCOperationMode, + fill_level_scale: float, + energy_unit: str, + fill_level: float, ) -> float | None: - """How much energy an operation mode puts into the storage for each unit of power it draws. + """How much energy an operation mode puts into the storage for each unit of power it draws, at a fill level. An operation mode states a fill rate and a power range, so the ratio between the two is what the mode converts drawn power into stored energy at: @@ -328,38 +332,38 @@ def operation_mode_conversion_efficiency( Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, and for a thermal actuator that ratio is well above one. - Each element's ratio is taken at the top of its range, rather than as the slope over that range. + A mode may convert differently at different fill levels, as a heat pump does when its buffer warms, + and FlexMeasures takes one charging efficiency for a whole schedule. + The element that applies at the fill level the schedule starts from answers, + since the start of a schedule is the part that runs before the next storage status asks for a new one. + Not the lowest element over the whole range: + that overstates what the actuator needs to fill the storage, so the plan overshoots its upper band and buys more than it uses, + and against a usage the plan cannot relax it can turn a problem the actuator can meet into one the scheduler finds infeasible. + + The ratio is taken at the top of that element's range, rather than as the slope over it. An actuator may well declare degenerate ranges, where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. - A mode whose elements convert differently, as a heat pump does when its buffer warms, states the lowest of them. - FlexMeasures takes one charging efficiency for a whole schedule, and does not know which fill levels it will pass through, - so the most cautious element answers, for the same reason the lowest mode answers across modes: - a plan that converts less than the actuator really does reaches its target early, where one that converts more reaches it late. - That also makes the answer independent of the order the elements were listed in, which S2 does not fix. - :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :returns: the efficiency, or None where no element of the mode converts drawn power into stored energy. - An element converts nothing where it draws no power, as an idle mode does, + :param fill_level: where the schedule starts, in the resource manager's own units, as the elements' ranges are. + :returns: the efficiency, or None where the element that applies converts no drawn power into stored energy. + It converts nothing where it draws no power, as an idle mode does, where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, or where it draws power while the storage does not fill, as a standby or circulation mode does: no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. """ - efficiencies = [] - # S2 gives every operation mode at least one element, and every element at least one power range. - for element in operation_mode.elements: - # todo: an element states one power range per commodity quantity, - # and no distinction is made between them here, - # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). - power = element.power_ranges[0].end_of_range - fill_rate = element.fill_rate.end_of_range * fill_level_scale - if power <= 0 or fill_rate <= 0: - continue - efficiencies.append(fill_rate_in_watts(fill_rate, energy_unit) / power) - return min(efficiencies) if efficiencies else None + element = element_for_fill_level(operation_mode.elements, fill_level) + # todo: an element states one power range per commodity quantity, + # and no distinction is made between them here, + # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). + power = element.power_ranges[0].end_of_range + fill_rate = element.fill_rate.end_of_range * fill_level_scale + if power <= 0 or fill_rate <= 0: + return None + return fill_rate_in_watts(fill_rate, energy_unit) / power def clip_fill_level_target_profile( @@ -472,15 +476,25 @@ def element_for_fill_level( ) -> ElementWithFillLevelRange: """The element that applies at a fill level, whatever order the elements were listed in. - S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, - and s2python checks that they are contiguous on a sorted copy, so the list itself may arrive in any order. + S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, in any order. + s2python checks that an operation mode's elements are contiguous, on a sorted copy, and leaves the list as it was sent; + it checks nothing of the kind for a leakage behaviour, whose elements may leave gaps or overlap. + Two neighbouring elements share a boundary, which belongs to the upper of the two: of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. - That includes an element whose range is a single point at the top, which no half-open rule could ever select. + A range that is a single point therefore applies only at the very top. + The standard says a range starts below where it ends, so such a range is outside it, though s2python lets it through. + Two elements with the same range are ambiguous, which overlapping elements always are, and the first listed answers. + A fill level outside the range the elements cover is answered as the nearest end of that range is, since a storage may be outside its declared range, and the answer should not jump as it crosses that end. - Elements with a gap between them, which S2 rules out, answer a fill level in the gap with the nearer, and on a tie the lower. + A fill level in a gap between elements is answered by the nearer, and on a tie the lower. + + :raises ValueError: for a fill level that is not a number, which no element applies at. """ + if math.isnan(fill_level): + raise ValueError("No element applies at a fill level that is not a number.") + def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: return e.fill_level_range.start_of_range, e.fill_level_range.end_of_range diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index 8b02b15e..a4cd5ecc 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -17,11 +17,14 @@ "Install it using `pip install flexmeasures-client[s2]`." ) +from flexmeasures_client.s2.control_types.FRBC.utils import element_for_fill_level + def leakage_behaviour_to_storage_efficiency( message: FRBCLeakageBehaviour, resolution=timedelta(minutes=15), fill_level_scale: float = 1, + fill_level: float | None = None, ) -> float: """ Convert a FRBC.LeakageBehaviour message into a FlexMeasures compatible storage efficiency. @@ -56,15 +59,18 @@ def leakage_behaviour_to_storage_efficiency( """ - # FlexMeasures takes one storage efficiency here, and does not know which fill levels a schedule will pass through, - # so the leakiest element answers, as the most cautious: a plan that expects to lose more reaches its target early. - # It is looked for rather than assumed to come last, since S2 does not fix the order the elements are listed in, - # and the leakiest is not always the top one. - leakiest = max(message.elements, key=lambda e: e.leakage_rate) + # FlexMeasures takes one storage efficiency here, for the whole window it covers. + # Where the fill level is known, the element that applies at it answers, as for a charging efficiency. + # Where it is not, the leakiest answers, looked for rather than assumed to come last, + # since S2 does not fix the order the elements are listed in, and the leakiest is not always the top one. + if fill_level is not None: + element = element_for_fill_level(message.elements, fill_level) + else: + element = max(message.elements, key=lambda e: e.leakage_rate) max_fill_level = max( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) - leakage = leakiest.leakage_rate * fill_level_scale + leakage = element.leakage_rate * fill_level_scale base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) # Discuss conversions diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 34b382c6..7217e9cd 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -326,8 +326,8 @@ def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): fill_rate_in_watts(1, "therm") -def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): - """A mode states an element per stretch of fill level, and the efficiency is the one at full tilt. +def test_an_efficiency_is_taken_at_the_top_of_the_element_that_applies(): + """A mode states an element per stretch of fill level, and the efficiency is that element's, at full tilt. The ratio, and not the slope over the element's range: a mode's ranges can be degenerate, where the actuator runs at one power and one fill rate only, @@ -353,6 +353,47 @@ def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): ) ) - efficiency = operation_mode_conversion_efficiency(mode, 1, "kWh") + below, above = ( + operation_mode_conversion_efficiency(mode, 1, "kWh", fill_level) + for fill_level in (RANGE_TOP / 2, 1.5 * RANGE_TOP) + ) + + assert below == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER) + assert above == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + + +@pytest.mark.asyncio +async def test_a_schedule_is_asked_for_against_the_efficiency_where_it_starts(): + """A heat pump that converts less once its buffer is warm is planned with the warm efficiency for a warm start.""" + cold = HEATER_HEAT_IN_WATTS / HEATER_POWER + warm = cold / 2 + mode = _operation_mode(HEATER_POWER, HEATER_FILL_RATE, "on") + mode.elements[0].fill_level_range.end_of_range = RANGE_TOP / 2 + mode.elements.append( + FRBCOperationModeElement( + fill_level_range=NumberRange( + start_of_range=RANGE_TOP / 2, end_of_range=RANGE_TOP + ), + fill_rate=NumberRange( + start_of_range=HEATER_FILL_RATE / 2, end_of_range=HEATER_FILL_RATE / 2 + ), + power_ranges=[ + PowerRange( + start_of_range=HEATER_POWER, + end_of_range=HEATER_POWER, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + ) + system_description = _system_description(mode) + frbc = _frbc(system_description) + # The latest status: a buffer warmed past the middle of its range since the one _frbc recorded. + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=0.75 * RANGE_TOP + ) + + await _asked_for(frbc, system_description) - assert efficiency == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + assert _recordings(frbc)[0]["values"] == [pytest.approx(warm)] + assert warm != pytest.approx(cold) diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 426b7d10..06ef021c 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -75,7 +75,9 @@ def _element( WARM = _element(40, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) -def _mode(*elements: FRBCOperationModeElement, label: str = "heat") -> FRBCOperationMode: +def _mode( + *elements: FRBCOperationModeElement, label: str = "heat" +) -> FRBCOperationMode: return FRBCOperationMode( id=get_unique_id(), diagnostic_label=label, @@ -164,7 +166,8 @@ def test_a_fill_level_in_a_gap_is_answered_by_the_nearer_element_and_on_a_tie_th def test_leakage_elements_are_selected_the_same_way(): low = FRBCLeakageBehaviourElement( - fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=1e-5 + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), + leakage_rate=1e-5, ) high = FRBCLeakageBehaviourElement( fill_level_range=NumberRange(start_of_range=40, end_of_range=RANGE_TOP), @@ -260,7 +263,8 @@ async def test_without_a_fill_level_the_measured_power_is_a_guess_said_out_loud( with caplog.at_level(logging.WARNING, logger="FRBCSimple"): assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) assert any( - "no FRBC.StorageStatus" in record.getMessage() for record in caplog.records + "no usable FRBC.StorageStatus" in record.getMessage() + for record in caplog.records ), caplog.records @@ -279,27 +283,48 @@ def test_a_schedule_is_translated_the_same_whichever_order_the_elements_come_in( # --------------------------------------------------------------------------- -def _efficiency(mode: FRBCOperationMode) -> float | None: +def _efficiency(mode: FRBCOperationMode, fill_level: float) -> float | None: return operation_mode_conversion_efficiency( - mode, fill_level_scale=1, energy_unit="kWh" + mode, fill_level_scale=1, energy_unit="kWh", fill_level=fill_level ) -def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(): - assert _efficiency(_mode(COLD, WARM)) == pytest.approx( - _efficiency(_mode(WARM, COLD)) +@pytest.mark.parametrize("fill_level", [-5.0, 20.0, 40.0, 60.0, 95.0]) +def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(fill_level): + assert _efficiency(_mode(COLD, WARM), fill_level) == _efficiency( + _mode(WARM, COLD), fill_level ) -def test_a_mode_whose_elements_convert_differently_states_the_lowest(): - """A COP of 4 when cold and 3 when warm plans as 3: a plan that converts less reaches its target early.""" - assert _efficiency(_mode(COLD, WARM)) == pytest.approx(3.0) +@pytest.mark.parametrize("fill_level, cop", [(20.0, 4.0), (60.0, 3.0)]) +def test_a_mode_states_the_efficiency_of_the_element_where_the_schedule_starts( + fill_level, cop +): + """A COP of 4 when cold and 3 when warm is 4 for a schedule starting cold, and 3 for one starting warm. + + Not the lowest of the two for both: that overstates what a cold buffer needs, + so the plan overshoots its upper band and buys more than it uses, + and against a usage it cannot relax it can find a problem the heat pump can meet infeasible. + The start is what matters most, since it runs before the next storage status asks for a new plan. + """ + assert _efficiency(_mode(COLD, WARM), fill_level) == pytest.approx(cop) + + +def test_an_element_that_tapers_off_applies_only_where_it_applies(): + """Near its highest flow temperature a heat pump may barely fill its buffer, and that does not hold for a cold one.""" + tapering = _element(50, 60, power=1200.0, heat_in_watts=10.0) + mode = _mode(_element(0, 50, power=1200.0, heat_in_watts=4800.0), tapering) + assert _efficiency(mode, 30.0) == pytest.approx(4.0) + assert _efficiency(mode, 55.0) == pytest.approx(10.0 / 1200.0) @pytest.mark.parametrize( "heat_in_watts", [0.0, -50.0], - ids=["standby, drawing power and delivering nothing", "circulation, drawing power and losing heat"], + ids=[ + "standby, drawing power and delivering nothing", + "circulation, drawing power and losing heat", + ], ) def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_watts): """Drawing power while the storage does not fill is not a conversion one ratio can state.""" @@ -317,7 +342,7 @@ def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_wa ) ], ) - assert _efficiency(_mode(standby)) is None + assert _efficiency(_mode(standby), 10.0) is None @pytest.mark.asyncio @@ -346,7 +371,9 @@ async def test_a_standby_mode_does_not_drag_the_recorded_efficiency_to_zero(): system_description, charging_efficiency_sensor_id=CHARGING_EFFICIENCY_SENSOR ) start = datetime(2024, 1, 2, tzinfo=timezone.utc) - assert await frbc._record_charging_efficiency(system_description, start) is True + assert ( + await frbc._record_charging_efficiency(system_description, start, 30.0) is True + ) recorded = [ call.kwargs["values"] for call in frbc._fm_client.post_sensor_data.await_args_list @@ -413,3 +440,102 @@ def test_the_storage_efficiency_states_the_leakiest_element(): ) assert together == top_only assert leakiest != top_only + + +# --------------------------------------------------------------------------- +# Which fill level: the latest, as the resource manager states it +# --------------------------------------------------------------------------- + + +def _report_fill_level(frbc: FRBCSimple, fill_level: float) -> None: + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=fill_level + ) + + +@pytest.mark.asyncio +async def test_the_measured_power_follows_the_latest_storage_status(): + """A buffer that has warmed since its first status runs on the derated element now.""" + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, 10.0) + _report_fill_level(frbc, 70.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_the_fill_level_is_compared_in_the_resource_managers_own_units(): + """The scale converts fill levels for FlexMeasures; the elements' ranges are still in the resource manager's units. + + At 7 in its own units the buffer is cold, though scaled by 10 it would read as 70, which is warm. + """ + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode), fill_level_scale=10) + _report_fill_level(frbc, 7.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + + +@pytest.mark.asyncio +async def test_with_one_element_there_is_nothing_to_guess(caplog): + mode = _mode(_element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0)) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + await _measured_power(frbc, mode, factor=1.0) + assert not caplog.records, caplog.records + + +@pytest.mark.asyncio +async def test_a_fill_level_that_is_not_a_number_counts_as_unknown(caplog): + """s2python lets a NaN fill level through, and no element applies at it, so it is treated as no fill level at all.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, float("nan")) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any("no usable FRBC.StorageStatus" in r.getMessage() for r in caplog.records) + + +def test_no_element_applies_at_a_fill_level_that_is_not_a_number(): + with pytest.raises(ValueError, match="not a number"): + element_for_fill_level([COLD, WARM], float("nan")) + + +def test_the_storage_efficiency_at_a_fill_level_is_that_elements(): + """Where the fill level is known, the leakage element that applies there answers, not the leakiest.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + at_20 = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=20.0 + ) + only_little = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_LITTLE.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert at_20 == only_little + assert at_20 != leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15) + ) + + +@pytest.mark.asyncio +async def test_the_handler_records_leakage_at_the_latest_fill_level(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + _report_fill_level(frbc, 20.0) + await frbc.send_leakage_behaviour(leakage) + posted = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == [ + [ + leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=20.0 + ) + ] + ] From 9587c94f213fe9574e82c27774518286a1569346 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:17 +0100 Subject: [PATCH 10/87] fix(TUNES): an operation mode without a label no longer stops planning S2 makes an operation mode's diagnostic label optional, and FillRateBasedControlTUNES tells its modes apart by looking for "THP" or "NES" inside it. On a mode without one that raised a TypeError, so a single unlabelled mode stopped both planning and the recording of every mode's conversion efficiency. A missing label now simply matches neither. Found by type checking against s2python's own types, which also asked for two variables that held a range and then a number, or a message and then a series, to be given a name for each. New tests cover the planning path, the efficiency path, and posting a usage forecast, none of which any test reached before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_tunes.py | 20 ++--- tests/s2/test_frbc_tunes.py | 75 +++++++++++++++++++ 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 4dae13ca..48ce8eb7 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -278,10 +278,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): active_operation_mode_fill_rate_sensor_id = self._nes_fill_rate_sensor_id # Operation Mode Factor to fill rate - fill_rate = operation_mode.elements[0].fill_rate + fill_rate_range = operation_mode.elements[0].fill_rate fill_rate = ( - fill_rate.start_of_range - + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + fill_rate_range.start_of_range + + (fill_rate_range.end_of_range - fill_rate_range.start_of_range) * factor ) * self._fill_level_scale # Send data to the sensor of the fill rate corresponding to the active operation mode @@ -331,10 +331,10 @@ async def trigger_schedule(self): # type: ignore[override] operation_mode = None efficiency_sensor_id = None for _operation_mode in actuator.operation_modes: - if "THP" in _operation_mode.diagnostic_label: + if "THP" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._thp_efficiency_sensor_id - elif "NES" in _operation_mode.diagnostic_label: + elif "NES" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._nes_efficiency_sensor_id else: @@ -607,9 +607,9 @@ async def send_conversion_efficiencies( ) for operation_mode in actuator.operation_modes: - if "THP" in operation_mode.diagnostic_label: + if "THP" in (operation_mode.diagnostic_label or ""): sensor_id = self._thp_efficiency_sensor_id - elif "NES" in operation_mode.diagnostic_label: + elif "NES" in (operation_mode.diagnostic_label or ""): sensor_id = self._nes_efficiency_sensor_id else: continue @@ -679,7 +679,7 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): minute=(start_time.minute // 15) * 15, second=0, microsecond=0 ) - usage_forecast = translate_usage_forecast_to_fm( + usage_forecast_series = translate_usage_forecast_to_fm( usage_forecast, RESOLUTION, strategy="mean", @@ -688,14 +688,14 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): # Scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) scale = timedelta(minutes=15) / timedelta(seconds=1) - scaled_usage_forecast = usage_forecast * scale + scaled_usage_forecast = usage_forecast_series * scale await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), unit=POWER_UNIT, # e.g. [0, 100] MW/(15 min) - duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), + duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) async def send_fill_level_target_profile( diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 4086b25b..0435ef37 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock import numpy as np +import pandas as pd import pytest import pytest_asyncio from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues @@ -342,3 +343,77 @@ async def test_a_leakage_behaviour_is_recorded_rather_than_refused( ) ] assert posts[0].kwargs["unit"] == "%" + + +@pytest.mark.asyncio +async def test_an_operation_mode_without_a_label_does_not_stop_tunes( + cem_in_frbc_control_type, +): + """S2 makes a diagnostic label optional, and TUNES tells its modes apart by label. + + It looked for "THP" and "NES" inside a label that may be None, which raised a TypeError, + so one unlabelled mode stopped the conversion efficiencies of every mode from being recorded. + """ + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + unlabelled.actuators[0].operation_modes[-1].diagnostic_label = None + fm_client.post_sensor_data.reset_mock() + + await frbc.send_conversion_efficiencies(unlabelled) + + assert fm_client.post_sensor_data.await_count > 0 + + +@pytest.mark.asyncio +async def test_planning_does_not_stop_on_an_unlabelled_operation_mode( + cem_in_frbc_control_type, +): + """The same optional label, where TUNES picks the mode to plan with.""" + from s2python.frbc import FRBCStorageStatus + + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + for operation_mode in unlabelled.actuators[0].operation_modes: + operation_mode.diagnostic_label = None + frbc._system_description_history[str(unlabelled.message_id)] = unlabelled + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + await frbc.trigger_schedule() + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_type): + """S2 states usage per second; TUNES posts it per quarter hour, on a quarter-hour grid, for as long as it was forecast.""" + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=2 * 3600 * 1000), + usage_rate_expected=0.001, + ) + ], + ) + ) + + posts = [ + call + for call in fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 12 + ] + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + values = posts[0].kwargs["values"] + assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) + assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) From 4585f2f60647b9f5f030fecf3b007da524924770 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:26 +0100 Subject: [PATCH 11/87] fix(client): drop a content-type check that could not fire, nor be printed request() ran check_content_type after its retry loop. It could never raise there: request_once parses every answer with aiohttp's json(), which refuses any content type but JSON, and the loop already reports that refusal as a ConnectionError, like every other failure to reach the API. Had it fired, it would have done harm. It built aiohttp's ContentTypeError from a message and a dict, where aiohttp expects the request's info and history, so anything that formatted the error, a log line included, raised an AttributeError of its own; and it read the body with response.text() without awaiting it. Its only test passed a mock whose text() was not a coroutine, and never formatted the error. The check is gone. The test that replaces it states where a non-JSON answer, such as a proxy's error page during a restart, is actually handled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 11 +++---- src/flexmeasures_client/response_handling.py | 14 --------- tests/client/test_init.py | 33 ++++++++++++-------- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 14de2cc4..f8f3d232 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -34,11 +34,7 @@ WrongAPIVersionError, WrongHostError, ) -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - check_response, -) +from flexmeasures_client.response_handling import check_for_status, check_response from flexmeasures_client.utils import apply_deprecated_parameter_aliases, merge_updates LOGGER = logging.getLogger(__name__) @@ -463,8 +459,9 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - check_content_type(response) - + # A body that is not JSON never gets here: + # request_once parses every answer with aiohttp's json(), which refuses any other content type, + # and the loop above reports that as a ConnectionError like any other failure to reach the API. return await response.json(), response.status async def request_once( diff --git a/src/flexmeasures_client/response_handling.py b/src/flexmeasures_client/response_handling.py index 70d1c4d5..1f22ae3f 100644 --- a/src/flexmeasures_client/response_handling.py +++ b/src/flexmeasures_client/response_handling.py @@ -8,11 +8,8 @@ from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING, Callable -from aiohttp import ContentTypeError from yarl import URL -from flexmeasures_client.constants import CONTENT_TYPE - if TYPE_CHECKING: # Only imports the below statements during type checking from flexmeasures_client.client import FlexMeasuresClient @@ -163,17 +160,6 @@ async def check_response( return polling_step, reauth_once, URL(url) -def check_content_type(response): - """Check if response is in format application/json""" - content_type = response.headers.get("Content-Type", "") - if CONTENT_TYPE not in content_type: - text = response.text() - raise ContentTypeError( - "Unexpected content type response from the API", - {"Content-Type": content_type, "response": text}, - ) - - def check_for_status(status, expected_status: int | Iterable[int]): """Check if status is an acceptable 2xx status code. diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 52a02af7..73e795cb 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -21,11 +21,7 @@ _parse_sensor_json_fields, ) from flexmeasures_client.constants import CLIENT_VERSION_HEADER -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - parse_retry_after, -) +from flexmeasures_client.response_handling import check_for_status, parse_retry_after @pytest.mark.parametrize( @@ -837,15 +833,26 @@ def test_parse_sensor_json_fields(): assert sensor["attributes"]["key"] == "value" -def test_check_content_type_failure(): - """check_content_type raises on non-JSON content type.""" - from aiohttp import ContentTypeError as AiohttpContentTypeError +@pytest.mark.asyncio +async def test_an_answer_that_is_not_json_is_a_connection_error() -> None: + """A proxy's HTML error page, as during a server restart, is reported like every other failure to reach the API. - response = MagicMock() - response.headers = {"Content-Type": "text/html"} - response.text.return_value = "some html" - with pytest.raises(AiohttpContentTypeError): - check_content_type(response) + aiohttp's own json() refuses it inside the request loop, which is where this is handled, + so the separate content-type check that used to follow the loop could never fire, and was removed. + """ + with aioresponses() as m: + m.get( + "http://localhost:5000/api/", + status=200, + body="502 Bad Gateway", + content_type="text/html", + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ConnectionError, match="text/html"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() def test_check_for_status_failure(): From c341c4af45270f690e5c3dd40672f13f0dbc5a97 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:38 +0100 Subject: [PATCH 12/87] chore(types): type check against what the code imports, S2 messages included mypy ran with --follow-imports skip, so s2python, pydantic, pandas and aiohttp were Any to it, and a misspelt attribute on an S2 message passed. It now follows imports, with pydantic's plugin, since S2 messages are pydantic models, and the pre-commit hook installs the s2 extra so those types are there to check. The plugin is also the guard against sliding back: without the extra it fails to load, rather than leaving every S2 type silently Any. A misspelt message_id on a leakage behaviour is now an error naming it. What it found, besides the two fixes in the previous commits: - get_validation_error_summary iterated a pydantic ValidationError, which is not iterable, so it raised a TypeError on any error it was given. Nothing called it, so it is gone rather than repaired. - Revocations were recorded by the RevokeObject's raw id, while every other id a handler keeps is text, so a lookup by the text of the id would have missed. They are recorded as text now. Nothing calls is_revoked yet, which means a revocation is acknowledged and then never consulted; that is recorded as a finding for later rather than changed here. - The rest is annotation: a message that arrives as a model, a dict or text is normalised to a dict once and checked as one; get_message returns the dict it always returned rather than claiming a str; the message type of a registered schema is read through model_fields rather than the deprecated __fields__; an operation mode's id is text where a label is wanted; and the actuator's power ranges are pairs rather than tuples of any length. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 3 +- pyproject.toml | 9 ++++-- src/flexmeasures_client/s2/__init__.py | 10 +++---- src/flexmeasures_client/s2/cem.py | 11 +++++--- .../s2/control_types/FRBC/frbc_simple.py | 7 +++-- .../s2/control_types/FRBC/utils.py | 2 +- .../s2/script/websockets_client.py | 6 ++-- src/flexmeasures_client/s2/utils.py | 19 ++++++------- tests/s2/test_frbc_simple_planning.py | 28 +++++++++++++++++++ 9 files changed, 66 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a53e82e7..99b3eef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,8 @@ repos: name: mypy (static typing) pass_filenames: false language: system - entry: uv run poe type-check + # With the s2 extra, so that the S2 message types are there to check against. + entry: uv run --extra s2 poe type-check verbose: true - repo: local diff --git a/pyproject.toml b/pyproject.toml index 8f1552e4..251b1d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,8 +89,13 @@ help = "Run S2 tests and generate coverage report." cmd = "pytest tests/s2 --cov --cov-report=" [tool.poe.tasks.type-check] -help = "Run mypy on every source file, including the bodies of functions without type hints." -cmd = "mypy --follow-imports skip --ignore-missing-imports --check-untyped-defs src" +help = "Run mypy on every source file, including the bodies of functions without type hints, against the types of what they import." +cmd = "mypy --ignore-missing-imports --check-untyped-defs src" + +[tool.mypy] +# The S2 messages are pydantic models, so without the plugin mypy misreads their constructors and fields. +# It also makes a missing s2 extra fail loudly, rather than leaving every S2 type silently Any. +plugins = ["pydantic.mypy"] [tool.poe.tasks.lint] help = "Run pre-commit hooks on all files." diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 981269e1..a1fcadc5 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -6,7 +6,7 @@ from collections import deque from dataclasses import dataclass from datetime import datetime -from typing import Callable, Coroutine, Dict, Type +from typing import Callable, Dict, Type from zoneinfo import ZoneInfo import pydantic @@ -75,7 +75,7 @@ async def wrap(*args, **kwargs): ) return outgoing_message - message_type = schema.__fields__.get("message_type").default + message_type = schema.model_fields["message_type"].default setattr(wrap, "_tag", Tag(message_type)) return wrap @@ -176,7 +176,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: message_type = "" if isinstance(message, pydantic.BaseModel): - message_type = message.message_type + message_type = message.model_dump().get("message_type", "") elif isinstance(message, dict): message_type = message.get("message_type", "") elif isinstance(message, str): @@ -186,7 +186,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: async def handle_message( self, message: pydantic.BaseModel | str | Dict - ) -> Coroutine: + ) -> pydantic.BaseModel | None: """ Calls the handler linked to the message_type and converts the output to a serialized dict, i.e. it converts all the inner objects to Python @@ -243,7 +243,7 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: Stores the revoked object ID into the objects_revoked list """ - self.objects_revoked.append(message.object_id) + self.objects_revoked.append(str(message.object_id)) return ReceptionStatus( subject_message_id=str(message.message_id), status=ReceptionStatusValues.OK diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 7e83cd8b..72ef9b3d 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -176,13 +176,16 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): the control type handler and in the CEM, then the one defined in the control type prevails. """ - response = None + response: pydantic.BaseModel | None = None if isinstance(message, pydantic.BaseModel): message = json.loads(message.json()) if isinstance(message, str): message = json.loads(message) + assert isinstance( + message, dict + ) # what a message is from here on, whatever form it came in self._logger.debug(f"Received: {message}") @@ -224,11 +227,11 @@ def update_control_type(self, control_type: ControlType): """ self._control_type = control_type - async def get_message(self) -> str: + async def get_message(self) -> dict: """Call this function to get the messages to be sent to the RM Returns: - str: message in JSON format + dict: the message, as JSON-serialisable Python objects """ message = await self._sending_queue.get() @@ -412,7 +415,7 @@ def handle_revoke_object(self, message: RevokeObject): for initials, control_type in control_types.items(): if initials in str(message.object_type): self._control_types_handlers[control_type].revoke_message( - message.object_id + str(message.object_id) ) return get_reception_status(message, ReceptionStatusValues.OK) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index c29ad86e..131a8753 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -570,7 +570,7 @@ async def _record_charging_efficiency( operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) if efficiency is not None: - efficiencies[operation_mode.id] = efficiency + efficiencies[str(operation_mode.id)] = efficiency except NotImplementedError: self._logger.exception( "Not asking for a schedule against a charging efficiency: a fill rate in %s cannot be compared against the watts S2 states a power range in.", @@ -773,7 +773,10 @@ async def trigger_schedule( # TODO: an element states one power range per commodity quantity, and no distinction is made between them here, # as the schedule translation does not make one either (see power_to_fill_rate_with_metrics). power_ranges = [ - tuple(sorted((power_range.start_of_range, power_range.end_of_range))) + ( + min(power_range.start_of_range, power_range.end_of_range), + max(power_range.start_of_range, power_range.end_of_range), + ) for operation_mode in actuator.operation_modes for element in operation_mode.elements for power_range in element.power_ranges diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 646b36bc..35e652a2 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -511,7 +511,7 @@ def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: def om_label(operation_mode: FRBCOperationMode) -> str: label = getattr(operation_mode, "diagnostic_label", None) - return label if isinstance(label, str) else operation_mode.id + return label if isinstance(label, str) else str(operation_mode.id) def explain_choice( diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 510062fe..942f1988 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -111,7 +111,7 @@ async def main_s2(): roles=[ Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) ], - instruction_processing_delay=Duration(1), + instruction_processing_delay=Duration(root=1), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, @@ -141,13 +141,13 @@ async def main_s2(): print("RECEIVING: ", control_type) - message = ReceptionStatus( + reception_status = ReceptionStatus( subject_message_id=control_type["message_id"], status=ReceptionStatusValues.OK, ) print("SENDING: ReceptionStatus") - await ws.send_str(message.model_dump_json()) + await ws.send_str(reception_status.model_dump_json()) electric_power = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 50eabaab..cfdcd55e 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import OrderedDict -from typing import Mapping, TypeVar +from typing import Any, Mapping, Protocol, TypeVar from uuid import uuid4 import pydantic @@ -49,15 +49,6 @@ def get_unique_id() -> str: return str(uuid4()) -def get_validation_error_summary(error: pydantic.ValidationError) -> str: - error_summary = "" - - for i, e in enumerate(error): - error_summary += f"\nValidationEror {i} -> \t {e['msg']}" - - return error_summary[1:] # skipping the first \n - - def get_message_id(message: pydantic.BaseModel) -> str | None: """ This function returns the message_id if it is found in the message, @@ -71,8 +62,14 @@ def get_message_id(message: pydantic.BaseModel) -> str | None: return None +class _HasMessageId(Protocol): + """Any S2 message: every one of them carries a message id.""" + + message_id: Any + + def get_reception_status( - subject_message: pydantic.BaseModel, + subject_message: _HasMessageId, status: ReceptionStatusValues = ReceptionStatusValues.OK, ): """ diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 5d5e5c77..c4c740d4 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -217,3 +217,31 @@ async def test_planning_without_a_storage_status_is_logged_not_printed(capsys, c record.levelname == "WARNING" and "FRBC.StorageStatus" in record.getMessage() for record in caplog.records ), caplog.records + + +@pytest.mark.asyncio +async def test_a_revoked_object_is_known_by_its_id_as_text(): + """Revocations are recorded by the id as text, which is how every other id in a handler is kept.""" + from s2python.common import RevokableObjects, RevokeObject + + frbc = frbc_simple() + object_id = get_unique_id() + await frbc.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_recognises_a_message_in_any_form_it_accepts(): + """supports_message takes a model, a dict or JSON text, and reads the type from each the same way.""" + frbc = frbc_simple() + description = system_description("2022-12-02T00:00:00+00:00") + as_dict = description.model_dump(mode="json") + assert frbc.supports_message(description) + assert frbc.supports_message(as_dict) + assert frbc.supports_message(description.model_dump_json()) + assert not frbc.supports_message(as_dict | {"message_type": "Handshake"}) From 1928d56749db3ad87751ffe7564c47c9352ef12d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:11:28 +0100 Subject: [PATCH 13/87] docs(changelog): an unlabelled TUNES mode, and two helpers that never worked Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ CHANGELOG.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c90d1871..51ef1771 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,6 +41,8 @@ Changed Fixed ----- +* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, and one unlabelled mode stopped both [`#257 `__] +* ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e030e46c..7b7fa787 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,11 @@ Changed * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] +Removed +------- + +* ``response_handling.check_content_type``, which could never fire where it was called, since every answer is already parsed as JSON before it, and which raised an ``AttributeError`` of its own when its error was printed. A non-JSON answer is reported as a ``ConnectionError``, as before [`#257 `__] + Fixed ----- From 2a644e6d85a9bec39d9a482c87ed0acf3b214159 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:37:09 +0100 Subject: [PATCH 14/87] fix: correct what this branch claimed, and test what review found untested An adversarial review ran every claim on this branch, and three were false: - The removed content-type check could fire. aiohttp's json() accepts JSON variants such as application/problem+json and application/vnd.api+json, and the check refused those, with an error that could not be printed. Removing it is right, but it changes behaviour: such an answer is now read like any other. The comment at the removed call, the test's docstring and the changelog entry now say so, instead of calling it dead code. - "A non-JSON answer is reported as a ConnectionError, as before" was only true of a wrong content type. A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError, as it was before. - One unlabelled TUNES mode stopped planning, and the recording of the efficiencies of the modes listed after it, not of every mode. Two more overstatements live only in pushed commit messages and the PR body: following imports leaves pandas Any, since it ships no type information and no stubs are installed, and the efficiency path was already reached by a test. Tests now cover what five mutations showed was unpinned: a revocation taken by the CEM itself, with no control type active; a model of a type a handler does not handle; a usage forecast starting off the quarter hour; and one averaged within a quarter hour rather than taking its highest value. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- CHANGELOG.rst | 2 +- src/flexmeasures_client/client.py | 8 ++-- tests/client/test_init.py | 5 ++- tests/s2/test_frbc_simple_planning.py | 32 ++++++++++++++ tests/s2/test_frbc_tunes.py | 63 +++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 51ef1771..a8cb28a7 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,7 +41,7 @@ Changed Fixed ----- -* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, and one unlabelled mode stopped both [`#257 `__] +* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7b7fa787..05d31188 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,7 +32,7 @@ Changed Removed ------- -* ``response_handling.check_content_type``, which could never fire where it was called, since every answer is already parsed as JSON before it, and which raised an ``AttributeError`` of its own when its error was printed. A non-JSON answer is reported as a ``ConnectionError``, as before [`#257 `__] +* ``response_handling.check_content_type``, which refused an answer in a JSON variant that aiohttp parses, such as ``application/problem+json``, with an error that raised an ``AttributeError`` of its own when printed. Such an answer is now read like any other. A content type that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 `__] Fixed ----- diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index f8f3d232..698f5bc6 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -459,9 +459,11 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - # A body that is not JSON never gets here: - # request_once parses every answer with aiohttp's json(), which refuses any other content type, - # and the loop above reports that as a ConnectionError like any other failure to reach the API. + # request_once has already parsed this answer with aiohttp's json(), + # which refuses a content type that is not JSON, and the loop above reports that as a ConnectionError. + # A check that used to follow here refused the JSON variants aiohttp accepts too, such as application/problem+json, + # with an error that could not even be printed, so it is gone. + # A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError. return await response.json(), response.status async def request_once( diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 73e795cb..a2b14436 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -837,8 +837,9 @@ def test_parse_sensor_json_fields(): async def test_an_answer_that_is_not_json_is_a_connection_error() -> None: """A proxy's HTML error page, as during a server restart, is reported like every other failure to reach the API. - aiohttp's own json() refuses it inside the request loop, which is where this is handled, - so the separate content-type check that used to follow the loop could never fire, and was removed. + aiohttp's own json() refuses it inside the request loop, which is where this is handled. + The separate content-type check that used to follow the loop is gone: + it could only fire on the JSON variants aiohttp accepts, such as application/problem+json, and it should not have refused those. """ with aioresponses() as m: m.get( diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index c4c740d4..be2da4e0 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -245,3 +245,35 @@ def test_a_handler_recognises_a_message_in_any_form_it_accepts(): assert frbc.supports_message(as_dict) assert frbc.supports_message(description.model_dump_json()) assert not frbc.supports_message(as_dict | {"message_type": "Handshake"}) + + +@pytest.mark.asyncio +async def test_the_cem_records_a_revocation_by_its_id_as_text(): + """With no control type active, the CEM itself takes the revocation and hands it to the handler it is for.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.cem import CEM + + cem = CEM(fm_client=frbc_simple()._fm_client) + frbc = frbc_simple() + cem.register_control_type(frbc) + object_id = get_unique_id() + await cem.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_turns_away_a_model_it_does_not_handle(): + from s2python.common import EnergyManagementRole, Handshake + + handshake = Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.CEM, + supported_protocol_versions=["0.0.2-beta"], + ) + assert not frbc_simple().supports_message(handshake) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 0435ef37..41b788c3 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -417,3 +417,66 @@ async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_t values = posts[0].kwargs["values"] assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_from_the_quarter_hour_it_starts_in( + cem_in_frbc_control_type, +): + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, 0, 7, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.001 + ) + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["start"] == datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( + cem_in_frbc_control_type, +): + """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths, per quarter hour.""" + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + half = Duration(root=int(7.5 * 60 * 1000)) + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.001), + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.003), + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) From 4d5435eda369891f8446ac3a81bcb1e6a8a7cb85 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:40:29 +0100 Subject: [PATCH 15/87] fix(FRBC): every rate per second, as S2 states it, under the instructions The schedule translation keeps a running fill level, so that each step is answered with the operation mode for the fill level the storage is then at. It advanced that level by a fill rate per second times a step in hours, so it moved 3600 times too little, and every step of a twelve-hour schedule was answered at the fill level the schedule started from (finding C1). Fixing that alone would have been worse: the level would then rise at the real rate and never fall, and pin every step at the top. So the integrator gets one written contract, the one S2 states for all three rates: a change in fill level per second, in the resource manager's own units, with usage and leakage lowering it where positive. Its step is a duration rather than a number of some unit. - Leakage now counts (C2). It read leakage_rate_per_second, which S2 does not have, and an actuator's leakage_behaviour, which cannot exist, both through getattr with a default, so the branch never ran. The handler now passes the latest FRBC.LeakageBehaviour, and the element that applies at the fill level. - Usage now counts. The handler passes the latest FRBC.UsageForecast, read per step by usage_rate_at; a frame may still carry a usage_forecast column instead, per second, which is what TUNES does, now dividing by the quarter hour it posts in. - The step is the schedule's own (B5): its duration over its number of values, which is the resolution of the sensor FlexMeasures answered at. It was taken to be a quarter hour whatever it was, so an hourly sensor's twelve values were laid on forty-eight quarter hours, pandas refused, and no instruction went out. - An instruction goes out wherever the mode or its factor changes, not only where the power does: at a constant power the fill level can pass from one element into another, which needs another factor. - The factor is the one that draws the power asked for (C4), taken from the power range, rather than worked back out of the fill rate, which failed wherever the fill rate was one value and the power a range: asked for 1200 W from a mode running 800 to 1200 W, that asked for 800 W. Where the power is one value and the fill rate a range, every factor draws that power, so the one that fills most is asked for. op_mode_compute_factor is gone. - The storage efficiency compounds a leakage rate per second (B10), where it compounded it per hour, declaring a buffer that loses about 7% over twelve hours as holding its heat to within 0.002%. - The handler's power and fill-rate measurements cover its own resolution (D9), rather than a quarter hour whatever it planned on. - translations.py is linted again: it opened with `# flake8: noqa`, which hid nothing, as it turns out. TUNES shares the translation, so its instructions change with it, for the better: it carried the same per-hour integration. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 13 +- .../s2/control_types/FRBC/frbc_tunes.py | 4 +- .../s2/control_types/FRBC/utils.py | 331 +++++++-------- .../s2/control_types/translations.py | 17 +- tests/s2/test_frbc_time_base.py | 391 ++++++++++++++++++ tests/s2/test_frbc_tunes.py | 40 ++ tests/s2/test_frbc_utils.py | 45 +- 7 files changed, 642 insertions(+), 199 deletions(-) create mode 100644 tests/s2/test_frbc_time_base.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 131a8753..36cbdc18 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -396,6 +396,11 @@ def _in_the_power_unit(self, power: float) -> float: return power return convert_units([power], S2_POWER_UNIT, self.power_unit)[0] + @staticmethod + def _latest(history: dict): + """The message a history holds that arrived last, or None while it holds none.""" + return list(history.values())[-1] if history else None + def _latest_fill_level(self) -> float | None: """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" if not self._storage_status_history: @@ -507,7 +512,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=recorded_at, values=[value], unit=unit, - duration=timedelta(minutes=15), + duration=self._resolution, ) except (JobFailedError, ValueError, ConnectionError, NotImplementedError): # This runs in a task nobody awaits, @@ -536,7 +541,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=self.now(), values=[rate], unit=self.fill_rate_unit(), - duration=timedelta(minutes=15), + duration=self._resolution, ) async def _record_charging_efficiency( @@ -914,6 +919,10 @@ async def trigger_schedule( soc_min / self._fill_level_scale, soc_max / self._fill_level_scale, ), + # What the storage loses by itself and what is drawn from it, as the resource manager last said, + # so that the fill level each step is answered at falls as well as rises. + leakage_behaviour=self._latest(self._leakage_behaviour_history), + usage_forecast=self._latest(self._usage_forecast_history), ) await self.send_instruction_batch( diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 48ce8eb7..7eef5496 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,9 +480,11 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) + # Posted per quarter hour by send_usage_forecast, and read back as such, + # while the schedule translation takes a usage rate per second, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) + ) / (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 35e652a2..587698ea 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -4,7 +4,6 @@ from math import isclose from typing import List, Sequence, TypeVar -import numpy as np import pandas as pd try: @@ -15,6 +14,7 @@ FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCSystemDescription, + FRBCUsageForecast, ) from s2python.frbc.frbc_operation_mode_element import FRBCOperationModeElement except ImportError: @@ -29,99 +29,49 @@ LOGGER = logging.getLogger(__name__) -def op_mode_compute_factor( - op_mode_elem: FRBCOperationModeElement, - fill_rate: float, - logger: logging.Logger = LOGGER, -) -> float: - """Compute the operation mode factor for a given fill rate.""" - start_fill_rate = op_mode_elem.fill_rate.start_of_range - end_fill_rate = op_mode_elem.fill_rate.end_of_range - delta_fill_rate = end_fill_rate - start_fill_rate - - fill_rate = max(fill_rate, start_fill_rate) - fill_rate = min(fill_rate, end_fill_rate) - - # Case that start_fill_rate == end_fill_rate - if np.isclose(delta_fill_rate, 0): - return 0 - - omf = (fill_rate - start_fill_rate) / delta_fill_rate - if omf < 0 or omf > 1: - logger.error( - f"Invalid operation mode factor {omf} computed from fill_rate {fill_rate}" - ) - return omf - - def compute_next_fill_level( fill_level: float, fill_rate: float, - leakage_behaviour: FRBCLeakageBehaviour | None, - deltaT: float, - usage: float = 0.0, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_rate: float = 0.0, ) -> float: - """ - Compute the next fill level of a storage system over a time step. - - Parameters - ---------- - fill_level - Current fill level in system units (e.g., kWh, °C, height) - fill_rate - Fill rate chosen from the operation mode (units per hour) - leakage_behaviour - FRBC LeakageBehaviour; may be None - deltaT - Time step in hours - usage - Consumption/usage over the time step (in system units per hour) - - Returns - ------- - next_fill_level : float - Fill level after applying fill rate, leakage, and usage - """ + """The fill level one step later. - # --- Apply fill rate over deltaT --- - delta_fill = fill_rate * deltaT - - # --- Apply leakage over deltaT --- - total_leakage = 0.0 - if leakage_behaviour and leakage_behaviour.elements: - # Sum contributions from all relevant elements (or pick one based on fill level) - element = next( - ( - e - for e in leakage_behaviour.elements - if e.fill_level_range.start_of_range - <= fill_level - <= e.fill_level_range.end_of_range - ), - None, - ) - if element is None: - # Take closest element - element = min( - leakage_behaviour.elements, - key=lambda e: max( - 0.0, - e.fill_level_range.start_of_range - fill_level, - fill_level - e.fill_level_range.end_of_range, - ), - ) + This is the unit contract for every rate it takes, and it is the one S2 states: + each is a change in fill level per second, in the resource manager's own fill-level units. + A fill rate raises the fill level, while a usage rate and a leakage rate both lower it where they are positive. + The step is a duration, rather than a number of some unit, so that it cannot be taken in the wrong one. + + :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; + none where the resource manager has not said. + :param usage_rate: what is drawn from the storage, as the usage forecast expects it over this step. + """ + leakage_rate = 0.0 + if leakage_behaviour is not None: + leakage_rate = element_for_fill_level( + leakage_behaviour.elements, fill_level + ).leakage_rate + return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() - # leakage_rate_per_second is in system units lost per second - total_leakage = ( - getattr(element, "leakage_rate_per_second", 0.0) * deltaT * 3600 - ) # convert hours → seconds - # --- Apply usage --- - total_usage = usage * deltaT +def usage_rate_at( + usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp +) -> float: + """The usage rate a forecast expects at a moment, per second, as S2 states it. - # --- Next fill level --- - next_fill_level = fill_level + delta_fill - total_leakage - total_usage - return next_fill_level + A forecast is a run of elements, each lasting its own duration from where the one before it ended. + Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. + """ + if usage_forecast is None: + return 0.0 + element_start = pd.Timestamp(usage_forecast.start_time) + for element in usage_forecast.elements: + element_end = element_start + timedelta(milliseconds=element.duration.root) + if element_start <= moment < element_end: + return element.usage_rate_expected + element_start = element_end + return 0.0 def fm_schedule_to_instructions( @@ -130,15 +80,30 @@ def fm_schedule_to_instructions( initial_fill_level: float, logger: logging.Logger = LOGGER, fill_level_bounds: tuple[float, float] | None = None, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_forecast: FRBCUsageForecast | None = None, ) -> List[FRBCInstruction]: """Turn a FlexMeasures schedule into the instructions that follow it. + Each step is answered with the operation mode, and the factor within it, that draws the power the schedule asks for + at the fill level the storage is then at, so a running fill level is kept alongside, under the contract of compute_next_fill_level. + An instruction is sent where that answer changes, which can happen at a constant power too, + where the fill level passes from one element of a mode into another. + + :param schedule: a schedule as FlexMeasures returns it, or a frame indexed by time with a "schedule" column. + A frame may also carry a "usage_forecast" column, per second as S2 states it, + which then takes the place of usage_forecast. :param fill_level_bounds: the fill levels this conversion holds its own running fill level between, in the resource manager's units. These have to be the hard bounds the schedule was planned against, or a step is answered with the operation mode for a fill level the scheduler never planned. Without them the declared fill-level range is used, which is right only for a caller that asked for a schedule against that same range. + :param leakage_behaviour: what the storage loses by itself, as the resource manager last said. + :param usage_forecast: what is drawn from the storage, as the resource manager last forecast it. + + The step is the schedule's own: a schedule from FlexMeasures states it as its duration over its number of values, + which is the resolution of the sensor it was asked for, whatever resolution the caller plans on. """ if fill_level_bounds is not None and fill_level_bounds[0] > fill_level_bounds[1]: @@ -153,85 +118,99 @@ def fm_schedule_to_instructions( return [] if isinstance(schedule, dict): + values = schedule["values"] + if len(values) == 0: + return [] start = pd.Timestamp(schedule["start"]) - schedule_duration = pd.Timedelta(schedule["duration"]) - idx = pd.DatetimeIndex( - pd.date_range( - start=start, - end=start + schedule_duration - timedelta(minutes=15), - freq="15min", - ) + step = pd.Timedelta(schedule["duration"]) / len(values) + idx = pd.date_range(start=start, periods=len(values), freq=step) + schedule = pd.Series(values, index=idx, name="schedule").to_frame() + else: + # A frame of one row has no later step for its fill level to matter to. + step = ( + pd.Timedelta(schedule.index[1] - schedule.index[0]) + if len(schedule) > 1 + else pd.Timedelta(0) ) - schedule = pd.Series(schedule["values"], index=idx, name="schedule").to_frame() - - previous_power = None - instructions = [] - - actuator = system_description.actuators[0] if len(system_description.actuators) != 1: raise NotImplementedError( f"This CEM only supports 1 actuator but" f"{len(system_description.actuators)} were provided" ) - + actuator = system_description.actuators[0] operation_modes: list[FRBCOperationMode] = actuator.operation_modes - fill_level = initial_fill_level + # Held between the hard bounds the schedule was planned against, + # so that each step is answered with the operation mode for the fill level the scheduler planned. + # Falling back to the declared range keeps every other caller as it was. + if fill_level_bounds is None: + fill_level_range: NumberRange = system_description.storage.fill_level_range + fill_level_min = fill_level_range.start_of_range + fill_level_max = fill_level_range.end_of_range + else: + fill_level_min, fill_level_max = fill_level_bounds - deltaT = timedelta(minutes=15) / timedelta(hours=1) + instructions: list[FRBCInstruction] = [] + last_instructed: tuple[str, float] | None = None + fill_level = initial_fill_level + has_usage_column = "usage_forecast" in schedule.columns for timestamp, row in schedule.iterrows(): power = row["schedule"] - usage = row.get("usage_forecast", 0) - - if previous_power is None or not isclose(previous_power, power): - # Convert from power to fill rate - results = [ - (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) - for om in operation_modes - ] - - # Step 1: minimize fill-level penalty (primary) - min_fill_penalty = min(r[2] for r in results) - fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] - - # Step 2: among those, minimize power penalty (secondary) - min_power_penalty = min(r[3] for r in fill_level_candidates) - power_candidates = [ - r for r in fill_level_candidates if r[3] == min_power_penalty - ] - - # Step 3: among those, pick the highest efficiency - best = max(power_candidates, key=lambda r: r[4]) - - ( - best_operation_mode, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - best_element, - ) = best - - logger.debug( - "Selected FRBC operation mode '%s' (element='%s') for power=%.3f, fill_level=%.3f: " - "fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", - om_label(best_operation_mode), - getattr(best_element, "id", "unknown"), - power, - fill_level, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - explain_choice(results, best), - ) - operation_mode_factor = op_mode_compute_factor( - best_element, fill_rate=best_fill_rate, logger=logger - ) + # The mode and factor for this power at this fill level, every step, + # since the element that applies can change while the power does not. + results = [ + (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) + for om in operation_modes + ] + + # Step 1: minimize fill-level penalty (primary) + min_fill_penalty = min(r[2] for r in results) + fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] + # Step 2: among those, minimize power penalty (secondary) + min_power_penalty = min(r[3] for r in fill_level_candidates) + power_candidates = [ + r for r in fill_level_candidates if r[3] == min_power_penalty + ] + + # Step 3: among those, pick the highest efficiency + best = max(power_candidates, key=lambda r: r[4]) + + ( + best_operation_mode, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + best_element, + operation_mode_factor, + ) = best + + logger.debug( + "Selected FRBC operation mode '%s' for power=%.3f, fill_level=%.3f: " + "factor=%.6f, fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", + om_label(best_operation_mode), + power, + fill_level, + operation_mode_factor, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + explain_choice(results, best), + ) + + # The factor is the one that draws the power asked for, taken from the power range itself. + # It used to be worked back out of the fill rate instead, + # which fails wherever the fill-rate range is a single value and the power range is not: + # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. + answer = (str(best_operation_mode.id), operation_mode_factor) + if last_instructed is None or not ( + answer[0] == last_instructed[0] and isclose(answer[1], last_instructed[1]) + ): instruction = FRBCInstruction( message_id=get_unique_id(), id=get_unique_id(), @@ -245,29 +224,22 @@ def fm_schedule_to_instructions( f"Instruction created: at {timestamp} set {actuator.diagnostic_label if isinstance(actuator.diagnostic_label, str) else actuator} to {best_operation_mode.diagnostic_label if isinstance(best_operation_mode.diagnostic_label, str) else best_operation_mode} with factor {operation_mode_factor}" ) instructions.append(instruction) + last_instructed = answer - # Update fill level + usage_rate = ( + float(row["usage_forecast"]) + if has_usage_column + else usage_rate_at(usage_forecast, pd.Timestamp(timestamp)) + ) fill_level = compute_next_fill_level( fill_level=fill_level, - fill_rate=best_fill_rate, # S2-derived fill rate - leakage_behaviour=getattr(actuator, "leakage_behaviour", None), - deltaT=deltaT, - usage=usage, + fill_rate=best_fill_rate, + step=step, + leakage_behaviour=leakage_behaviour, + usage_rate=usage_rate, ) - - # Hold the running fill level between the hard bounds the schedule was planned against, - # so that each step is answered with the operation mode for the fill level the scheduler planned. - # Falling back to the declared range keeps every other caller as it was. - if fill_level_bounds is None: - fill_level_range: NumberRange = system_description.storage.fill_level_range - fill_level_min = fill_level_range.start_of_range - fill_level_max = fill_level_range.end_of_range - else: - fill_level_min, fill_level_max = fill_level_bounds fill_level = max(fill_level_min, min(fill_level_max, fill_level)) - previous_power = power - return instructions @@ -408,7 +380,7 @@ def power_to_fill_rate_with_metrics( operation_mode: FRBCOperationMode, power: float, fill_level: float, -) -> tuple[float, float, float, float, FRBCOperationModeElement]: +) -> tuple[float, float, float, float, FRBCOperationModeElement, float]: """ Evaluate an operation mode for given power and fill level. @@ -418,6 +390,7 @@ def power_to_fill_rate_with_metrics( power_penalty: float efficiency: float element: FRBCOperationModeElement that produced fill_rate + factor: the operation mode factor that draws this power, which is what an instruction asks for """ # --- 1. Select the element that applies at this fill level --- best_element = element_for_fill_level(operation_mode.elements, fill_level) @@ -441,7 +414,12 @@ def power_to_fill_rate_with_metrics( pr_end = power_range.end_of_range if pr_end == pr_start: - factor = 0.0 + # Every factor draws this one power, so the power cannot say which factor is meant. + # The one that fills the storage most for it is asked for, which is either end of the fill-rate range, + # and where that range is a single value too, the factor makes no difference and stays at zero. + fr_start = best_element.fill_rate.start_of_range + fr_end = best_element.fill_rate.end_of_range + factor = 1.0 if fr_end > fr_start else 0.0 efficiency = 0.0 else: factor = (power - pr_start) / (pr_end - pr_start) @@ -455,7 +433,7 @@ def power_to_fill_rate_with_metrics( fr_end = best_element.fill_rate.end_of_range fill_rate = fr_start + factor * (fr_end - fr_start) - return fill_rate, fill_penalty, power_penalty, efficiency, best_element + return fill_rate, fill_penalty, power_penalty, efficiency, best_element, factor def clamp_distance(value: float, start: float, end: float) -> float: @@ -516,10 +494,18 @@ def om_label(operation_mode: FRBCOperationMode) -> str: def explain_choice( results: list[ - tuple[FRBCOperationMode, float, float, float, float, FRBCOperationModeElement] + tuple[ + FRBCOperationMode, + float, + float, + float, + float, + FRBCOperationModeElement, + float, + ] ], best: tuple[ - FRBCOperationMode, float, float, float, float, FRBCOperationModeElement + FRBCOperationMode, float, float, float, float, FRBCOperationModeElement, float ], ) -> str: """ @@ -532,11 +518,12 @@ def explain_choice( best_power_penalty, best_efficiency, best_element, + _best_factor, ) = best lines: list[str] = [] - for om, fr, fill_pen, power_pen, eff, element in results: + for om, fr, fill_pen, power_pen, eff, element, _factor in results: label = om_label(om) element_label = getattr(element, "id", "unknown") diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a4cd5ecc..f12b2d01 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -1,5 +1,3 @@ -# flake8: noqa - from datetime import timedelta import numpy as np @@ -35,7 +33,7 @@ def leakage_behaviour_to_storage_efficiency( due to leakage within the given range of the fill level. This is defined as a function of the fill level. - Storage Efficiency: percentage of the storage that remains after one time period. + Storage Efficiency: percentage of the storage that remains after one time step of the given resolution. Example: @@ -71,12 +69,13 @@ def leakage_behaviour_to_storage_efficiency( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) leakage = element.leakage_rate * fill_level_scale - base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) - - # Discuss conversions - storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** ( - resolution / base_resolution - ) + # S2 states a leakage rate per second, so what remains after one second is compounded over the seconds in a step. + # This used to compound it over the hours in a step instead, + # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, + # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. + # The loss is taken as a share of the top of the range, as before, which understates it at lower fill levels. + seconds_per_step = pd.Timedelta(resolution) / pd.Timedelta(seconds=1) + storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** seconds_per_step return storage_efficiency diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py new file mode 100644 index 00000000..d1e91623 --- /dev/null +++ b/tests/s2/test_frbc_time_base.py @@ -0,0 +1,391 @@ +"""The time base under the instructions: every rate per second, as S2 states it. + +S2 states a fill rate, a usage rate and a leakage rate each as a change in fill level per second. +The schedule translation keeps a running fill level, to answer each step with the operation mode for the fill level +the storage is then at, and it used to advance that level by a rate per second times a step in hours, +so it moved 3600 times too little, and every step of a twelve-hour schedule was answered at the fill level it started from. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import pandas as pd +import pytest +from s2python.common import ( + Commodity, + CommodityQuantity, + Duration, + NumberRange, + PowerRange, +) +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, +) + +from flexmeasures_client.client import FlexMeasuresClient, _server_version_at_least +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + compute_next_fill_level, + fm_schedule_to_instructions, + usage_rate_at, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +START = datetime(2024, 1, 1, tzinfo=timezone.utc) +QUARTER_HOUR = timedelta(minutes=15) + + +def _leakage(*elements: tuple[float, float, float]) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=START, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + leakage_rate=rate, + ) + for lo, hi, rate in elements + ], + ) + + +def _usage(*elements: tuple[timedelta, float]) -> FRBCUsageForecast: + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=int(duration / timedelta(milliseconds=1))), + usage_rate_expected=rate, + ) + for duration, rate in elements + ], + ) + + +# --------------------------------------------------------------------------- +# The integrator's contract +# --------------------------------------------------------------------------- + + +def test_a_fill_rate_is_integrated_per_second(): + """0.001 per second for a quarter hour is 0.9, which the old integrator made 0.00025.""" + assert compute_next_fill_level(1.0, 0.001, QUARTER_HOUR) == pytest.approx(1.9) + + +def test_usage_lowers_the_fill_level_per_second(): + assert compute_next_fill_level( + 10.0, 0.0, QUARTER_HOUR, usage_rate=0.0005 + ) == pytest.approx(10.0 - 0.45) + + +def test_leakage_lowers_the_fill_level_at_the_rate_of_the_element_that_applies(): + """At 50, in the leakier upper element, an hour loses 3.6; the lower element would lose 0.36.""" + leakage = _leakage((50, 100, 0.001), (0, 50, 0.0001)) + assert compute_next_fill_level( + 50.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(46.4) + assert compute_next_fill_level( + 20.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(19.64) + + +@pytest.mark.parametrize( + "moment, expected", + [ + (START - timedelta(minutes=1), 0.0), + (START + timedelta(minutes=30), 0.001), + (START + timedelta(minutes=90), 0.002), + # Where one element ends, the next begins, and its rate applies. + (START + timedelta(minutes=60), 0.002), + (START + timedelta(minutes=150), 0.0), + ], +) +def test_the_usage_rate_a_forecast_expects_at_a_moment(moment, expected): + """Elements follow one another for their own durations; outside them nothing is forecast.""" + forecast = _usage((timedelta(hours=1), 0.001), (timedelta(hours=1), 0.002)) + assert usage_rate_at(forecast, pd.Timestamp(moment)) == pytest.approx(expected) + + +def test_without_a_forecast_nothing_is_used(): + assert usage_rate_at(None, pd.Timestamp(START)) == 0.0 + + +# --------------------------------------------------------------------------- +# The translation +# --------------------------------------------------------------------------- + + +def _element(lo, hi, power_top, fill_rate_top, power_bottom=0.0, fill_rate_bottom=0.0): + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + fill_rate=NumberRange( + start_of_range=fill_rate_bottom, end_of_range=fill_rate_top + ), + power_ranges=[ + PowerRange( + start_of_range=power_bottom, + end_of_range=power_top, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +def _system_description(*elements) -> FRBCSystemDescription: + mode = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heat", + elements=list(elements), + abnormal_condition_only=False, + ) + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=START, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=[mode], + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=True, + provides_fill_level_target_profile=False, + provides_usage_forecast=True, + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + ), + ) + + +#: Below 50 a mode draws up to 1000 W, above it up to 3000 W, filling at most 1/150 per second either way. +#: At a constant 500 W it fills 3 per quarter hour below 50, and 1 per quarter hour above it. +LOW = _element(0, 50, power_top=1000.0, fill_rate_top=1 / 150) +HIGH = _element(50, 100, power_top=3000.0, fill_rate_top=1 / 150) + + +def _schedule(values, step=QUARTER_HOUR) -> dict: + return { + "start": START.isoformat(), + "duration": pd.Timedelta(step * len(values)).isoformat(), + "values": values, + } + + +def _factors(instructions) -> list[tuple[datetime, float]]: + return [(i.execution_time, i.operation_mode_factor) for i in instructions] + + +def test_a_constant_power_is_instructed_anew_where_the_fill_level_crosses_into_another_element(): + """500 W is half of what the lower element draws at full tilt, and a sixth of the upper one's. + + From 45, three quarter hours of filling at 3 each take the buffer past 50, + where the same 500 W needs another factor, which is a new instruction although the power did not change. + """ + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + 2 * QUARTER_HOUR, pytest.approx(1 / 6)), + ] + + +def test_usage_brings_the_running_fill_level_down_across_a_boundary(): + """Filling 1 per quarter hour above 50 while 4 is used leaves 3 less each step, so from 55 the buffer drops below 50.""" + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_leakage_brings_the_running_fill_level_down_across_a_boundary(): + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + leakage_behaviour=_leakage((0, 100, 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_a_usage_column_takes_the_place_of_a_forecast(): + """A caller holding its usage per step in the frame, as TUNES does, passes it per second.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=55.0 + ) + assert _factors(instructions)[1] == ( + START + 2 * QUARTER_HOUR, + pytest.approx(0.5), + ) + + +@pytest.mark.parametrize( + "step", [timedelta(minutes=5), QUARTER_HOUR, timedelta(hours=1)] +) +def test_a_schedule_is_read_at_the_resolution_it_comes_in(step): + """FlexMeasures answers at the power sensor's resolution, which was taken to be a quarter hour whatever it was. + + Twelve hourly values were laid on forty-eight quarter hours, and pandas refused, so no instruction went out. + """ + instructions = fm_schedule_to_instructions( + _schedule([0.0] * 6 + [1000.0] * 6, step=step), + _system_description(_element(0, 100, power_top=1000.0, fill_rate_top=1e-9)), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.0)), + (START + 6 * step, pytest.approx(1.0)), + ] + + +def test_the_instruction_asks_for_the_factor_that_draws_the_power(): + """A mode running 800 to 1200 W at one fill rate, asked for 1200 W, is asked for factor 1, which draws 1200 W.""" + instructions = fm_schedule_to_instructions( + _schedule([1200.0]), + _system_description( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=2e-3, + ) + ), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [(START, pytest.approx(1.0))] + + +# --------------------------------------------------------------------------- +# The storage efficiency +# --------------------------------------------------------------------------- + + +def test_the_storage_efficiency_compounds_a_leakage_rate_per_second(): + """A 10 kWh buffer losing 1.5 kWh a day keeps 99.844% per quarter hour, about 92.8% over twelve hours.""" + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage((0, 10, 1.5 / 86400)), resolution=QUARTER_HOUR + ) + assert efficiency == pytest.approx(100 * (1 - 1.5 / 86400 / 10) ** 900, rel=1e-12) + assert 100 * (efficiency / 100) ** 48 == pytest.approx(92.774, abs=1e-3) + + +# --------------------------------------------------------------------------- +# The handler +# --------------------------------------------------------------------------- + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._fm_client.server_version_at_least = AsyncMock( + side_effect=lambda minimum: _server_version_at_least("1.1.0", minimum) + ) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=45.0 + ) + frbc._fm_client.trigger_and_get_schedule = AsyncMock( + return_value=_schedule([500.0] * 4) + ) + frbc.send_instruction_batch = AsyncMock(return_value=0) + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_leakage_and_usage(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description, resolution=timedelta(minutes=5)) + older, latest = _leakage((0, 100, 1e-4)), _leakage((0, 100, 2e-4)) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(latest.message_id)] = latest + forecast = _usage((timedelta(hours=1), 1e-3)) + frbc._usage_forecast_history[str(forecast.message_id)] = forecast + + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + + kwargs = translate.call_args.kwargs + assert kwargs["leakage_behaviour"] is latest + assert kwargs["usage_forecast"] is forecast + + +@pytest.mark.asyncio +async def test_a_measurement_covers_the_handlers_own_resolution(): + """A power measurement for one step of a five-minute grid is not posted as covering a quarter hour.""" + system_description = _system_description(LOW, HIGH) + frbc = _frbc( + system_description, resolution=timedelta(minutes=5), fill_rate_sensor_id=6 + ) + actuator = system_description.actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=0.5, + ) + ) + durations = { + (call.args[:1] or (call.kwargs.get("sensor_id"),))[0]: call.kwargs["duration"] + for call in frbc._fm_client.post_sensor_data.await_args_list + } + assert durations == {1: timedelta(minutes=5), 6: timedelta(minutes=5)} + + +def test_a_schedule_without_values_asks_for_nothing(): + """Its step would be its duration over no values, so there is none, and no instruction either.""" + assert ( + fm_schedule_to_instructions( + {"start": START.isoformat(), "duration": "PT0H", "values": []}, + _system_description(LOW, HIGH), + initial_fill_level=45.0, + ) + == [] + ) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 41b788c3..ef02599f 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -480,3 +480,43 @@ async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( if c.kwargs.get("sensor_id") == 12 ] assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) + + +@pytest.mark.asyncio +async def test_planning_reads_its_usage_forecast_back_per_second( + cem_in_frbc_control_type, +): + """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" + from unittest.mock import patch + + from s2python.frbc import FRBCStorageStatus + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + def sensor_data(sensor_id, start=None, duration=None, **kwargs): + # As many quarter hours as were asked for: the fixture's fixed day of values fits no other window. + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + # The usage forecast as posted, 900 per quarter hour, and anything else as a plain efficiency. + value = 900.0 if sensor_id == 12 else 1.0 + return { + "start": str(start), + "duration": str(duration), + "values": [value] * steps, + } + + fm_client.get_sensor_data.side_effect = sensor_data + # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. + fm_client.get_schedule.return_value = {"values": [0.0] * 24} + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_tunes.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule() + + frame = translate.call_args.args[0] + usage = list(frame["usage_forecast"].dropna()) + assert usage and usage == [pytest.approx(1.0)] * len(usage) diff --git a/tests/s2/test_frbc_utils.py b/tests/s2/test_frbc_utils.py index e9461821..13d29c4e 100644 --- a/tests/s2/test_frbc_utils.py +++ b/tests/s2/test_frbc_utils.py @@ -12,7 +12,6 @@ clamp_distance, fm_schedule_to_instructions, get_unique_id, - op_mode_compute_factor, power_to_fill_rate_with_metrics, ) @@ -39,28 +38,44 @@ def example_op_mode_elem(default_power_range): @pytest.mark.parametrize( - "fill_rate_range, input_fill_rate, expected_factor", + "power_range, fill_rate_range, power, expected_factor", [ - ((0.0, 1.0), 0.0, 0.0), - ((0.0, 1.0), 0.5, 0.5), - ((0.0, 1.0), 1.0, 1.0), - ((2.0, 4.0), 3.0, 0.5), - ((1.0, 1.0), 1.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 100.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 150.0, 0.5), + ((100.0, 200.0), (0.0, 1.0), 200.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 250.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 50.0, 0.0), + # One fill rate, a range of power: the factor still follows from the power. + # Worked back out of the fill rate instead, this asked for the bottom of the power range. + ((800.0, 1200.0), (2.0, 2.0), 1200.0, 1.0), + ((800.0, 1200.0), (2.0, 2.0), 1000.0, 0.5), + # One power, a range of fill rate: every factor draws this power, so the one that fills most is asked for. + ((1000.0, 1000.0), (1.0, 2.0), 1000.0, 1.0), + ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 0.0), + # One power and one fill rate, as an on-off actuator states them: the factor makes no difference. + ((1000.0, 1000.0), (2.0, 2.0), 1000.0, 0.0), ], ) -def test_op_mode_compute_factor( - fill_rate_range, input_fill_rate, expected_factor, default_power_range +def test_the_factor_is_the_one_that_draws_the_power_asked_for( + power_range, fill_rate_range, power, expected_factor ): - op_mode_elem = FRBCOperationModeElement( + element = FRBCOperationModeElement( fill_level_range=NumberRange(start_of_range=0.0, end_of_range=1.0), fill_rate=NumberRange( start_of_range=fill_rate_range[0], end_of_range=fill_rate_range[1] ), - power_ranges=default_power_range, - running_costs=NumberRange(start_of_range=1.0, end_of_range=1.0), + power_ranges=[ + PowerRange( + start_of_range=power_range[0], + end_of_range=power_range[1], + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], ) - - factor = op_mode_compute_factor(op_mode_elem, fill_rate=input_fill_rate) + operation_mode = FRBCOperationMode( + id=get_unique_id(), elements=[element], abnormal_condition_only=False + ) + *_, factor = power_to_fill_rate_with_metrics(operation_mode, power, 0.5) assert np.isclose(factor, expected_factor) @@ -150,7 +165,7 @@ def test_power_to_fill_rate_with_metrics_comprehensive( ) # --- Call the method under test --- - fill_rate, fill_penalty, power_penalty, efficiency, element = ( + fill_rate, fill_penalty, power_penalty, efficiency, element, _factor = ( power_to_fill_rate_with_metrics(op_mode, test_power, test_fill_level) ) From d0ae8ce6aed82716a4f819c40f9b25f289f7c65b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:45:49 +0100 Subject: [PATCH 16/87] docs(changelog): every rate per second under the instructions Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index a8cb28a7..f92543e1 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -38,9 +38,20 @@ Changed * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] +Removed +------- + +* ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] + Fixed ----- +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast`` [`#258 `__] +* A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes [`#258 `__] +* An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] +* A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] +* An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] * ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] From 23872889dc4171f4fccb5e6226fe50a0c0cb4d07 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:07:49 +0100 Subject: [PATCH 17/87] fix(FRBC): follow each step as a resource manager does, which review showed it did not An adversarial review built its own integrator from S2's definitions, a second at a time, and found the previous commit agreed with it only while nothing changed within a step. Where something did, it did not: - A step that crossed from one element into another was integrated as if it had not: from 46, filling at 0.02 per second below 50 and 0.01 above, a quarter hour at factor 1 reaches 57, where holding the first element said 64, and at an hour's step the client thought a buffer full that was at 84. The same held for leakage. - Usage was sampled at the instant a step began, where FlexMeasures planned with the mean over the step, so a five-minute draw in each quarter hour counted fully or not at all, and a forecast starting off the step counted as nothing. follow_operation_mode now follows a step a minute at a time, as a resource manager follows an instruction: the mode and the factor are what the instruction fixes, while the element of the mode and of the leakage behaviour, and the usage a forecast expects, are each taken where and when they apply. The fill level is held between its bounds throughout, rather than only at the end of a step. Also from the review: - An instruction no longer goes out for a solver's own rounding: near factor 0, math.isclose without an absolute tolerance is exact equality, so 800 W give or take a ten-millionth from a mode running 800 to 1200 W sent 30 instructions in twelve hours instead of one. - A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, rather than a NaN that pinned the running fill level at the top of its bounds for the rest of the schedule. - FillRateBasedControlTUNES reads its usage forecast back divided by the fill-level scale it posted it with, which left it ten times too small at TUNES' default scale, and plans with its latest leakage behaviour. The previous commit message called TUNES' change "for the better"; without these it was not. - One power and a fill-rate range listed downwards asks for the top of the range, where operation_mode_conversion_efficiency declares the efficiency FlexMeasures plans with, so the instruction and the plan agree. - FRBCSimple refuses a schedule_duration that is not a whole number of resolution steps: FlexMeasures answers with whole steps only, so the step, read as the duration over the number of values, would come out wrong. - The handler's storage efficiency is what remains after one step of its own resolution, and FlexMeasures reads it per step of the leakage sensor's resolution, so the two have to agree. That is now said where the sensor is taken and where the value is posted. Tests cover each of these, and each of the four mutations the review found surviving, among them an on/off heat pump told to switch off: both modes ask for factor 0, and only the mode tells them apart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 6 +- .../s2/control_types/FRBC/frbc_simple.py | 12 ++ .../s2/control_types/FRBC/frbc_tunes.py | 15 +- .../s2/control_types/FRBC/utils.py | 86 +++++++-- tests/s2/test_frbc_time_base.py | 179 ++++++++++++++++++ tests/s2/test_frbc_tunes.py | 24 ++- tests/s2/test_frbc_utils.py | 5 +- 7 files changed, 303 insertions(+), 24 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index f92543e1..260e969a 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,9 +46,11 @@ Removed Fixed ----- -* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast`` [`#258 `__] +* ``FillRateBasedControlTUNES`` reads back the usage forecast it posted in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FRBCSimple`` refuses a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed a minute at a time, as a resource manager follows an instruction, so that a fill level crossing into another element, or a change in usage within a step, is taken when it happens [`#258 `__] * A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is a solver's own rounding, is not a new instruction [`#258 `__] * An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] * A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] * An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 36cbdc18..187981e6 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -165,6 +165,9 @@ def __init__( soc_minima_sensor_id: int | None = None, soc_maxima_sensor_id: int | None = None, usage_forecast_sensor_id: int | None = None, + # Its resolution has to be the handler's resolution: a storage efficiency is what remains after one step, + # computed here for a step of the handler's resolution, + # and FlexMeasures reads it as one step of the sensor's own resolution. leakage_behaviour_sensor_id: int | None = None, # Likewise optional: without a sensor to record it on, no charging efficiency is declared, # and the scheduler plans the storage as if the actuator converted power into stored energy one for one. @@ -211,6 +214,13 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + if schedule_duration % resolution: + # FlexMeasures answers with whole steps of the power sensor's resolution only, + # so a duration between two whole numbers of steps comes back with fewer values than it spans, + # and the schedule's step, read as its duration over its number of values, would come out wrong. + raise ValueError( + f"schedule_duration has to be a whole number of resolution steps, but {schedule_duration} is not a multiple of {resolution}." + ) if schedule_attempts < 1: # A schedule is asked for at least once, or no schedule is asked for at all. raise ValueError( @@ -1028,6 +1038,8 @@ async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): ) return + # What remains after one step of this handler's resolution, + # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. storage_efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 7eef5496..f6b29c6b 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,11 +480,14 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) - # Posted per quarter hour by send_usage_forecast, and read back as such, - # while the schedule translation takes a usage rate per second, as S2 states it. + # Posted per quarter hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, + # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) / (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) + ) / ( + (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) + * self._fill_level_scale + ) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") @@ -507,6 +510,12 @@ async def trigger_schedule(self): # type: ignore[override] system_description, initial_fill_level=soc_at_start / self._fill_level_scale, logger=self._logger, + # What the storage loses by itself, so that the running fill level falls as well as rises. + leakage_behaviour=( + list(self._leakage_behaviour_history.values())[-1] + if self._leakage_behaviour_history + else None + ), ) # Revoke all previous instructions diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 587698ea..a513d9eb 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -45,7 +45,7 @@ def compute_next_fill_level( :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; none where the resource manager has not said. - :param usage_rate: what is drawn from the storage, as the usage forecast expects it over this step. + :param usage_rate: what is drawn from the storage, held over this step. """ leakage_rate = 0.0 if leakage_behaviour is not None: @@ -55,6 +55,55 @@ def compute_next_fill_level( return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() +#: How finely a step is followed, so that an element boundary, or a change in usage, is taken when it happens. +FOLLOWED_EVERY = timedelta(minutes=1) +#: How close two factors are to count as the same instruction: a solver's own rounding is not a new instruction. +FACTOR_TOLERANCE = 1e-6 + + +def follow_operation_mode( + fill_level: float, + operation_mode: FRBCOperationMode, + factor: float, + start: pd.Timestamp, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_forecast: FRBCUsageForecast | None = None, + usage_rate: float | None = None, + bounds: tuple[float, float] | None = None, +) -> float: + """The fill level after following one operation mode at one factor for a step, as a resource manager does. + + An instruction fixes the mode and the factor, not the element: + within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, + and the usage a forecast expects can change. Each is taken where and when it applies, to within FOLLOWED_EVERY, + rather than held at what it was when the step began. + + :param usage_rate: one usage rate for the whole step, which takes the place of the forecast. + :param bounds: the fill levels the storage is held between, at every moment rather than only at the end. + """ + elapsed = timedelta(0) + while elapsed < step: + dt = min(FOLLOWED_EVERY, step - elapsed) + element = element_for_fill_level(operation_mode.elements, fill_level) + low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range + fill_level = compute_next_fill_level( + fill_level, + low + (high - low) * factor, + dt, + leakage_behaviour=leakage_behaviour, + usage_rate=( + usage_rate + if usage_rate is not None + else usage_rate_at(usage_forecast, start + elapsed) + ), + ) + if bounds is not None: + fill_level = max(bounds[0], min(bounds[1], fill_level)) + elapsed += dt + return fill_level + + def usage_rate_at( usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp ) -> float: @@ -209,7 +258,8 @@ def fm_schedule_to_instructions( # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. answer = (str(best_operation_mode.id), operation_mode_factor) if last_instructed is None or not ( - answer[0] == last_instructed[0] and isclose(answer[1], last_instructed[1]) + answer[0] == last_instructed[0] + and isclose(answer[1], last_instructed[1], abs_tol=FACTOR_TOLERANCE) ): instruction = FRBCInstruction( message_id=get_unique_id(), @@ -226,19 +276,24 @@ def fm_schedule_to_instructions( instructions.append(instruction) last_instructed = answer - usage_rate = ( - float(row["usage_forecast"]) - if has_usage_column - else usage_rate_at(usage_forecast, pd.Timestamp(timestamp)) - ) - fill_level = compute_next_fill_level( - fill_level=fill_level, - fill_rate=best_fill_rate, - step=step, + # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: + # a NaN fill level would be held at the top of the bounds, and answered there from then on. + usage_rate = None + if has_usage_column: + usage_rate = ( + 0.0 if pd.isna(row["usage_forecast"]) else float(row["usage_forecast"]) + ) + fill_level = follow_operation_mode( + fill_level, + best_operation_mode, + operation_mode_factor, + pd.Timestamp(timestamp), + step, leakage_behaviour=leakage_behaviour, + usage_forecast=usage_forecast, usage_rate=usage_rate, + bounds=(fill_level_min, fill_level_max), ) - fill_level = max(fill_level_min, min(fill_level_max, fill_level)) return instructions @@ -415,11 +470,12 @@ def power_to_fill_rate_with_metrics( if pr_end == pr_start: # Every factor draws this one power, so the power cannot say which factor is meant. - # The one that fills the storage most for it is asked for, which is either end of the fill-rate range, - # and where that range is a single value too, the factor makes no difference and stays at zero. + # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency + # declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. + # Where the fill-rate range is a single value too, the factor makes no difference and stays at zero. fr_start = best_element.fill_rate.start_of_range fr_end = best_element.fill_rate.end_of_range - factor = 1.0 if fr_end > fr_start else 0.0 + factor = 0.0 if fr_end == fr_start else 1.0 efficiency = 0.0 else: factor = (power - pr_start) / (pr_end - pr_start) diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py index d1e91623..257903a1 100644 --- a/tests/s2/test_frbc_time_base.py +++ b/tests/s2/test_frbc_time_base.py @@ -389,3 +389,182 @@ def test_a_schedule_without_values_asks_for_nothing(): ) == [] ) + + +# --------------------------------------------------------------------------- +# Following a step as a resource manager does (from the review of #258) +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + FOLLOWED_EVERY, + follow_operation_mode, +) + + +def _mode(*elements, label="heat") -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _system_description_of(*modes) -> FRBCSystemDescription: + description = _system_description(LOW) + description.actuators[0].operation_modes = list(modes) + return description + + +def test_an_on_off_heat_pump_is_told_to_switch_off(): + """On and off each run at one power, so both ask for factor 0: only the mode tells them apart.""" + on = _mode( + _element( + 0, + 100, + power_top=883.7, + power_bottom=883.7, + fill_rate_top=1e-3, + fill_rate_bottom=1e-3, + ), + label="on", + ) + off = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0), label="off") + instructions = fm_schedule_to_instructions( + _schedule([883.7, 883.7, 0.0, 0.0]), + _system_description_of(on, off), + initial_fill_level=10.0, + ) + assert [(i.execution_time, str(i.operation_mode)) for i in instructions] == [ + (START, str(on.id)), + (START + 2 * QUARTER_HOUR, str(off.id)), + ] + + +def test_a_solvers_rounding_is_not_a_new_instruction(): + """At the bottom of a modulating range, 800 W give or take a ten-millionth is one instruction, not one per step.""" + modulating = _mode( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=1e-3, + ) + ) + noisy = [800.0 + (1e-7 if i % 2 else -1e-7) for i in range(48)] + instructions = fm_schedule_to_instructions( + _schedule(noisy), _system_description_of(modulating), initial_fill_level=10.0 + ) + assert len(instructions) == 1 + + +def test_usage_is_followed_through_the_step_not_sampled_at_its_start(): + """A five-minute draw in each quarter hour takes five minutes' worth, not none and not fifteen.""" + burst = _usage(*[(timedelta(minutes=5), 0.03), (timedelta(minutes=10), 0.0)] * 4) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=burst + ) == pytest.approx(50.0 - 0.03 * 300) + + +def test_usage_counts_from_where_the_forecast_starts(): + """A forecast from 7 minutes past counts from then: 8 minutes of usage in a step that began on the hour.""" + late = FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START + timedelta(minutes=7), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.01 + ) + ], + ) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=late + ) == pytest.approx(50.0 - 0.01 * 480) + + +#: Fills 0.02 per second at full tilt below 50, half that above, as a heat pump does when its buffer warms. +STEEP = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) +SHALLOW = _element(50, 100, power_top=2000.0, fill_rate_top=0.01) + + +def test_a_step_that_crosses_into_another_element_is_followed_across_it(): + """From 46, factor 1 reaches 50 after 200 s, then fills at half the rate: 57, not the 64 of holding the first element.""" + level = follow_operation_mode( + 46.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), QUARTER_HOUR + ) + assert level == pytest.approx(57.0, abs=0.02 * FOLLOWED_EVERY.total_seconds()) + + +def test_leakage_is_followed_across_an_element_boundary_too(): + """Leaking 0.01 per second above 50 and nothing below, a buffer at 54 settles at 50 rather than draining to 45.""" + leakage = _leakage((0, 50, 0.0), (50, 100, 0.01)) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + level = follow_operation_mode( + 54.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, leakage_behaviour=leakage + ) + assert level == pytest.approx(50.0, abs=0.01 * FOLLOWED_EVERY.total_seconds()) + + +def test_a_gap_in_a_usage_column_is_no_usage_rather_than_a_buffer_full_to_the_top(): + index = pd.date_range(START, periods=4, freq=QUARTER_HOUR) + with_gap = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [float("nan")] * 4}, index=index + ) + without = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [0.0] * 4}, index=index + ) + description = _system_description(LOW, HIGH) + assert _factors( + fm_schedule_to_instructions(with_gap, description, initial_fill_level=45.0) + ) == _factors( + fm_schedule_to_instructions(without, description, initial_fill_level=45.0) + ) + + +def test_a_frame_is_read_at_its_own_step(): + """An hourly frame: an hour at 500 W fills 12, so the buffer is past 50 by the second row.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 3}, + index=pd.date_range(START, periods=3, freq=timedelta(hours=1)), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + timedelta(hours=1), pytest.approx(1 / 6)), + ] + + +def test_a_schedule_duration_between_two_whole_steps_is_refused(): + """FlexMeasures answers with whole steps only, so such a schedule would be read at the wrong step.""" + with pytest.raises(ValueError, match="whole number of resolution steps"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + schedule_duration=timedelta(hours=2, minutes=30), + resolution=timedelta(hours=1), + ) + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_usage_forecast(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description) + older, latest = _usage((timedelta(hours=1), 1e-4)), _usage( + (timedelta(hours=1), 2e-4) + ) + frbc._usage_forecast_history[str(older.message_id)] = older + frbc._usage_forecast_history[str(latest.message_id)] = latest + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["usage_forecast"] is latest diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index ef02599f..ffbe7098 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -489,7 +489,12 @@ async def test_planning_reads_its_usage_forecast_back_per_second( """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" from unittest.mock import patch - from s2python.frbc import FRBCStorageStatus + from s2python.common import NumberRange + from s2python.frbc import ( + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCStorageStatus, + ) cem, fm_client, _ = cem_in_frbc_control_type frbc = cem._control_types_handlers[cem.control_type] @@ -509,6 +514,19 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): } fm_client.get_sensor_data.side_effect = sensor_data + # Not 1, which would hide a missing division: TUNES posts usage scaled into FlexMeasures' units. + frbc._fill_level_scale = 0.1 + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. fm_client.get_schedule.return_value = {"values": [0.0] * 24} with patch( @@ -519,4 +537,6 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): frame = translate.call_args.args[0] usage = list(frame["usage_forecast"].dropna()) - assert usage and usage == [pytest.approx(1.0)] * len(usage) + # 900 per quarter hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. + assert usage and usage == [pytest.approx(10.0)] * len(usage) + assert translate.call_args.kwargs["leakage_behaviour"] is leakage diff --git a/tests/s2/test_frbc_utils.py b/tests/s2/test_frbc_utils.py index 13d29c4e..26061aa8 100644 --- a/tests/s2/test_frbc_utils.py +++ b/tests/s2/test_frbc_utils.py @@ -49,9 +49,10 @@ def example_op_mode_elem(default_power_range): # Worked back out of the fill rate instead, this asked for the bottom of the power range. ((800.0, 1200.0), (2.0, 2.0), 1200.0, 1.0), ((800.0, 1200.0), (2.0, 2.0), 1000.0, 0.5), - # One power, a range of fill rate: every factor draws this power, so the one that fills most is asked for. + # One power, a range of fill rate: every factor draws this power, + # so the top of the fill-rate range is asked for, which is where the efficiency FlexMeasures plans with is declared. ((1000.0, 1000.0), (1.0, 2.0), 1000.0, 1.0), - ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 0.0), + ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 1.0), # One power and one fill rate, as an on-off actuator states them: the factor makes no difference. ((1000.0, 1000.0), (2.0, 2.0), 1000.0, 0.0), ], From 2f8902cb3d5df1b165fddee0b659c7095dfeb0b8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:59:08 +0100 Subject: [PATCH 18/87] fix(example): a heat pump with a buffer that is planned for, instructed, and simulated The shipped example sent no instruction at all, for two reasons that only show with both halves running (findings A1, A2). Its resource manager sent its storage status before its system description, which planned nothing, since planning hangs off a storage status and needs a description to turn a schedule into instructions; and its description was dated 2026-01-15, while the server seeds prices from the day it starts, so a schedule's window had no prices. It also could not have shown steering if it had: one operation mode, with a power range of -0.5 to 0.5 W against a fill rate of -0.5 to 0.5 per second, which is 1.8 MW, over a declared range of 0.4. The resource manager is now a heat pump keeping a hot-water buffer warm. It is idle, or heats at anything from 400 to 1500 W, at a COP of 4 while the buffer is below 6 kWh and 3 above it. It declares its leakage, warmer losing more; a day of hot-water use with showers in the morning and washing up in the evening; and a comfort profile that pre-heats before the showers. It sends its description first, dated now, and then simulates its buffer from S2's own definitions, written separately from the CEM's code: it follows each instruction from the moment it is due, and reports its fill level every so often, which asks for a new schedule. That simulation is a second opinion on the CEM's arithmetic, and a test holds the two to agreement across an element boundary. The server gives the handler a sensor for every declaration (fill rate, band, usage, leakage, charging efficiency), so each can be seen next to the schedule planned against it, and plans from now rather than a day after the description. Its connection handling is split out of the aiohttp handler, so a test can serve the shipped resource manager with the shipped CEM, over an in-memory websocket, with only FlexMeasures stood in for. Two library changes the example needed: - A system description that arrives after the latest storage status, or that changes what the device can do, is planned for against that status (finding A3), rather than waiting for a status that a resource manager whose fill level holds steady may never send. It plans without re-posting the old status as a new measurement. - A schedule never starts before now. Planning happens on every storage status while a description can be days old, so a schedule anchored to the description alone slid into the past, and its instructions all arrived as one overdue burst. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/__init__.py | 20 +- .../s2/control_types/FRBC/frbc_simple.py | 16 +- .../s2/script/websockets_client.py | 449 +++++++++++++----- .../s2/script/websockets_server.py | 121 +++-- tests/s2/test_example_end_to_end.py | 342 +++++++++++++ tests/s2/test_frbc_simple_planning.py | 57 +++ 6 files changed, 807 insertions(+), 198 deletions(-) create mode 100644 tests/s2/test_example_end_to_end.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 830f3b0d..b4c695e3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -230,11 +230,21 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message - # A description says what the device can do, not what it is doing, so it is stored and - # no schedule is asked for here. - # Planning happens on a storage status, which is what carries a new state of charge, - # and which is the only one of the two a resource manager is required to keep sending - # (see handle_storage_status). + # A description says what the device can do, and a storage status what it holds now, + # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), + # but a description that arrives after the latest status, or that changes what the device can do, + # is answered with a schedule against that latest status. + # Otherwise a resource manager that sends its status first, or whose fill level holds steady + # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. + if self._storage_status_history: + request_number = self._count_request(STORAGE_STATUS_REQUESTS) + task = asyncio.create_task( + self.trigger_schedule( + system_description_id, request_number=request_number + ) + ) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCUsageForecast) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 187981e6..a8655a77 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -291,8 +291,15 @@ def align_to_resolution(self, moment: datetime) -> datetime: return moment.replace(microsecond=0) - timedelta(seconds=seconds_into_the_day) def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary.""" - return self.align_to_resolution(valid_from + self._valid_from_shift) + """Where a schedule asked for at valid_from starts: shifted, never before now, then floored to a boundary. + + Never before now, since planning happens on every storage status while a description may be days old: + anchored to that description alone, every status in a long session was answered with the same window, + which slid into the past, so its instructions were all due already and arrived as one burst. + """ + return self.align_to_resolution( + max(valid_from + self._valid_from_shift, self.now()) + ) async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None @@ -309,9 +316,8 @@ async def send_storage_status( # The state of charge this status reports is what a schedule starts from, so this is # where a schedule is asked for. - # A system description is needed to turn that schedule into instructions, and a resource - # manager sends one before its first status, so its absence here means the two have gone - # out of step rather than that nothing has arrived yet. + # A system description is needed to turn that schedule into instructions. + # Without one yet, nothing is planned here, and the description plans against this status when it arrives. if not self._system_description_history: self._logger.warning( "Cannot ask for a schedule: no FRBC.SystemDescription has arrived yet." diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 942f1988..3413e4b4 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -1,6 +1,20 @@ +"""An example resource manager: a heat pump keeping a hot-water buffer warm, speaking S2's FRBC to a CEM. + +Run it by path, next to the example CEM server, as the CEM documentation shows. +It declares what the heat pump can do and what its buffer needs, then simulates that buffer: +it follows each instruction from the moment it is due, and reports its fill level every so often, +which is what asks the CEM for a new schedule. + +The buffer is simulated from S2's own definitions, and not with the CEM's code, +so that it is a second opinion on where the instructions take the storage rather than a copy of the first. +""" + +from __future__ import annotations + import asyncio -from datetime import datetime, timezone -from zoneinfo import ZoneInfo +import os +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone import aiohttp from s2python.common import ( @@ -21,19 +35,241 @@ from s2python.frbc import ( FRBCActuatorDescription, FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, FRBCInstruction, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCOperationModeElement, FRBCStorageDescription, FRBCStorageStatus, FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, ) from flexmeasures_client.s2.utils import get_unique_id +#: The range the buffer's fill level should stay in, in kilowatt hours of stored heat. +FILL_LEVEL_RANGE = (2.0, 10.0) +#: Below this fill level the water is cold enough for the heat pump to run at a COP of 4; above it, 3. +COLD_BELOW = 6.0 +#: What the buffer holds when the simulation starts, and what it can physically hold at most. +STARTING_FILL_LEVEL = 5.0 +PHYSICAL_CAPACITY = 12.0 +#: How often the simulated buffer reports its fill level, in seconds of wall-clock time. +STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) +ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC + + +def heat(watts: float) -> float: + """A rate of heat in watts as a fill rate in kilowatt hours per second, which is how S2 states a fill rate.""" + return watts / 3.6e6 + + +def per_second(kwh_per_hour: float) -> float: + return kwh_per_hour / 3600 + + +def _element( + lowest: float, + highest: float, + power: tuple[float, float], + heat_in_watts: tuple[float, float], +) -> FRBCOperationModeElement: + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=heat(heat_in_watts[0]), end_of_range=heat(heat_in_watts[1]) + ), + power_ranges=[ + PowerRange( + start_of_range=power[0], + end_of_range=power[1], + commodity_quantity=ELECTRIC_POWER, + ) + ], + ) + + +def operation_modes() -> list[FRBCOperationMode]: + """Idle, or heating at anything from 400 to 1500 W, at a COP of 4 while the buffer is cold and 3 once it is warm.""" + idle = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="idle", + elements=[_element(0, PHYSICAL_CAPACITY, (0, 0), (0, 0))], + abnormal_condition_only=False, + ) + heating = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heating", + elements=[ + _element(0, COLD_BELOW, (400, 1500), (4 * 400, 4 * 1500)), + _element(COLD_BELOW, PHYSICAL_CAPACITY, (400, 1500), (3 * 400, 3 * 1500)), + ], + abnormal_condition_only=False, + ) + return [idle, heating] + + +def system_description( + modes: list[FRBCOperationMode], valid_from: datetime +) -> FRBCSystemDescription: + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=valid_from, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + diagnostic_label="heat pump", + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=modes, + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + diagnostic_label="hot-water buffer", + fill_level_label="kWh of stored heat", + provides_leakage_behaviour=True, + provides_fill_level_target_profile=True, + provides_usage_forecast=True, + fill_level_range=NumberRange( + start_of_range=FILL_LEVEL_RANGE[0], end_of_range=FILL_LEVEL_RANGE[1] + ), + ), + ) + + +def leakage_behaviour(valid_from: datetime) -> FRBCLeakageBehaviour: + """A warmer buffer loses more: 0.8 kWh a day while cold, 1.5 kWh a day once warm.""" + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=valid_from, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=COLD_BELOW), + leakage_rate=per_second(0.8 / 24), + ), + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=COLD_BELOW, end_of_range=PHYSICAL_CAPACITY + ), + leakage_rate=per_second(1.5 / 24), + ), + ], + ) + + +def expected_usage(hour: int) -> float: + """Hot water drawn in a given hour of the day, in kWh: showers in the morning, washing up in the evening.""" + return {7: 3.0, 19: 1.0, 20: 1.0}.get(hour, 0.1) + + +def usage_forecast(start: datetime) -> FRBCUsageForecast: + """A day of hot-water use, hour by hour, from the hour start falls in.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), + usage_rate_expected=per_second(expected_usage((start.hour + h) % 24)), + ) + for h in range(24) + ], + ) + + +def fill_level_target_profile(start: datetime) -> FRBCFillLevelTargetProfile: + """Kept within the declared range, and at least 6 kWh in the hour before the morning showers.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration(root=3600 * 1000), + fill_level_range=NumberRange( + start_of_range=6.0 if (start.hour + h) % 24 == 6 else 3.0, + end_of_range=FILL_LEVEL_RANGE[1], + ), + ) + for h in range(24) + ], + ) + + +@dataclass +class Buffer: + """The simulated buffer: where its fill level goes, following S2's definitions. + + S2 states every rate per second, in the fill level's own units: + a fill rate raises the fill level, and usage and leakage lower it. + An operation mode's fill rate is interpolated between the ends of its element's range by the factor, + from the element whose fill-level range holds the fill level. + """ + + modes: dict[str, FRBCOperationMode] + leakage: FRBCLeakageBehaviour + fill_level: float = STARTING_FILL_LEVEL + mode_id: str | None = None + factor: float = 0.0 + at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @staticmethod + def _holding(elements, fill_level: float): + """The element whose range holds the fill level, the upper one at a shared boundary, the nearest outside.""" + lowest = min(e.fill_level_range.start_of_range for e in elements) + highest = max(e.fill_level_range.end_of_range for e in elements) + level = min(max(fill_level, lowest), highest) + return max( + ( + e + for e in elements + if e.fill_level_range.start_of_range + <= level + <= e.fill_level_range.end_of_range + ), + key=lambda e: ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ), + ) + + def rates(self, moment: datetime) -> tuple[float, float, float]: + """Fill, usage and leakage, each per second, at this fill level and moment.""" + fill = 0.0 + if self.mode_id is not None: + element = self._holding(self.modes[self.mode_id].elements, self.fill_level) + low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range + fill = low + (high - low) * self.factor + usage = per_second(expected_usage(moment.hour)) + leakage = self._holding(self.leakage.elements, self.fill_level).leakage_rate + return fill, usage, leakage + + def advance(self, until: datetime, step: timedelta = timedelta(minutes=1)) -> None: + """Move the fill level forward to a moment, a minute at a time, so that an element boundary is crossed in time.""" + while self.at < until: + dt = min(step, until - self.at) + fill, usage, leakage = self.rates(self.at) + self.fill_level += (fill - usage - leakage) * dt.total_seconds() + self.fill_level = min(max(self.fill_level, 0.0), PHYSICAL_CAPACITY) + self.at += dt + + def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: + self.advance(moment) + self.mode_id = str(instruction.operation_mode) + self.factor = instruction.operation_mode_factor + async def apply_at_execution_time( - ws, instruction: FRBCInstruction, previous_operation_mode_id=None + ws, + instruction: FRBCInstruction, + previous_operation_mode_id=None, + buffer: Buffer | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -49,6 +285,9 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) + now = datetime.now(timezone.utc) + if buffer is not None: + buffer.follow(instruction, now) status = FRBCActuatorStatus( message_id=get_unique_id(), actuator_id=instruction.actuator_id, @@ -57,13 +296,13 @@ async def apply_at_execution_time( previous_operation_mode_id=previous_operation_mode_id, # The moment the transition happened, so that what is recorded about it is placed there, # rather than wherever the message is read. - transition_timestamp=datetime.now(timezone.utc), + transition_timestamp=now, ) print("SENDING: FRBC.ActuatorStatus") await ws.send_str(status.model_dump_json()) -async def answer_instructions(ws) -> None: +async def answer_instructions(ws, buffer: Buffer | None = None) -> None: """Acknowledge every instruction, and report the transition each one asks for when it is due.""" active_operation_mode_id = None @@ -81,138 +320,100 @@ async def answer_instructions(ws) -> None: ) # In a task of its own, so that an instruction due later does not hold up the ones behind it. asyncio.create_task( - apply_at_execution_time(ws, instruction, active_operation_mode_id) + apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) ) active_operation_mode_id = instruction.operation_mode -async def main_s2(): - async with aiohttp.ClientSession() as session: - async with session.ws_connect("http://localhost:8080/ws") as ws: - message = Handshake( - message_id=get_unique_id(), - role=EnergyManagementRole.RM, - supported_protocol_versions=["0.1.0"], - ) - - print("SENDING: HANDSHAKE") - - await ws.send_str(message.model_dump_json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # send resource manager details - - resource_manager_details_message = ResourceManagerDetails( - message_id=get_unique_id(), - resource_id=get_unique_id(), - roles=[ - Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) - ], - instruction_processing_delay=Duration(root=1), - available_control_types=[ - ControlType.FILL_RATE_BASED_CONTROL, - ControlType.NO_SELECTION, - ], - provides_forecast=True, - provides_power_measurement_types=[ - CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - ], +async def report_fill_level( + ws, buffer: Buffer, interval: float = STATUS_INTERVAL +) -> None: + """Report where the buffer is every interval, which is what asks the CEM for a new schedule.""" + while True: + await asyncio.sleep(interval) + buffer.advance(datetime.now(timezone.utc)) + print(f"SENDING: FRBC.StorageStatus at {buffer.fill_level:.2f} kWh") + await ws.send_str( + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level ).model_dump_json() + ) - print("SENDING: ResourceManagerDetails") - - await ws.send_str(resource_manager_details_message) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # Let the server activate the control type - control_type = await ws.receive() - - control_type = control_type.json() - - print("RECEIVING: ", control_type) - - reception_status = ReceptionStatus( - subject_message_id=control_type["message_id"], - status=ReceptionStatusValues.OK, - ) - - print("SENDING: ReceptionStatus") - await ws.send_str(reception_status.model_dump_json()) - - electric_power = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - - # send storage status - storage_status = FRBCStorageStatus( - message_id=get_unique_id(), present_fill_level=0.4 - ) - - print("SENDING: FRBC.StorageStatus") - - await ws.send_str(storage_status.model_dump_json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - # send system description - operation_mode_element = FRBCOperationModeElement( - fill_level_range=NumberRange(start_of_range=0.0, end_of_range=0.5), - fill_rate=NumberRange(start_of_range=-0.5, end_of_range=0.5), - power_ranges=[ - PowerRange( - start_of_range=-0.5, - end_of_range=0.5, - commodity_quantity=electric_power, - ) - ], - ) +async def receive_until(ws, message_type: str) -> dict: + """The next message of a type, past the reception statuses that arrive in between.""" + while True: + message = (await ws.receive()).json() + print("RECEIVED: ", message) + if message.get("message_type") == message_type: + return message - operation_mode = FRBCOperationMode( - id=get_unique_id(), - elements=[operation_mode_element], - abnormal_condition_only=False, - diagnostic_label="full operating range of a perfectly efficiency battery", - ) - actuator = FRBCActuatorDescription( - id=get_unique_id(), - supported_commodities=[Commodity.ELECTRICITY], - operation_modes=[operation_mode], - transitions=[], - timers=[], - ) +async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: + """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. - storage = FRBCStorageDescription( - provides_leakage_behaviour=False, - provides_fill_level_target_profile=False, - provides_usage_forecast=False, - fill_level_range=NumberRange(start_of_range=0.05, end_of_range=0.45), - ) - - valid_from = datetime(2026, 1, 15, 20, tzinfo=ZoneInfo("Europe/Amsterdam")) - - system_description_message = FRBCSystemDescription( - message_id=get_unique_id(), - valid_from=valid_from, - actuators=[actuator], - storage=storage, - ).model_dump_json() + In the order S2 suggests: the system description first, then the storage's status. + Returns the simulated buffer, for whoever wants to look at where the instructions took it. + """ + await ws.send_str( + Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.RM, + supported_protocol_versions=["0.1.0"], + ).model_dump_json() + ) + await receive_until(ws, "HandshakeResponse") + + await ws.send_str( + ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + name="heat pump with a hot-water buffer", + roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + instruction_processing_delay=Duration(root=1000), + available_control_types=[ + ControlType.FILL_RATE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + provides_forecast=True, + provides_power_measurement_types=[ELECTRIC_POWER], + ).model_dump_json() + ) + select = await receive_until(ws, "SelectControlType") + await ws.send_str( + ReceptionStatus( + subject_message_id=select["message_id"], status=ReceptionStatusValues.OK + ).model_dump_json() + ) - print("SENDING: FRBC.SystemDescription") + now = datetime.now(timezone.utc) + modes = operation_modes() + leakage = leakage_behaviour(now) + buffer = Buffer(modes={str(m.id): m for m in modes}, leakage=leakage, at=now) + for message in ( + system_description(modes, now), + leakage, + usage_forecast(now), + fill_level_target_profile(now), + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level + ), + ): + print(f"SENDING: {message.message_type}") + await ws.send_str(message.model_dump_json()) + + reporting = asyncio.create_task(report_fill_level(ws, buffer, interval)) + try: + await answer_instructions(ws, buffer) + finally: + reporting.cancel() + return buffer - await ws.send_str(system_description_message) - await answer_instructions(ws) +async def main_s2(): + async with aiohttp.ClientSession() as session: + async with session.ws_connect("http://localhost:8080/ws") as ws: + await run_resource_manager(ws) async def main_websocket(): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 108a475b..175cc7d7 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -28,9 +28,28 @@ # system description that asked for it and runs for twelve hours, so two days would just do. SEED_PRICE_DAYS = 3 +#: Each sensor the site needs: its unit, its resolution, and any attributes. +#: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. +SITE_SENSORS: dict[str, tuple[str, str, dict]] = { + "price": ("EUR/kWh", "PT15M", {}), + # S2 states a power as positive when it is consumed, + # and this sensor holds both an actuator's power and the schedule planned for it. + # Left unsaid, FlexMeasures reads a power sensor the other way round, + # and flips a schedule's sign on the way in, which puts the two in opposite directions. + "power": ("kW", "PT15M", {"consumption_is_positive": True}), + "state of charge": ("kWh", "PT0M", {}), + "RM discharge": ("dimensionless", "PT15M", {}), + "fill rate": ("kWh/s", "PT15M", {}), + "soc minima": ("kWh", "PT15M", {}), + "soc maxima": ("kWh", "PT15M", {}), + "usage forecast": ("kWh/s", "PT15M", {}), + "storage efficiency": ("%", "PT15M", {}), + "charging efficiency": ("dimensionless", "PT15M", {}), +} + # Keys under which the configured site is kept on the aiohttp application. FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) -SENSORS = web.AppKey("sensors", tuple) +SENSORS = web.AppKey("sensors", dict) async def rm_details_watchdog(ws, cem: CEM): @@ -54,7 +73,8 @@ async def rm_details_watchdog(ws, cem: CEM): cem._logger.debug(f"CONTROL TYPE: {cem._control_type}") - # after this, schedule will be triggered on reception of a new system description + # From here on, a schedule is asked for on each storage status, + # and on a system description that arrives after one. async def websocket_producer(ws, cem: CEM): @@ -103,19 +123,34 @@ async def websocket_handler(request): # calls, which takes about ten seconds against a local server, while s2-python's # reception-status timeout is five: a resource manager gave up before the CEM had read its # first message. - fm_client = request.app[FM_CLIENT] - price_sensor, power_sensor, soc_sensor, rm_discharge_sensor = request.app[SENSORS] + await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + return ws + +async def serve_resource_manager( + ws, fm_client: FlexMeasuresClient, sensors: dict[str, dict] +) -> None: + """Act as the CEM for one resource manager connected over ws, until it disconnects.""" cem = CEM( - sensor_id=power_sensor["id"], + sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, ) frbc = FRBCSimple( - power_sensor_id=power_sensor["id"], - price_sensor_id=price_sensor["id"], - soc_sensor_id=soc_sensor["id"], - rm_discharge_sensor_id=rm_discharge_sensor["id"], + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + # Where the resource manager's declarations are recorded, so that each can be seen next to the schedule planned against it. + fill_rate_sensor_id=sensors["fill rate"]["id"], + soc_minima_sensor_id=sensors["soc minima"]["id"], + soc_maxima_sensor_id=sensors["soc maxima"]["id"], + usage_forecast_sensor_id=sensors["usage forecast"]["id"], + leakage_behaviour_sensor_id=sensors["storage efficiency"]["id"], + charging_efficiency_sensor_id=sensors["charging efficiency"]["id"], + timezone=SITE_TIMEZONE, + # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. + valid_from_shift=timedelta(0), ) cem.register_control_type(frbc) @@ -126,12 +161,10 @@ async def websocket_handler(request): rm_details_watchdog(ws, cem), ) - return ws - async def configure_site( site_name: str, fm_client: FlexMeasuresClient -) -> tuple[dict, dict, dict, dict]: +) -> dict[str, dict]: """Create or update the site asset and its sensors, and seed it with demo prices.""" account = await fm_client.get_account() if account is None: @@ -164,32 +197,21 @@ async def configure_site( site_asset["id"], site_asset_specs, parse_json_fields=True ) - sensors = site_asset.get("sensors", []) - price_sensor = None - power_sensor = None - soc_sensor = None - rm_discharge_sensor = None - for sensor in sensors: - if sensor["name"] == "price": - price_sensor = sensor - elif sensor["name"] == "power": - power_sensor = sensor - elif sensor["name"] == "state of charge": - soc_sensor = sensor - elif sensor["name"] == "RM discharge": - rm_discharge_sensor = sensor - - if price_sensor is None: - price_sensor = await fm_client.add_sensor( - name="price", - event_resolution="PT15M", - unit="EUR/kWh", + existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} + sensors: dict[str, dict] = {} + for name, (unit, resolution, attributes) in SITE_SENSORS.items(): + sensors[name] = existing.get(name) or await fm_client.add_sensor( + name=name, + event_resolution=resolution, + unit=unit, generic_asset_id=site_asset["id"], timezone=SITE_TIMEZONE, + **({"attributes": attributes} if attributes else {}), ) + # Demo prices, so that a freshly configured site has something to schedule against. # One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - # plans a day and a half ahead of the moment a resource manager sends its system description: + # plans ahead of the moment a resource manager reports its storage: # a fixed date here goes stale the day after it is written, and the scheduler then fails the # job with UnknownPricesException rather than returning a schedule. one_day = [ @@ -201,42 +223,13 @@ async def configure_site( hour=0, minute=0, second=0, microsecond=0 ) await fm_client.post_sensor_data( - sensor_id=price_sensor["id"], + sensor_id=sensors["price"]["id"], start=today, duration=timedelta(days=SEED_PRICE_DAYS), values=one_day * SEED_PRICE_DAYS, unit="EUR/kWh", ) - if power_sensor is None: - power_sensor = await fm_client.add_sensor( - name="power", - event_resolution="PT15M", - unit="kW", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. - attributes={"consumption_is_positive": True}, - ) - if soc_sensor is None: - soc_sensor = await fm_client.add_sensor( - name="state of charge", - event_resolution="PT0M", - unit="kWh", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - if rm_discharge_sensor is None: - rm_discharge_sensor = await fm_client.add_sensor( - name="RM discharge", - event_resolution="PT15M", - unit="dimensionless", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - return price_sensor, power_sensor, soc_sensor, rm_discharge_sensor + return sensors async def configure_site_once(app: web.Application) -> None: diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py new file mode 100644 index 00000000..9b97b64e --- /dev/null +++ b/tests/s2/test_example_end_to_end.py @@ -0,0 +1,342 @@ +"""The shipped example, both halves, end to end: the resource manager it ships, served by the CEM it ships. + +Only FlexMeasures is stood in for, and the websocket between the two is held in memory. +The example used to send no instruction at all, for two reasons the halves only show together: +the resource manager sent its storage status before its system description, which planned nothing, +and its description was dated ten months before the prices the server seeds. +""" + +from __future__ import annotations + +import asyncio +import gc +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import aiohttp +import pandas as pd +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.script import websockets_server as server +from flexmeasures_client.s2.script.websockets_client import run_resource_manager + + +class _Text: + type = aiohttp.WSMsgType.TEXT + + def __init__(self, data: str): + self.data = data + + def json(self) -> dict: + return json.loads(self.data) + + +#: Put on an end's inbox, it ends that end's stream of messages, as a websocket closed by its peer does. +_CLOSED = object() + + +class _End: + """One end of an in-memory websocket: what it sends arrives at the other end.""" + + def __init__(self, inbox: asyncio.Queue, outbox: asyncio.Queue): + self._inbox, self._outbox = inbox, outbox + self.received: list[dict] = [] + + async def send_str(self, data: str) -> None: + await self._outbox.put(data) + + async def send_json(self, message) -> None: + await self._outbox.put(json.dumps(message)) + + async def receive(self) -> _Text: + data = await self._inbox.get() + self.received.append(json.loads(data)) + return _Text(data) + + def __aiter__(self): + return self + + async def __anext__(self) -> _Text: + data = await self._inbox.get() + if data is _CLOSED: + raise StopAsyncIteration + self.received.append(json.loads(data)) + return _Text(data) + + async def close(self) -> None: + pass + + def of_type(self, message_type: str) -> list[dict]: + return [m for m in self.received if m.get("message_type") == message_type] + + +def _connected() -> tuple[_End, _End]: + to_resource_manager, to_cem = asyncio.Queue(), asyncio.Queue() + return _End(to_resource_manager, to_cem), _End(to_cem, to_resource_manager) + + +#: What the stand-in FlexMeasures plans, per quarter hour: idle, then full heat, then minimum heat. +SHAPE = [0.0, 0.0, 1500.0, 1500.0, 400.0, 400.0, 0.0, 0.0] + + +def _flexmeasures() -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.server_version_at_least = AsyncMock(return_value=True) + fm.get_sensor = AsyncMock( + return_value={"attributes": {"consumption_is_positive": True}} + ) + + def schedule(start, duration, **kwargs) -> dict: + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + return { + "start": pd.Timestamp(start).isoformat(), + "duration": pd.Timedelta(duration).isoformat(), + "values": [SHAPE[i % len(SHAPE)] for i in range(steps)], + "scheduler_info": {"scheduler": "StorageScheduler"}, + } + + fm.trigger_and_get_schedule = AsyncMock(side_effect=lambda **kw: schedule(**kw)) + return fm + + +async def _until(condition, timeout: float = 15.0) -> None: + async def wait(): + while not condition(): + await asyncio.sleep(0.05) + + await asyncio.wait_for(wait(), timeout) + + +@pytest.mark.asyncio +async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + unhandled: list[dict] = [] + loop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: unhandled.append(context)) + began = datetime.now(timezone.utc) + + tasks = [ + asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), + asyncio.create_task(run_resource_manager(resource_manager, interval=0.3)), + ] + try: + # Two storage statuses past the first, each a new plan, so that replanning is part of what is checked. + await _until(lambda: fm.trigger_and_get_schedule.await_count >= 3) + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 3) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + gc.collect() + loop.set_exception_handler(None) + + instructions = resource_manager.of_type("FRBC.Instruction") + distinct = { + (i["operation_mode"], round(i["operation_mode_factor"], 6)) + for i in instructions + } + assert len(distinct) >= 2, instructions + + refused = [ + m for m in resource_manager.of_type("ReceptionStatus") if m["status"] != "OK" + ] + assert refused == [] + + start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] + assert began - timedelta(minutes=15) < start <= began + timedelta(seconds=5) + assert unhandled == [] + + # Every declaration reached the sensor the server made for it, to be seen next to the schedule planned against it. + posted_to = { + call.args[0] if call.args else call.kwargs.get("sensor_id") + for call in fm.post_sensor_data.await_args_list + + fm.post_measurements.await_args_list + } + for name in ( + "power", + "state of charge", + "fill rate", + "soc minima", + "soc maxima", + "usage forecast", + "storage efficiency", + "charging efficiency", + ): + assert sensors[name]["id"] in posted_to, name + + +# --------------------------------------------------------------------------- +# The example's own buffer, as a second opinion on the CEM's arithmetic +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + compute_next_fill_level, + usage_rate_at, +) +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + Buffer, + heat, + leakage_behaviour, + operation_modes, + per_second, + usage_forecast, +) + +NOON = datetime(2024, 1, 1, 12, tzinfo=timezone.utc) + + +def _heating_buffer(fill_level: float) -> Buffer: + modes = operation_modes() + heating = next(m for m in modes if m.diagnostic_label == "heating") + return Buffer( + modes={str(m.id): m for m in modes}, + leakage=leakage_behaviour(NOON), + fill_level=fill_level, + mode_id=str(heating.id), + factor=1.0, + at=NOON, + ) + + +def test_the_example_buffer_follows_s2s_definitions(): + """A quarter hour at full heat from a cold 2 kWh: 6 kW of heat in, a tenth of a kWh an hour drawn, and a cold buffer's leakage.""" + buffer = _heating_buffer(2.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 2.0 + (heat(6000) - per_second(0.1) - per_second(0.8 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_the_example_and_the_cem_agree_on_where_the_buffer_goes(): + """Two integrators written separately from S2's definitions, one in the CEM and one in the example's buffer. + + While no boundary between elements is crossed they hold every rate constant over a step, and so must agree. + """ + buffer = _heating_buffer(2.0) + element_fill_rate = heat(6000) + cem_fill_level = 2.0 + for step in range(2): + moment = pd.Timestamp(NOON + step * timedelta(minutes=15)) + cem_fill_level = compute_next_fill_level( + cem_fill_level, + element_fill_rate, + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage_rate=usage_rate_at(usage_forecast(NOON), moment), + ) + buffer.advance(NOON + timedelta(minutes=30)) + assert cem_fill_level < 6.0 # no element boundary crossed + assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) + + +def test_a_warm_buffer_heats_at_the_warm_coefficient(): + """At 7 kWh the buffer is past the cold element, so full heat delivers 4.5 kW, and it leaks as a warm buffer does.""" + buffer = _heating_buffer(7.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 7.0 + (heat(4500) - per_second(0.1) - per_second(1.5 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_an_instruction_takes_over_from_where_the_last_one_left_the_buffer(): + """Idle for a quarter hour, losing only what is drawn and what leaks, then a quarter hour at full heat.""" + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + buffer = _heating_buffer(4.0) + buffer.mode_id, buffer.factor = None, 0.0 # idle until told otherwise + heating = next(m for m in buffer.modes.values() if m.diagnostic_label == "heating") + buffer.follow( + FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=heating.id, + operation_mode_factor=1.0, + execution_time=NOON + timedelta(minutes=15), + abnormal_condition=False, + ), + NOON + timedelta(minutes=15), + ) + buffer.advance(NOON + timedelta(minutes=30)) + losses = per_second(0.1) + per_second(0.8 / 24) + expected = 4.0 - losses * 900 + (heat(6000) - losses) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +@pytest.mark.asyncio +async def test_the_resource_manager_hands_back_its_buffer_when_the_cem_goes_away(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + serving = asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)) + managing = asyncio.create_task(run_resource_manager(resource_manager, interval=60)) + try: + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 1) + await resource_manager._inbox.put(_CLOSED) + buffer = await asyncio.wait_for(managing, timeout=5) + finally: + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + assert 0 <= buffer.fill_level <= 12 + + +@pytest.mark.asyncio +async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_today(): + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [{"id": 50, "name": "price"}], + } + ] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + + sensors = await server.configure_site(server.SITE_NAME, fm) + + assert set(sensors) == set(server.SITE_SENSORS) + assert sensors["price"]["id"] == 50 # kept, not made again + made = {call.kwargs["name"]: call.kwargs for call in fm.add_sensor.await_args_list} + assert "price" not in made + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["fill rate"]["unit"] == "kWh/s" + assert made["soc minima"]["unit"] == "kWh" + prices = fm.post_sensor_data.await_args.kwargs + assert prices["sensor_id"] == 50 + assert prices["start"].date() == datetime.now(prices["start"].tzinfo).date() + + +def test_the_example_and_the_cem_agree_across_an_element_boundary(): + """Both follow a step a minute at a time, so from 5 kWh, through the cold element and into the warm one, they agree.""" + from flexmeasures_client.s2.control_types.FRBC.utils import follow_operation_mode + + buffer = _heating_buffer(5.0) + heating = buffer.modes[buffer.mode_id] + cem_fill_level = 5.0 + for step in range(4): + cem_fill_level = follow_operation_mode( + cem_fill_level, + heating, + 1.0, + pd.Timestamp(NOON + step * timedelta(minutes=15)), + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage_forecast=usage_forecast(NOON), + ) + buffer.advance(NOON + timedelta(hours=1)) + assert ( + cem_fill_level > 6.0 + ) # the boundary between the cold and the warm element was crossed + assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index be2da4e0..e3c0a745 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -31,6 +31,8 @@ from flexmeasures_client.s2.utils import get_unique_id POWER_SENSOR_ID = 1 +#: A clock set before the descriptions below are valid, so that their own times decide where a schedule starts. +BEFORE_THE_EXAMPLES = datetime(2022, 12, 1, tzinfo=timezone.utc) SOC_SENSOR_ID = 3 @@ -175,6 +177,7 @@ async def test_a_storage_status_before_any_description_still_records_the_measure def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): """An unaligned start leaves FlexMeasures with no prices for the window, and the job fails.""" frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES start = frbc.schedule_start(datetime.fromisoformat(valid_from)) @@ -187,6 +190,7 @@ def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): @pytest.mark.asyncio async def test_the_schedule_is_asked_for_from_an_aligned_start(): frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES await frbc.handle_message(system_description("2022-12-02T09:07:33.512+00:00")) await settle(frbc) @@ -277,3 +281,56 @@ def test_a_handler_turns_away_a_model_it_does_not_handle(): supported_protocol_versions=["0.0.2-beta"], ) assert not frbc_simple().supports_message(handshake) + + +def test_a_schedule_never_starts_before_now(): + """A description days old still plans from now, on a boundary, rather than a window that has already passed.""" + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_status_before_its_description_is_planned_for_when_the_description_arrives(): + """The order the example resource manager used to send them in, which left it without a single instruction.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + frbc._fm_client.trigger_and_get_schedule.assert_awaited_once() + assert not frbc._sending_queue.empty() + + +@pytest.mark.asyncio +async def test_a_changed_description_is_planned_for_against_the_latest_status(): + """What the device can do changed, so the plan made for what it could do before no longer answers.""" + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 1 + + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_the_soc_is_not_reposted_when_a_description_asks_for_a_schedule(): + """The latest status is planned from, not recorded again as if it had been measured now.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + posts_after_the_status = frbc._fm_client.post_measurements.await_count + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.post_measurements.await_count == posts_after_the_status From fa3dc5925fe0efc178e8a48152b6d7a80ca62cf2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:16:13 +0100 Subject: [PATCH 19/87] docs(changelog): the example, and the planning triggers it needed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 260e969a..95b9c2ae 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,6 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, and plans from now [`#259 `__] Removed ------- @@ -73,9 +74,9 @@ Fixed * An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks, and recorded none of what they declared [`#244 `__] * A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] -* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, so that the window it schedules for has prices [`#241 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, instead of when a system description arrives, so a resource manager that sends its description before its first status is planned for at all [`#240 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window [`#240 `__] +* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] +* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status or changes what the device can do, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from before now, which a description days old made it slide into [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] From 10f1cc724d3a4c3b13e94297544a98add8d68499 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:20:40 +0100 Subject: [PATCH 20/87] fix(FRBC): a state of charge in any supported energy unit reaches FlexMeasures FRBCSimple passed its energy_unit straight through as the flex model's soc-unit, while FlexMeasures validates soc-unit with OneOf(["kWh", "MWh"]). So four of the six units fill_rate_in_watts documents, J, kJ, MJ and Wh, made every trigger a 422, and the storage got no schedule at all (finding B2). The plan for this was to refuse those units when the handler is made. The carve source answers it better, by converting joules at the edge, and that is what happens now, for every supported unit: the state of charge and its two hard bounds go to FlexMeasures in kWh, converted from the recorded unit with the table fill_rate_in_watts already keeps, while kWh and MWh pass through as they are. The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself, and the bounds the schedule is read back against stay in the recorded unit, since the operation modes are stated in it. An energy unit outside that table is now refused when the handler is made, where it used to surface only at planning time, as a logged failure to record a charging efficiency. The handler for that failure is gone, since nothing can now reach it. Not changed, and now pinned by a test: the measured state-of-charge sensor is never named in the flex model (finding B13 stands). Named there, FlexMeasures would save the state of charge it plans onto that sensor, beside the measurements. The carve source does name it, and the carve will have to leave that line behind. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 48 +++++++---- tests/s2/test_frbc_charging_efficiency.py | 79 +++++++++++++++---- 2 files changed, 96 insertions(+), 31 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a8655a77..110b3f26 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -30,6 +30,7 @@ from flexmeasures_client.exceptions import JobFailedError from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( + WATTS_PER_ENERGY_UNIT_PER_SECOND, clip_fill_level_target_profile, element_for_fill_level, fm_schedule_to_instructions, @@ -248,6 +249,12 @@ def __init__( ) self._hard_bound_factor = hard_bound_factor self.power_unit = power_unit + if energy_unit not in WATTS_PER_ENERGY_UNIT_PER_SECOND: + # A fill rate is per second of this unit, and has to be compared with the watts S2 states a power in, + # so a unit that cannot be converted to watts cannot be planned with. + raise ValueError( + f"energy_unit has to be one of {', '.join(WATTS_PER_ENERGY_UNIT_PER_SECOND)}, but {energy_unit!r} was given." + ) self.energy_unit = energy_unit # delay the start of the schedule from the time `valid_from` @@ -417,6 +424,15 @@ def _latest(history: dict): """The message a history holds that arrived last, or None while it holds none.""" return list(history.values())[-1] if history else None + def _soc_unit(self) -> tuple[str, float]: + """The unit a state of charge goes to FlexMeasures in, and what a fill level is multiplied by to be in it.""" + if self.energy_unit in ("kWh", "MWh"): + return self.energy_unit, 1.0 + return "kWh", ( + WATTS_PER_ENERGY_UNIT_PER_SECOND[self.energy_unit] + / WATTS_PER_ENERGY_UNIT_PER_SECOND["kWh"] + ) + def _latest_fill_level(self) -> float | None: """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" if not self._storage_status_history: @@ -585,19 +601,13 @@ async def _record_charging_efficiency( # One actuator, as everywhere else in this handler. actuator = system_description.actuators[0] efficiencies: dict[str, float] = {} - try: - for operation_mode in actuator.operation_modes: - efficiency = operation_mode_conversion_efficiency( - operation_mode, self._fill_level_scale, self.energy_unit, fill_level - ) - if efficiency is not None: - efficiencies[str(operation_mode.id)] = efficiency - except NotImplementedError: - self._logger.exception( - "Not asking for a schedule against a charging efficiency: a fill rate in %s cannot be compared against the watts S2 states a power range in.", - self.fill_rate_unit(), + # The energy unit is one fill_rate_in_watts knows, which the constructor saw to. + for operation_mode in actuator.operation_modes: + efficiency = operation_mode_conversion_efficiency( + operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) - return False + if efficiency is not None: + efficiencies[str(operation_mode.id)] = efficiency if not efficiencies: # Every mode is idle, or every mode only feeds back, and neither converts drawn power into anything. @@ -828,12 +838,16 @@ async def trigger_schedule( # and the window a charging efficiency has to be recorded over to be planned against. start = self.schedule_start(system_description.valid_from) + # FlexMeasures takes a state of charge in kWh or MWh only, so a fill level recorded in another energy unit + # is converted here, at the edge, rather than refused, and the bounds read back below stay in the recorded unit. + # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. + soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { - # The same unit the fill level is recorded in, or the scheduler would plan against bounds a thousandfold off the measurements it plans from. - "soc-unit": self.energy_unit, - "soc-at-start": soc_at_start, # TODO: use forecast of the SOC instead - "soc-min": soc_min, - "soc-max": soc_max, + "soc-unit": soc_unit, + "soc-at-start": soc_at_start + * to_soc_unit, # TODO: use forecast of the SOC instead + "soc-min": soc_min * to_soc_unit, + "soc-max": soc_max * to_soc_unit, } # What the resource manager declared about its storage, each entry only where a sensor holds it: # an entry naming a sensor that was never configured has the scheduler plan against nothing at all. diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 7217e9cd..6baea1dd 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -301,20 +301,16 @@ async def test_a_fill_level_scale_is_applied_to_the_efficiency(): ] -@pytest.mark.asyncio -async def test_an_energy_unit_that_cannot_be_compared_to_watts_is_not_guessed_at( - caplog, -): - """A fill rate in a unit nothing converts is refused, rather than recorded thousandfold off.""" - system_description = _heater() - frbc = _frbc(system_description, energy_unit="therm") - - with caplog.at_level(logging.ERROR): - flex_model = await _asked_for(frbc, system_description) - - assert "charging-efficiency" not in flex_model - assert _recordings(frbc) == [] - assert "cannot be compared" in caplog.text +def test_an_energy_unit_that_cannot_be_compared_to_watts_is_refused_at_once(): + """A fill rate in a unit nothing converts would be recorded thousandfold off, so the handler is not made at all.""" + with pytest.raises(ValueError, match="energy_unit has to be one of"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + energy_unit="therm", + ) def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): @@ -397,3 +393,58 @@ async def test_a_schedule_is_asked_for_against_the_efficiency_where_it_starts(): assert _recordings(frbc)[0]["values"] == [pytest.approx(warm)] assert warm != pytest.approx(cold) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "energy_unit, soc_unit, per_unit", + [ + ("MJ", "kWh", 1 / 3.6), + ("Wh", "kWh", 1 / 1000), + ("J", "kWh", 1 / 3.6e6), + ("kWh", "kWh", 1.0), + ("MWh", "MWh", 1.0), + ], +) +async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( + energy_unit, soc_unit, per_unit +): + """FlexMeasures takes kWh or MWh only, so any other energy unit is converted at the edge, not refused by the server.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit=energy_unit) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-unit"] == soc_unit + assert flex_model["soc-at-start"] == pytest.approx(FILL_LEVEL_AT_START * per_unit) + assert flex_model["soc-max"] == pytest.approx(1.5 * RANGE_TOP * per_unit) + + +@pytest.mark.asyncio +async def test_the_bounds_read_back_stay_in_the_recorded_unit(): + """Only what goes to FlexMeasures is converted: the schedule is matched against the operation modes in their own units.""" + from unittest.mock import patch + + system_description = _heater() + frbc = _frbc(system_description, energy_unit="MJ") + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["fill_level_bounds"] == ( + pytest.approx(0.0), + pytest.approx(1.5 * RANGE_TOP), + ) + + +@pytest.mark.asyncio +async def test_the_measured_state_of_charge_is_not_named_in_the_flex_model(): + """The state of charge goes out as a number, never as the sensor it is measured on. + + Named there, FlexMeasures would save the state of charge it plans onto that sensor, + beside the measurements, where a later schedule would read a plan as if it had been measured. + """ + system_description = _heater() + frbc = _frbc(system_description) + flex_model = await _asked_for(frbc, system_description) + assert "state-of-charge" not in flex_model + assert isinstance(flex_model["soc-at-start"], float) From 5ba8ee240b9ec111bd1a6d03f53acc0f507d43bf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:21:06 +0100 Subject: [PATCH 21/87] docs(changelog): which energy units a handler takes, and what FlexMeasures is sent Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 95b9c2ae..ee497dba 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -17,7 +17,7 @@ Added ----- * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] -* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] +* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] * ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] From 324e7f26d6c2359b8247f4919a9c0ed9cf17d713 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:43:41 +0100 Subject: [PATCH 22/87] fix(FRBC): follow each step exactly, between the moments anything changes A second round of review showed that following a step a minute at a time traded one class of error for four others, and cost ten times the time: - Usage was sampled at the start of each minute. Half-minute draws starting on the half minute were never seen at all, and the same draws starting on the minute were counted twice. - A level settling on an element boundary, where filling stops above it, flipped across that boundary every step: 127 instructions in 144 steps, against one. - An element narrower than a minute's travel was skipped: a level that stops at 50 was carried to 80. - A 48-hour schedule against the largest forecast S2 allows took 1.7 s, on the event loop, scanning the forecast once a minute. Every rate is constant between the moments the usage changes and the moments the level reaches an element boundary, of the mode or of the leakage behaviour, or one of its bounds. So follow_operation_mode now steps from one such moment to the next and is exact. A level that reaches a boundary whose far side would push it back stays at that boundary, as a real storage does, and so does a level at one of its bounds, which rises again from there once filling wins. The usage forecast is read through a UsageProfile, built once per schedule and looked up by bisection. Also from the second round: - FillRateBasedControlTUNES posted its usage forecast as an amount per quarter hour labelled MW. FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour, so it planned a quarter of the real draw. It now posts per hour, and reads back accordingly. - FRBCSimple refuses a resolution or schedule duration that is not positive, where a zero resolution raised a ZeroDivisionError. - The factor tolerance's justification was wrong: FlexMeasures rounds a schedule to six decimals of MW, so what it absorbs is the noise of converting to watts, not a solver's rounding. The tolerance stays; the comment and the changelog now say what it is for. - One of the first round's mutations had stopped applying after black split its line, so the gate's count overstated what was checked. The harness now rejects a mutant that cannot run, and every mutation is anchored to the formatted code. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 8 +- .../s2/control_types/FRBC/frbc_simple.py | 4 + .../s2/control_types/FRBC/frbc_tunes.py | 15 +- .../s2/control_types/FRBC/utils.py | 192 +++++++++++++----- tests/s2/test_frbc_time_base.py | 165 ++++++++++++++- tests/s2/test_frbc_tunes.py | 25 ++- 6 files changed, 329 insertions(+), 80 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 260e969a..cce80717 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,11 +46,11 @@ Removed Fixed ----- -* ``FillRateBasedControlTUNES`` reads back the usage forecast it posted in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] -* ``FRBCSimple`` refuses a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] -* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed a minute at a time, as a resource manager follows an instruction, so that a fill level crossing into another element, or a change in usage within a step, is taken when it happens [`#258 `__] +* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] * A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is a solver's own rounding, is not a new instruction [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#258 `__] * An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] * A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] * An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 187981e6..efcb97e5 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -214,6 +214,10 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + if resolution <= timedelta(0) or schedule_duration <= timedelta(0): + raise ValueError( + f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." + ) if schedule_duration % resolution: # FlexMeasures answers with whole steps of the power sensor's resolution only, # so a duration between two whole numbers of steps comes back with fewer values than it spans, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index f6b29c6b..d13e5708 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,14 +480,11 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) - # Posted per quarter hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, + # Posted per hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) / ( - (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) - * self._fill_level_scale - ) + ) / ((pd.Timedelta(hours=1) / pd.Timedelta(seconds=1)) * self._fill_level_scale) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") @@ -697,15 +694,17 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): fill_level_scale=self._fill_level_scale, ) - # Scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) - scale = timedelta(minutes=15) / timedelta(seconds=1) + # From per second, as S2 states a usage rate, to per hour, which is what a value labelled MW means: + # FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour. + # Posted per quarter hour instead, it was planned as a quarter of the real draw. + scale = timedelta(hours=1) / timedelta(seconds=1) scaled_usage_forecast = usage_forecast_series * scale await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), - unit=POWER_UNIT, # e.g. [0, 100] MW/(15 min) + unit=POWER_UNIT, # a rate per hour, as MW is duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index a513d9eb..49f07c2a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,3 +1,4 @@ +import bisect import logging import math from datetime import timedelta @@ -55,12 +56,73 @@ def compute_next_fill_level( return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() -#: How finely a step is followed, so that an element boundary, or a change in usage, is taken when it happens. -FOLLOWED_EVERY = timedelta(minutes=1) -#: How close two factors are to count as the same instruction: a solver's own rounding is not a new instruction. +#: How close two factors are to count as the same instruction. +#: It absorbs the float noise of converting a schedule from MW to W, which is not a new instruction. FACTOR_TOLERANCE = 1e-6 +class UsageProfile: + """What a usage forecast expects, per second, at any moment: piecewise constant, looked up by bisection. + + A forecast is a run of elements, each lasting its own duration from where the one before it ended. + Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. + Built once per schedule, since a forecast may hold hundreds of elements and a schedule asks about it often. + """ + + def __init__( + self, usage_forecast: FRBCUsageForecast | None = None, rate: float | None = None + ): + #: The moments the rate changes at, and the rate from each of them on; one constant rate has none. + self._changes: list[pd.Timestamp] = [] + self._rates: list[float] = [] + self._before = 0.0 if rate is None else rate + if usage_forecast is None: + return + moment = pd.Timestamp(usage_forecast.start_time) + for element in usage_forecast.elements: + self._changes.append(moment) + self._rates.append(element.usage_rate_expected) + moment = moment + timedelta(milliseconds=element.duration.root) + self._changes.append(moment) + self._rates.append(0.0) + + def rate_at(self, moment: pd.Timestamp) -> float: + index = bisect.bisect_right(self._changes, moment) - 1 + return self._before if index < 0 else self._rates[index] + + def next_change_after(self, moment: pd.Timestamp) -> pd.Timestamp | None: + index = bisect.bisect_right(self._changes, moment) + return self._changes[index] if index < len(self._changes) else None + + +def usage_rate_at( + usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp +) -> float: + """The usage rate a forecast expects at a moment, per second, as S2 states it (see UsageProfile).""" + return UsageProfile(usage_forecast).rate_at(moment) + + +def _sides(elements, fill_level: float): + """The element that applies just below a fill level, and the one just above it. + + Inside an element both are that element; at a boundary two elements share, they are the two neighbours. + """ + above = element_for_fill_level(elements, fill_level) + below_candidates = [ + e + for e in elements + if e.fill_level_range.start_of_range + < fill_level + <= e.fill_level_range.end_of_range + ] + below = ( + max(below_candidates, key=lambda e: e.fill_level_range.start_of_range) + if below_candidates + else above + ) + return below, above + + def follow_operation_mode( fill_level: float, operation_mode: FRBCOperationMode, @@ -68,59 +130,79 @@ def follow_operation_mode( start: pd.Timestamp, step: timedelta, leakage_behaviour: FRBCLeakageBehaviour | None = None, - usage_forecast: FRBCUsageForecast | None = None, - usage_rate: float | None = None, + usage: UsageProfile | None = None, bounds: tuple[float, float] | None = None, ) -> float: """The fill level after following one operation mode at one factor for a step, as a resource manager does. An instruction fixes the mode and the factor, not the element: within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, - and the usage a forecast expects can change. Each is taken where and when it applies, to within FOLLOWED_EVERY, - rather than held at what it was when the step began. - - :param usage_rate: one usage rate for the whole step, which takes the place of the forecast. - :param bounds: the fill levels the storage is held between, at every moment rather than only at the end. + and the usage a forecast expects can change. + Every rate is constant between such moments, so the step is followed exactly, from one of them to the next. + A level that reaches a boundary its far side would push it back from, as where a heat pump cannot heat above it, + stays at that boundary, as a real storage does, rather than flipping across it. + So does a level that reaches one of the bounds. + + :param usage: what is drawn from the storage, per second; none where nothing is known. + :param bounds: the fill levels the storage is held between. """ - elapsed = timedelta(0) - while elapsed < step: - dt = min(FOLLOWED_EVERY, step - elapsed) - element = element_for_fill_level(operation_mode.elements, fill_level) - low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range - fill_level = compute_next_fill_level( - fill_level, - low + (high - low) * factor, - dt, - leakage_behaviour=leakage_behaviour, - usage_rate=( - usage_rate - if usage_rate is not None - else usage_rate_at(usage_forecast, start + elapsed) - ), - ) - if bounds is not None: - fill_level = max(bounds[0], min(bounds[1], fill_level)) - elapsed += dt - return fill_level - - -def usage_rate_at( - usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp -) -> float: - """The usage rate a forecast expects at a moment, per second, as S2 states it. + usage = usage or UsageProfile() + leakage_elements = leakage_behaviour.elements if leakage_behaviour else [] + lowest, highest = bounds if bounds is not None else (-math.inf, math.inf) + boundaries = sorted( + { + edge + for e in list(operation_mode.elements) + list(leakage_elements) + for edge in ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ) + } + | ({lowest, highest} - {-math.inf, math.inf}) + ) - A forecast is a run of elements, each lasting its own duration from where the one before it ended. - Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. - """ - if usage_forecast is None: - return 0.0 - element_start = pd.Timestamp(usage_forecast.start_time) - for element in usage_forecast.elements: - element_end = element_start + timedelta(milliseconds=element.duration.root) - if element_start <= moment < element_end: - return element.usage_rate_expected - element_start = element_end - return 0.0 + def net_rate(mode_element, leakage_element, moment) -> float: + low, high = ( + mode_element.fill_rate.start_of_range, + mode_element.fill_rate.end_of_range, + ) + leakage = leakage_element.leakage_rate if leakage_element is not None else 0.0 + return low + (high - low) * factor - usage.rate_at(moment) - leakage + + moment, end = pd.Timestamp(start), pd.Timestamp(start) + step + fill_level = min(max(fill_level, lowest), highest) + while moment < end: + mode_below, mode_above = _sides(operation_mode.elements, fill_level) + leak_below, leak_above = ( + _sides(leakage_elements, fill_level) if leakage_elements else (None, None) + ) + rising = net_rate(mode_above, leak_above, moment) + falling = net_rate(mode_below, leak_below, moment) + if rising > 0 and fill_level < highest: + rate = rising + elif falling < 0 and fill_level > lowest: + rate = falling + else: + rate = 0.0 # held where it is, at a boundary or a bound, until something changes + horizon = end + change = usage.next_change_after(moment) + if change is not None and change < horizon: + horizon = change + target = None + if rate > 0: + target = next((b for b in boundaries if b > fill_level), None) + elif rate < 0: + target = next((b for b in reversed(boundaries) if b < fill_level), None) + if target is not None: + # Compared in seconds first: a slow rate can put a boundary further off than a timedelta can say. + seconds_to_target = (target - fill_level) / rate + if seconds_to_target < (horizon - moment).total_seconds(): + fill_level = target + moment = moment + pd.Timedelta(seconds=seconds_to_target) + continue + fill_level += rate * (horizon - moment).total_seconds() + moment = horizon + return min(max(fill_level, lowest), highest) def fm_schedule_to_instructions( @@ -202,6 +284,7 @@ def fm_schedule_to_instructions( instructions: list[FRBCInstruction] = [] last_instructed: tuple[str, float] | None = None + forecast_usage = UsageProfile(usage_forecast) fill_level = initial_fill_level has_usage_column = "usage_forecast" in schedule.columns @@ -278,10 +361,14 @@ def fm_schedule_to_instructions( # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: # a NaN fill level would be held at the top of the bounds, and answered there from then on. - usage_rate = None + usage = forecast_usage if has_usage_column: - usage_rate = ( - 0.0 if pd.isna(row["usage_forecast"]) else float(row["usage_forecast"]) + usage = UsageProfile( + rate=( + 0.0 + if pd.isna(row["usage_forecast"]) + else float(row["usage_forecast"]) + ) ) fill_level = follow_operation_mode( fill_level, @@ -290,8 +377,7 @@ def fm_schedule_to_instructions( pd.Timestamp(timestamp), step, leakage_behaviour=leakage_behaviour, - usage_forecast=usage_forecast, - usage_rate=usage_rate, + usage=usage, bounds=(fill_level_min, fill_level_max), ) diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py index 257903a1..eb940673 100644 --- a/tests/s2/test_frbc_time_base.py +++ b/tests/s2/test_frbc_time_base.py @@ -396,7 +396,7 @@ def test_a_schedule_without_values_asks_for_nothing(): # --------------------------------------------------------------------------- from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 - FOLLOWED_EVERY, + UsageProfile, follow_operation_mode, ) @@ -465,7 +465,7 @@ def test_usage_is_followed_through_the_step_not_sampled_at_its_start(): burst = _usage(*[(timedelta(minutes=5), 0.03), (timedelta(minutes=10), 0.0)] * 4) idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) assert follow_operation_mode( - 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=burst + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(burst) ) == pytest.approx(50.0 - 0.03 * 300) @@ -482,7 +482,7 @@ def test_usage_counts_from_where_the_forecast_starts(): ) idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) assert follow_operation_mode( - 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=late + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(late) ) == pytest.approx(50.0 - 0.01 * 480) @@ -496,7 +496,7 @@ def test_a_step_that_crosses_into_another_element_is_followed_across_it(): level = follow_operation_mode( 46.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), QUARTER_HOUR ) - assert level == pytest.approx(57.0, abs=0.02 * FOLLOWED_EVERY.total_seconds()) + assert level == pytest.approx(57.0) def test_leakage_is_followed_across_an_element_boundary_too(): @@ -506,7 +506,7 @@ def test_leakage_is_followed_across_an_element_boundary_too(): level = follow_operation_mode( 54.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, leakage_behaviour=leakage ) - assert level == pytest.approx(50.0, abs=0.01 * FOLLOWED_EVERY.total_seconds()) + assert level == pytest.approx(50.0) def test_a_gap_in_a_usage_column_is_no_usage_rather_than_a_buffer_full_to_the_top(): @@ -568,3 +568,158 @@ async def test_the_handler_translates_with_the_latest_usage_forecast(): ) as translate: await frbc.trigger_schedule(system_description.message_id) assert translate.call_args.kwargs["usage_forecast"] is latest + + +# --------------------------------------------------------------------------- +# Followed exactly, between the moments anything changes (second round of review) +# --------------------------------------------------------------------------- + +IDLE = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + + +@pytest.mark.parametrize("offset", [timedelta(0), timedelta(seconds=30)]) +def test_a_draw_shorter_than_a_minute_counts_exactly_once(offset): + """Half-minute draws every two minutes, starting on the minute or off it: a quarter hour holds 7 or 8 of them, exactly.""" + elements = [] + if offset: + elements.append((offset, 0.0)) + while sum((d for d, _ in elements), timedelta(0)) < QUARTER_HOUR: + elements += [(timedelta(seconds=30), 0.1), (timedelta(seconds=90), 0.0)] + pulses = _usage(*elements) + drawn_for = sum( + ( + min(QUARTER_HOUR, start + d) - start + for start, (d, rate) in zip( + [ + sum((e[0] for e in elements[:k]), timedelta(0)) + for k in range(len(elements)) + ], + elements, + ) + if rate and start < QUARTER_HOUR + ), + timedelta(0), + ) + level = follow_operation_mode( + 50.0, IDLE, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(pulses) + ) + assert level == pytest.approx(50.0 - 0.1 * drawn_for.total_seconds()) + + +def test_a_level_held_at_a_boundary_stays_there_rather_than_flipping_across(): + """Filling below 50 and not above, while leaking everywhere: the level settles at 50, and one instruction serves 12 hours.""" + below = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) + above = _element(50, 100, power_top=1000.0, power_bottom=200.0, fill_rate_top=0.0) + leaking = _leakage((0, 100, 0.01)) + instructions = fm_schedule_to_instructions( + _schedule([600.0] * 48), + _system_description_of(_mode(below, above)), + initial_fill_level=40.0, + leakage_behaviour=leaking, + ) + assert len(instructions) <= 2, _factors(instructions) + + +def test_an_element_narrower_than_a_minutes_travel_is_not_skipped(): + """Filling at 1 per second below 50 and not at all above: from 20, the level stops at 50, not 30 past it.""" + stop = _mode( + _element(0, 50, power_top=1000.0, fill_rate_top=1.0), + _element(50, 100, power_top=1000.0, fill_rate_top=0.0), + ) + assert follow_operation_mode( + 20.0, stop, 1.0, pd.Timestamp(START), QUARTER_HOUR + ) == pytest.approx(50.0) + + +def test_a_step_that_is_not_a_whole_number_of_minutes_is_followed_to_its_end(): + level = follow_operation_mode( + 10.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), timedelta(seconds=90) + ) + assert level == pytest.approx(10.0 + 0.02 * 90) + + +def test_a_level_draining_to_the_lower_bound_is_held_there(): + drained = follow_operation_mode( + 5.0, + IDLE, + 0.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(rate=0.01), + bounds=(0.0, 100.0), + ) + assert drained == 0.0 + + +def test_a_small_but_real_change_of_factor_is_instructed(): + """The tolerance absorbs conversion noise, not a change the schedule asks for: 600 W then 605 W is two instructions.""" + modulating = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=1e-6)) + instructions = fm_schedule_to_instructions( + _schedule([600.0, 605.0]), + _system_description_of(modulating), + initial_fill_level=10.0, + ) + assert [round(f, 3) for _, f in _factors(instructions)] == [0.6, 0.605] + + +def test_a_usage_column_takes_precedence_over_a_forecast(): + """A caller that gives both means its column: 4 per quarter hour drawn, not the forecast's nothing.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + with_both = fm_schedule_to_instructions( + frame, + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 0.0)), + ) + assert _factors(with_both)[1] == (START + 2 * QUARTER_HOUR, pytest.approx(0.5)) + + +@pytest.mark.parametrize( + "resolution, duration", + [(timedelta(0), timedelta(hours=12)), (QUARTER_HOUR, timedelta(0))], +) +def test_a_handler_needs_a_positive_resolution_and_duration(resolution, duration): + with pytest.raises(ValueError, match="have to be positive"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + resolution=resolution, + schedule_duration=duration, + ) + + +def test_a_long_schedule_against_a_detailed_forecast_is_translated_quickly(): + """48 hours, at quarter hours, against the largest forecast S2 allows: well under the 1.7 s that stepping each minute took.""" + import time + + detailed = _usage(*[(timedelta(minutes=10), 1e-4 * (i % 3)) for i in range(288)]) + started = time.perf_counter() + fm_schedule_to_instructions( + _schedule([500.0, 0.0] * 96), + _system_description(LOW, HIGH), + initial_fill_level=45.0, + usage_forecast=detailed, + leakage_behaviour=_leakage((0, 50, 1e-4), (50, 100, 2e-4)), + ) + assert time.perf_counter() - started < 0.5 + + +def test_a_level_held_at_its_lower_bound_rises_from_there_once_filling_wins(): + """Drained to 0 in the first 5 minutes, held there while the draw lasts, then filling from 0, not from below it.""" + draw_then_none = _usage((timedelta(minutes=10), 0.02), (timedelta(minutes=5), 0.0)) + filling = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=0.01)) + level = follow_operation_mode( + 3.0, + filling, + 1.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(draw_then_none), + bounds=(0.0, 100.0), + ) + assert level == pytest.approx(0.01 * 300) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index ffbe7098..8363afb0 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -386,8 +386,11 @@ async def test_planning_does_not_stop_on_an_unlabelled_operation_mode( @pytest.mark.asyncio -async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_type): - """S2 states usage per second; TUNES posts it per quarter hour, on a quarter-hour grid, for as long as it was forecast.""" +async def test_a_usage_forecast_is_posted_per_hour(cem_in_frbc_control_type): + """S2 states usage per second; TUNES posts it per hour, labelled MW, on a quarter-hour grid, for as long as it was forecast. + + FlexMeasures reads soc-usage as a power, a rate per hour, so a value per quarter hour would be planned as a quarter of the draw. + """ from s2python.common import Duration from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement @@ -415,7 +418,7 @@ async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_t ] assert len(posts) == 1, fm_client.post_sensor_data.await_args_list values = posts[0].kwargs["values"] - assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) + assert values == [pytest.approx(0.001 * 3600)] * len(values) assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) @@ -454,7 +457,7 @@ async def test_a_usage_forecast_is_posted_from_the_quarter_hour_it_starts_in( async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( cem_in_frbc_control_type, ): - """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths, per quarter hour.""" + """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths per second, per hour.""" from s2python.common import Duration from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement @@ -479,14 +482,14 @@ async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( for c in fm_client.post_sensor_data.await_args_list if c.kwargs.get("sensor_id") == 12 ] - assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) + assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 3600) @pytest.mark.asyncio async def test_planning_reads_its_usage_forecast_back_per_second( cem_in_frbc_control_type, ): - """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" + """TUNES posts a usage forecast per hour; the schedule translation takes a rate per second, as S2 states it.""" from unittest.mock import patch from s2python.common import NumberRange @@ -505,8 +508,8 @@ async def test_planning_reads_its_usage_forecast_back_per_second( def sensor_data(sensor_id, start=None, duration=None, **kwargs): # As many quarter hours as were asked for: the fixture's fixed day of values fits no other window. steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) - # The usage forecast as posted, 900 per quarter hour, and anything else as a plain efficiency. - value = 900.0 if sensor_id == 12 else 1.0 + # The usage forecast as posted, 3600 per hour, and anything else as a plain efficiency. + value = 3600.0 if sensor_id == 12 else 1.0 return { "start": str(start), "duration": str(duration), @@ -526,7 +529,9 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): ) ], ) - frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + older = leakage.model_copy(update={"message_id": get_unique_id()}) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage # the latest # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. fm_client.get_schedule.return_value = {"values": [0.0] * 24} with patch( @@ -537,6 +542,6 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): frame = translate.call_args.args[0] usage = list(frame["usage_forecast"].dropna()) - # 900 per quarter hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. + # 3600 per hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. assert usage and usage == [pytest.approx(10.0)] * len(usage) assert translate.call_args.kwargs["leakage_behaviour"] is leakage From 392ba5008d5d4da60a57ca6150e463d28713301c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:05:45 +0100 Subject: [PATCH 23/87] fix(example): follow only the latest plan, keep the forecasts current, and what review found An adversarial review of the example, and of the two library changes it needed, found the example resource manager following every plan it had ever been sent. Each instruction ran in a task of its own, nothing cancelled the tasks of a replaced batch, and the server sent no revocations. In the live demo against a sandboxed FlexMeasures, thirty instructions for the same quarter hour fired at once when it came, one from each replan. The server now sets send_revocations, so a newer plan revokes what is left of the one before, and the resource manager cancels a revoked instruction it has not yet followed, and acknowledges the revocation. Also from the review: - The leakage, the usage forecast and the comfort profile were sent once, at connect, and the storage efficiency was recorded once, for twelve hours from the message, so planning went without leakage after twelve hours and without usage or a band after a day. The resource manager now sends all three again every S2_EXAMPLE_FORECAST_INTERVAL seconds, from then on, and the server seeds its demo prices again every twelve hours. - In the library, a storage efficiency is now recorded each time a schedule is asked for, over the window it covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. On arrival, it covered the wrong hours, at a fill level nobody had reported yet, and as a share of the top of the range it understated the loss at lower fill levels. - align_to_resolution floored the local clock, which in the repeated hour of an autumn DST change took the first pass of that hour, so a schedule could start 67 minutes in the past. It now floors the instant, in UTC. - A state of charge is posted as a reading at an instant, where it was posted over the minute after it, which FlexMeasures stored as a forecast. - The example declares what it does: transitions between its two modes, which S2 lists as the ones possible; no PowerForecast, since it sends none; and a consumer of electricity rather than a storage of it. - Claims corrected: a description that differs from the one before it, its validity included, asks for a schedule, not only one that "changes what the device can do"; and a schedule starts in the current step, which has begun, not "never before now". - The test that held the example's buffer to the CEM's integrator compared two copies of one discretisation. The CEM is now exact and the buffer steps a minute at a time, so the test bounds their difference by a minute of the rate difference at the crossing, which is what two correct discretisations owe each other. Tests cover everything the review's thirteen mutations found unpinned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 9 +- .../s2/control_types/FRBC/__init__.py | 4 +- .../s2/control_types/FRBC/frbc_simple.py | 76 ++++-- .../s2/control_types/translations.py | 12 +- .../s2/script/websockets_client.py | 94 ++++++- .../s2/script/websockets_server.py | 46 +++- tests/s2/test_example_end_to_end.py | 247 +++++++++++++++++- tests/s2/test_frbc_declared_profiles.py | 73 +++++- tests/s2/test_frbc_element_selection.py | 11 +- tests/s2/test_frbc_simple_planning.py | 61 +++++ tests/s2/test_frbc_soft_band.py | 23 ++ 11 files changed, 599 insertions(+), 57 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b9a0de76..7aec9f68 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,7 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, and plans from now [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] Removed ------- @@ -47,6 +47,9 @@ Removed Fixed ----- +* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels [`#259 `__] +* A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] +* A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] * Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] @@ -75,8 +78,8 @@ Fixed * A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] * The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status or changes what the device can do, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from before now, which a description days old made it slide into [`#240 `__, `#259 `__] +* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from a step before the current one, which a description days old made it slide into [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index b4c695e3..39ef719e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -232,8 +232,10 @@ def handle_system_description( # A description says what the device can do, and a storage status what it holds now, # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), - # but a description that arrives after the latest status, or that changes what the device can do, + # but a description that arrives after the latest status, and differs from the description before it, # is answered with a schedule against that latest status. + # Anything but the message id counts as a difference, its validity included, + # so a resource manager that re-dates its description on every resend asks for a schedule every time. # Otherwise a resource manager that sends its status first, or whose fill level holds steady # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. if self._storage_status_history: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 2eb6d535..d6331642 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -214,6 +214,7 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + self._storage_efficiency_recorded: tuple[datetime, float] | None = None if resolution <= timedelta(0) or schedule_duration <= timedelta(0): raise ValueError( f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." @@ -289,17 +290,22 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - seconds_into_the_day = ( - moment.hour * 3600 + moment.minute * 60 + moment.second - ) % resolution_seconds - return moment.replace(microsecond=0) - timedelta(seconds=seconds_into_the_day) + # Floored as an instant, in UTC, and handed back in the moment's own time zone. + # Flooring the local clock instead took the repeated hour of an autumn DST change for its first pass, + # an hour earlier than meant. For Europe/Amsterdam and a resolution dividing an hour the two agree everywhere else. + seconds = int(moment.timestamp()) + floored = datetime.fromtimestamp( + seconds - seconds % resolution_seconds, tz=ZoneInfo("UTC") + ) + return floored.astimezone(moment.tzinfo) def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, never before now, then floored to a boundary. + """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. - Never before now, since planning happens on every storage status while a description may be days old: + Not into an earlier step, since planning happens on every storage status while a description may be days old: anchored to that description alone, every status in a long session was answered with the same window, which slid into the past, so its instructions were all due already and arrived as one burst. + The step now falls in has begun already, so its first instruction is due at once. """ return self.align_to_resolution( max(valid_from + self._valid_from_shift, self.now()) @@ -315,7 +321,8 @@ async def send_storage_status( # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, - duration=timedelta(minutes=1), + # A reading at an instant, not over the minute after it, which FlexMeasures took for a forecast. + duration=timedelta(0), ) # The state of charge this status reports is what a schedule starts from, so this is @@ -861,7 +868,7 @@ async def trigger_schedule( ) if self._usage_forecast_sensor_id is not None: flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] - if self._leakage_behaviour_sensor_id is not None: + if await self._record_storage_efficiency(start, fill_level_at_start): flex_model["storage-efficiency"] = { "sensor": self._leakage_behaviour_sensor_id } @@ -1041,28 +1048,53 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): ) async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): - """Record what a resource manager's storage loses on its own, as a storage efficiency.""" + """Keep a resource manager's leakage behaviour for planning, which records it when it asks for a schedule. + + Recorded on arrival instead, it covered the hours after the message rather than the window being planned, + at a fill level nobody had reported yet, and planning went on without it once those hours had passed. + """ if self._leakage_behaviour_sensor_id is None: self._logger.info( "Not recording an FRBC.LeakageBehaviour: no leakage-behaviour sensor is configured." ) - return + async def _record_storage_efficiency( + self, start: datetime, fill_level: float + ) -> bool: + """Record what the storage keeps of itself per step, over the window a schedule covers, from where it starts. + + :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. + """ + leakage_behaviour = self._latest(self._leakage_behaviour_history) + if self._leakage_behaviour_sensor_id is None or leakage_behaviour is None: + return False # What remains after one step of this handler's resolution, # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. - storage_efficiency = leakage_behaviour_to_storage_efficiency( + efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, fill_level_scale=self._fill_level_scale, - fill_level=self._latest_fill_level(), - ) - - now = self.now() - await self._fm_client.post_sensor_data( - sensor_id=self._leakage_behaviour_sensor_id, - start=self.align_to_resolution(leakage_behaviour.valid_from or now), - prior=now, - values=[storage_efficiency], - unit="%", - duration=self._schedule_duration, + fill_level=fill_level, + share_of=fill_level, ) + if self._storage_efficiency_recorded == (start, efficiency): + return True + try: + await self._fm_client.post_sensor_data( + sensor_id=self._leakage_behaviour_sensor_id, + start=start, + prior=self.now(), + values=[efficiency], + unit="%", + duration=self._schedule_duration, + ) + except (JobFailedError, ValueError, ConnectionError): + self._logger.exception( + "Not asking for a schedule against a storage efficiency: could not record %s %% on sensor %s for %s.", + efficiency, + self._leakage_behaviour_sensor_id, + start, + ) + return False + self._storage_efficiency_recorded = (start, efficiency) + return True diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index f12b2d01..e61744db 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -23,6 +23,7 @@ def leakage_behaviour_to_storage_efficiency( resolution=timedelta(minutes=15), fill_level_scale: float = 1, fill_level: float | None = None, + share_of: float | None = None, ) -> float: """ Convert a FRBC.LeakageBehaviour message into a FlexMeasures compatible storage efficiency. @@ -73,9 +74,16 @@ def leakage_behaviour_to_storage_efficiency( # This used to compound it over the hours in a step instead, # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. - # The loss is taken as a share of the top of the range, as before, which understates it at lower fill levels. + # FlexMeasures applies a storage efficiency to the stock as it stands, so the loss is a share of a fill level: + # of share_of where given, as the fill level a schedule starts from is, + # and otherwise of the top of the range, which understates it at lower fill levels. + reference = ( + share_of * fill_level_scale + if share_of is not None and share_of > 0 + else max_fill_level + ) seconds_per_step = pd.Timedelta(resolution) / pd.Timedelta(seconds=1) - storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** seconds_per_step + storage_efficiency = 100 * max(0.0, 1 - leakage / reference) ** seconds_per_step return storage_efficiency diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 3413e4b4..e17b7e37 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -29,8 +29,10 @@ ReceptionStatus, ReceptionStatusValues, ResourceManagerDetails, + RevokeObject, Role, RoleType, + Transition, ) from s2python.frbc import ( FRBCActuatorDescription, @@ -60,6 +62,8 @@ PHYSICAL_CAPACITY = 12.0 #: How often the simulated buffer reports its fill level, in seconds of wall-clock time. STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) +#: How often the forecasts are sent again, each covering the day from then on, so that planning never runs out of them. +FORECAST_INTERVAL = float(os.getenv("S2_EXAMPLE_FORECAST_INTERVAL", "3600")) ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC @@ -116,6 +120,21 @@ def operation_modes() -> list[FRBCOperationMode]: def system_description( modes: list[FRBCOperationMode], valid_from: datetime ) -> FRBCSystemDescription: + # S2 lists the transitions an actuator can make, so the heat pump says it can go from either mode to the other, + # at once and at no cost. + transitions = [ + Transition( + id=get_unique_id(), + from_=a.id, + to=b.id, + start_timers=[], + blocking_timers=[], + abnormal_condition_only=False, + ) + for a in modes + for b in modes + if a.id != b.id + ] return FRBCSystemDescription( message_id=get_unique_id(), valid_from=valid_from, @@ -125,7 +144,7 @@ def system_description( diagnostic_label="heat pump", supported_commodities=[Commodity.ELECTRICITY], operation_modes=modes, - transitions=[], + transitions=transitions, timers=[], ) ], @@ -303,13 +322,31 @@ async def apply_at_execution_time( async def answer_instructions(ws, buffer: Buffer | None = None) -> None: - """Acknowledge every instruction, and report the transition each one asks for when it is due.""" + """Acknowledge every instruction, and report the transition each one asks for when it is due. + + An instruction the CEM revokes, because a newer plan replaces the one it belonged to, is not followed: + without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. + """ active_operation_mode_id = None + pending: dict[str, asyncio.Task] = {} async for msg in ws: payload = msg.json() print("RECEIVED: ", payload) - if payload.get("message_type") != "FRBC.Instruction": + message_type = payload.get("message_type") + if message_type == "RevokeObject": + revocation = RevokeObject.model_validate(payload) + task = pending.pop(str(revocation.object_id), None) + if task is not None: + task.cancel() + await ws.send_str( + ReceptionStatus( + subject_message_id=revocation.message_id, + status=ReceptionStatusValues.OK, + ).model_dump_json() + ) + continue + if message_type != "FRBC.Instruction": continue instruction = FRBCInstruction.model_validate(payload) await ws.send_str( @@ -319,9 +356,16 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: ).model_dump_json() ) # In a task of its own, so that an instruction due later does not hold up the ones behind it. - asyncio.create_task( + task = asyncio.create_task( apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) ) + key = str(instruction.id) + pending[key] = task + + def forget(done: asyncio.Task, key: str = key) -> None: + pending.pop(key, None) + + task.add_done_callback(forget) active_operation_mode_id = instruction.operation_mode @@ -340,6 +384,27 @@ async def report_fill_level( ) +async def refresh_forecasts( + ws, leakage: FRBCLeakageBehaviour, interval: float = FORECAST_INTERVAL +) -> None: + """Send the leakage, the usage forecast and the comfort profile again every interval, each from now on. + + Sent only once, they would cover the first day, and the CEM would plan without them from then on. + """ + while True: + await asyncio.sleep(interval) + now = datetime.now(timezone.utc) + for message in ( + leakage.model_copy( + update={"message_id": get_unique_id(), "valid_from": now} + ), + usage_forecast(now), + fill_level_target_profile(now), + ): + print(f"SENDING: {message.message_type}") + await ws.send_str(message.model_dump_json()) + + async def receive_until(ws, message_type: str) -> dict: """The next message of a type, past the reception statuses that arrive in between.""" while True: @@ -349,7 +414,9 @@ async def receive_until(ws, message_type: str) -> dict: return message -async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: +async def run_resource_manager( + ws, interval: float = STATUS_INTERVAL, forecast_interval: float = FORECAST_INTERVAL +) -> Buffer: """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. In the order S2 suggests: the system description first, then the storage's status. @@ -369,13 +436,18 @@ async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: message_id=get_unique_id(), resource_id=get_unique_id(), name="heat pump with a hot-water buffer", - roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + # It draws electricity and stores heat, so it is a consumer of the commodity it is planned in. + roles=[ + Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY) + ], instruction_processing_delay=Duration(root=1000), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, ], - provides_forecast=True, + # It sends no PowerForecast. It could measure its power, which is what the next field says, + # and S2 requires at least one type there. + provides_forecast=False, provides_power_measurement_types=[ELECTRIC_POWER], ).model_dump_json() ) @@ -402,11 +474,15 @@ async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: print(f"SENDING: {message.message_type}") await ws.send_str(message.model_dump_json()) - reporting = asyncio.create_task(report_fill_level(ws, buffer, interval)) + background = [ + asyncio.create_task(report_fill_level(ws, buffer, interval)), + asyncio.create_task(refresh_forecasts(ws, leakage, forecast_interval)), + ] try: await answer_instructions(ws, buffer) finally: - reporting.cancel() + for task in background: + task.cancel() return buffer diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 175cc7d7..9b1a8b5f 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -24,9 +24,10 @@ SITE_NAME = "My CEM" SITE_TIMEZONE = "Europe/Amsterdam" -# Enough days of demo prices to cover what the CEM plans for: a schedule starts a day after the -# system description that asked for it and runs for twelve hours, so two days would just do. +# Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, +# with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 +SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. #: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. @@ -152,6 +153,8 @@ async def serve_resource_manager( # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. valid_from_shift=timedelta(0), ) + # A newer plan revokes what is left of the one before, so the resource manager does not follow both. + frbc.send_revocations = True cem.register_control_type(frbc) # create "parallel" tasks for the message producer and consumer @@ -209,11 +212,18 @@ async def configure_site( **({"attributes": attributes} if attributes else {}), ) - # Demo prices, so that a freshly configured site has something to schedule against. - # One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - # plans ahead of the moment a resource manager reports its storage: - # a fixed date here goes stale the day after it is written, and the scheduler then fails the - # job with UnknownPricesException rather than returning a schedule. + await seed_prices(fm_client, sensors["price"]["id"]) + return sensors + + +async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: + """Demo prices, so that a site has something to schedule against. + + One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM + plans ahead of the moment a resource manager reports its storage: + a fixed date here goes stale the day after it is written, and the scheduler then fails the + job with UnknownPricesException rather than returning a schedule. + """ one_day = [ 0.10, 0.11, 0.12, 0.15, 0.18, 0.17, 0.11, 0.09, 0.10, 0.09, 0.09, 0.10, 0.08, 0.05, 0.04, 0.04, @@ -223,13 +233,30 @@ async def configure_site( hour=0, minute=0, second=0, microsecond=0 ) await fm_client.post_sensor_data( - sensor_id=sensors["price"]["id"], + sensor_id=price_sensor_id, start=today, duration=timedelta(days=SEED_PRICE_DAYS), values=one_day * SEED_PRICE_DAYS, unit="EUR/kWh", ) - return sensors + + +async def keep_prices_seeded(app: web.Application): + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them.""" + + async def reseed(): + while True: + await asyncio.sleep(SEED_PRICES_EVERY.total_seconds()) + try: + await seed_prices(app[FM_CLIENT], app[SENSORS]["price"]["id"]) + except (ConnectionError, ValueError): + LOGGER.exception( + "Could not seed the demo prices again; trying at the next interval." + ) + + task = asyncio.create_task(reseed()) + yield + task.cancel() async def configure_site_once(app: web.Application) -> None: @@ -258,6 +285,7 @@ async def close_fm_client(app: web.Application) -> None: app = web.Application() app.on_startup.append(configure_site_once) +app.cleanup_ctx.append(keep_prices_seeded) app.on_cleanup.append(close_fm_client) app.add_routes([web.get("/ws", websocket_handler)]) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 9b97b64e..bc4741e6 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -121,7 +121,9 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): tasks = [ asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), - asyncio.create_task(run_resource_manager(resource_manager, interval=0.3)), + asyncio.create_task( + run_resource_manager(resource_manager, interval=0.3, forecast_interval=0.3) + ), ] try: # Two storage statuses past the first, each a new plan, so that replanning is part of what is checked. @@ -144,6 +146,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): refused = [ m for m in resource_manager.of_type("ReceptionStatus") if m["status"] != "OK" ] + assert len(cem.of_type("FRBC.UsageForecast")) >= 2 # sent again, not only once assert refused == [] start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] @@ -319,8 +322,15 @@ async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_toda def test_the_example_and_the_cem_agree_across_an_element_boundary(): - """Both follow a step a minute at a time, so from 5 kWh, through the cold element and into the warm one, they agree.""" - from flexmeasures_client.s2.control_types.FRBC.utils import follow_operation_mode + """The CEM follows a step exactly, the example's buffer a minute at a time: two discretisations of S2's definitions. + + Through the cold element and into the warm one they can differ only within the minute of the crossing, + by at most a minute of the difference between the two elements' fill rates. + """ + from flexmeasures_client.s2.control_types.FRBC.utils import ( + UsageProfile, + follow_operation_mode, + ) buffer = _heating_buffer(5.0) heating = buffer.modes[buffer.mode_id] @@ -333,10 +343,237 @@ def test_the_example_and_the_cem_agree_across_an_element_boundary(): pd.Timestamp(NOON + step * timedelta(minutes=15)), timedelta(minutes=15), leakage_behaviour=leakage_behaviour(NOON), - usage_forecast=usage_forecast(NOON), + usage=UsageProfile(usage_forecast(NOON)), ) buffer.advance(NOON + timedelta(hours=1)) assert ( cem_fill_level > 6.0 ) # the boundary between the cold and the warm element was crossed - assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) + a_minute_of_the_difference = (heat(6000) - heat(4500)) * 60 + assert buffer.fill_level == pytest.approx( + cem_fill_level, abs=a_minute_of_the_difference + ) + + +# --------------------------------------------------------------------------- +# What the review of #259 found unpinned +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + COLD_BELOW, + PHYSICAL_CAPACITY, + answer_instructions, + expected_usage, + fill_level_target_profile, + refresh_forecasts, + report_fill_level, + system_description, +) + + +class _Collect: + """A websocket that collects what is sent and hands over what it is given, then ends.""" + + def __init__(self, incoming: list[str] | None = None): + self.sent: list[dict] = [] + self._incoming = list(incoming or []) + + async def send_str(self, data: str) -> None: + self.sent.append(json.loads(data)) + + def __aiter__(self): + async def messages(): + for data in self._incoming: + yield _Text(data) + await asyncio.sleep(0) + + return messages() + + +def test_the_forecast_the_example_sends_is_the_usage_it_simulates(): + """Hour by hour, through a whole day, so that what is planned for is what is drawn.""" + for start_hour in (0, 13): + start = datetime(2024, 1, 1, start_hour, tzinfo=timezone.utc) + forecast = usage_forecast(start) + for h, element in enumerate(forecast.elements): + assert element.usage_rate_expected == pytest.approx( + per_second(expected_usage((start_hour + h) % 24)) + ) + + +def test_the_comfort_profile_preheats_before_the_morning_showers(): + profile = fill_level_target_profile(datetime(2024, 1, 1, tzinfo=timezone.utc)) + lows = [e.fill_level_range.start_of_range for e in profile.elements] + assert lows[6] == 6.0 and expected_usage(7) > expected_usage(6) + assert set(lows[:6] + lows[7:]) == {3.0} + + +def test_at_the_boundary_the_buffer_heats_at_the_warm_coefficient(): + buffer = _heating_buffer(COLD_BELOW) + fill, _, _ = buffer.rates(NOON) + assert fill == pytest.approx(heat(4500)) + + +def test_the_buffer_never_holds_more_than_it_physically_can(): + buffer = _heating_buffer(PHYSICAL_CAPACITY - 0.1) + buffer.advance(NOON + timedelta(hours=2)) + assert buffer.fill_level == PHYSICAL_CAPACITY + + +def test_the_example_declares_what_it_sends_and_how_it_can_switch(): + modes = operation_modes() + description = system_description(modes, NOON) + storage = description.storage + assert storage.provides_leakage_behaviour and storage.provides_usage_forecast + assert storage.provides_fill_level_target_profile + switches = {(str(t.from_), str(t.to)) for t in description.actuators[0].transitions} + ids = [str(m.id) for m in modes] + assert switches == {(ids[0], ids[1]), (ids[1], ids[0])} + + +@pytest.mark.asyncio +async def test_the_reported_fill_level_is_where_the_buffer_has_got_to(): + buffer = _heating_buffer(5.0) + buffer.at = datetime.now(timezone.utc) - timedelta(minutes=30) + ws = _Collect() + reporting = asyncio.create_task(report_fill_level(ws, buffer, interval=0.01)) + await _until(lambda: ws.sent) + reporting.cancel() + assert ( + ws.sent[0]["present_fill_level"] > 5.5 + ) # half an hour of heating, not where it started + + +@pytest.mark.asyncio +async def test_a_revoked_instruction_is_not_followed(): + """A newer plan replaced it; following it anyway left the heat pump in whatever the oldest plan said last.""" + from s2python.common import RevokableObjects, RevokeObject + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + due_soon = FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=get_unique_id(), + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + timedelta(milliseconds=200), + abnormal_condition=False, + ) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=due_soon.id, + ) + ws = _Collect([due_soon.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await asyncio.sleep(0.4) + assert not [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + acknowledged = { + m["subject_message_id"] + for m in ws.sent + if m["message_type"] == "ReceptionStatus" + } + assert str(revoke.message_id) in acknowledged + + +@pytest.mark.asyncio +async def test_the_forecasts_are_sent_again_from_now_on(): + ws = _Collect() + refreshing = asyncio.create_task( + refresh_forecasts(ws, leakage_behaviour(NOON), interval=0.01) + ) + await _until(lambda: len(ws.sent) >= 3) + refreshing.cancel() + assert [m["message_type"] for m in ws.sent[:3]] == [ + "FRBC.LeakageBehaviour", + "FRBC.UsageForecast", + "FRBC.FillLevelTargetProfile", + ] + assert pd.Timestamp(ws.sent[0]["valid_from"]) > NOON + + +@pytest.mark.asyncio +async def test_the_server_plans_from_now_in_the_site_time_zone_and_revokes_what_it_replaces(): + from unittest.mock import patch + + made = [] + real = server.FRBCSimple + + def capture(**kwargs): + frbc = real(**kwargs) + made.append((kwargs, frbc)) + return frbc + + resource_manager, cem = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "FRBCSimple", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem, _flexmeasures(), sensors) + ) + await _until(lambda: made) + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + kwargs, frbc = made[0] + assert kwargs["timezone"] == server.SITE_TIMEZONE + assert kwargs["valid_from_shift"] == timedelta(0) + assert frbc.send_revocations is True + + +def test_the_site_sensors_are_what_the_cem_records_in(): + assert server.SITE_SENSORS["state of charge"][:2] == ("kWh", "PT0M") + assert server.SITE_SENSORS["storage efficiency"][0] == "%" + assert server.SITE_SENSORS["usage forecast"][0] == "kWh/s" + + +@pytest.mark.asyncio +async def test_the_demo_prices_are_seeded_again_while_the_server_runs(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + assert fm.post_sensor_data.await_args.kwargs["sensor_id"] == 10 + + +def test_the_loss_is_a_share_of_the_fill_level_a_schedule_starts_from(): + """1.5 kWh a day from a 5 kWh buffer is a larger share of it than of a 12 kWh one.""" + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + leakage = leakage_behaviour(NOON) + at_five = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=7.0, share_of=5.0 + ) + rate = per_second(1.5 / 24) # the warm element, which applies at 7 + assert at_five == pytest.approx(100 * (1 - rate / 5.0) ** 900, rel=1e-12) + + +@pytest.mark.asyncio +async def test_a_failed_reseed_is_tried_again_at_the_next_interval(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index e783d876..b1af0a3b 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -9,8 +9,9 @@ from __future__ import annotations +import asyncio import logging -from datetime import timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch import pandas as pd @@ -265,6 +266,7 @@ async def test_usage_forecast_is_recorded_from_a_resolution_boundary( async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( system_description, ): + """Recorded when a schedule is asked for, over the window it covers, rather than on arrival.""" frbc = _frbc(system_description, with_sensors=True) leakage = FRBCLeakageBehaviour( message_id=get_unique_id(), @@ -279,12 +281,22 @@ async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( ], ) - await frbc.send_leakage_behaviour(leakage) + await frbc.handle_message(leakage) + await asyncio.gather(*frbc.background_tasks) + assert not [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] # nothing on arrival + + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, (RANGE_BOTTOM + RANGE_TOP) / 2) posted = _posted(frbc, LEAKAGE_SENSOR_ID) assert posted["unit"] == "%" assert len(posted["values"]) == 1 - assert posted["start"].minute % 15 == 0 and posted["start"].second == 0 + assert posted["start"] == start + assert posted["duration"] == frbc._schedule_duration # --- Without the sensors: nothing recorded, and still no exception --- @@ -601,3 +613,58 @@ async def test_an_actuator_s_fill_rate_is_posted_in_the_fill_rate_unit( ) # The power sensor really does carry a power, and is untouched by this. assert _posted(frbc, 1)["unit"] == frbc.power_unit + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_already_recorded_for_the_window_is_not_recorded_again( + system_description, +): + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) + assert await frbc._record_storage_efficiency(start, 50.0) + posts = [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_that_could_not_be_recorded_is_not_planned_against( + system_description, caplog +): + """And it is tried again next time, rather than remembered as recorded.""" + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + frbc._fm_client.post_sensor_data.side_effect = ConnectionError("unreachable") + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) is False + assert frbc._storage_efficiency_recorded is None + assert "could not record" in caplog.text diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 06ef021c..5909b4a1 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -525,8 +525,10 @@ def test_the_storage_efficiency_at_a_fill_level_is_that_elements(): async def test_the_handler_records_leakage_at_the_latest_fill_level(): leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) - _report_fill_level(frbc, 20.0) - await frbc.send_leakage_behaviour(leakage) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), 20.0 + ) posted = [ call.kwargs["values"] for call in frbc._fm_client.post_sensor_data.await_args_list @@ -535,7 +537,10 @@ async def test_the_handler_records_leakage_at_the_latest_fill_level(): assert posted == [ [ leakage_behaviour_to_storage_efficiency( - leakage, resolution=timedelta(minutes=15), fill_level=20.0 + leakage, + resolution=timedelta(minutes=15), + fill_level=20.0, + share_of=20.0, ) ] ] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index e3c0a745..f8b9f8d3 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -334,3 +334,64 @@ async def test_the_soc_is_not_reposted_when_a_description_asks_for_a_schedule(): await settle(frbc) assert frbc._fm_client.post_measurements.await_count == posts_after_the_status + + +def test_a_step_boundary_in_the_repeated_autumn_hour_is_not_an_hour_early(): + """At 02:07 the second time round, on the night the clocks go back, the step began at 02:00 of that second pass.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 7, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + aligned = frbc.align_to_resolution(second_pass) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +@pytest.mark.asyncio +async def test_a_plan_asked_for_by_a_late_description_supersedes_one_in_flight(): + """A status's plan is still out when a changed description asks for its own: only the newer batch goes out.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + assert sent[2] > 0 # the description's plan, which is the newer request + assert sent[1] == 0 # the status's plan, overtaken while it was out + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_recorded_as_a_reading_at_an_instant(): + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + assert frbc._fm_client.post_measurements.await_args.kwargs["duration"] == timedelta( + 0 + ) diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 1aa734bd..64055d78 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -156,7 +156,20 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned system_description, ): """A storage that is drawn from and leaks is not planned for by its prices alone.""" + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + frbc = _frbc(system_description) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T00:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage flex_model, _ = await _asked_for(frbc, system_description) @@ -164,6 +177,16 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned assert flex_model["storage-efficiency"] == {"sensor": LEAKAGE_SENSOR_ID} +@pytest.mark.asyncio +async def test_no_storage_efficiency_is_named_before_a_leakage_behaviour_is_known( + system_description, +): + """A sensor holding nothing for the window would be planned against as a storage that keeps everything.""" + frbc = _frbc(system_description) + flex_model, _ = await _asked_for(frbc, system_description) + assert "storage-efficiency" not in flex_model + + @pytest.mark.asyncio async def test_the_hard_bounds_are_wider_than_the_declared_range( system_description, From e3a556755bf483ce81a11e229885668ceb1ee65c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:33:43 +0100 Subject: [PATCH 24/87] fix(example, FRBC): what the second review of the example round found - The example now tells the CEM where each instruction stands: accepted on arrival, succeeded once carried out, revoked once revoked. Without that every instruction stayed NEW, so each replan revoked the whole previous batch, the instructions already carried out included. - The mode a transition reports leaving is the one the last carried-out instruction put the heat pump in. It was the mode the previous instruction to arrive asked for, which after a revocation the heat pump was never in. - A storage efficiency takes the loss as a share of the start fill level or of the middle of the storage's range, whichever is larger. A share of a near-empty start left almost nothing per step, and overstated the loss many times over once the plan filled the storage. - A state of charge is posted as an instant only on a sensor that records instants: FlexMeasures refuses a single value of no duration on a sensor with a resolution, so there it is posted over a minute, as before. - align_to_resolution floors on the moment's own clock, which is the grid FlexMeasures floors to, with the pass of an autumn DST hour taken from the moment. Flooring in UTC put hourly steps half an hour off in zones that are, and daily steps on the wrong midnight. - The example server's price reseeding survives a failed ingestion job, a wrong content type and a wrong API version, not only a lost connection. - Statuses about different steps that interleave within a second get belief times a second apart per step, where only the latest status was compared. The demo gate's first run had thirty statuses for one quarter hour arrive at once, and one was refused as a replacement. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 92 +++++++++++---- .../s2/script/websockets_client.py | 39 +++++-- .../s2/script/websockets_server.py | 13 ++- tests/s2/test_example_end_to_end.py | 108 ++++++++++++++++++ tests/s2/test_frbc_element_selection.py | 55 ++++++++- tests/s2/test_frbc_simple_planning.py | 60 +++++++++- 7 files changed, 334 insertions(+), 35 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 7aec9f68..d343b2ef 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -47,7 +47,7 @@ Removed Fixed ----- -* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels [`#259 `__] +* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] * A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index d6331642..a266a8cc 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -27,7 +27,7 @@ import pandas as pd from flexmeasures_client.client import convert_units -from flexmeasures_client.exceptions import JobFailedError +from flexmeasures_client.exceptions import ContentTypeError, JobFailedError from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( clip_fill_level_target_profile, @@ -144,7 +144,7 @@ class FRBCSimple(FRBC): _power_sensor_takes_consumption_as_positive: bool | None #: When a power value was last recorded for the actuator, and for which moment, #: so that a second status inside one event does not collide with the first. - _last_power_recorded: tuple[datetime, datetime] | None + _power_recorded_at: dict[datetime, datetime] #: The charging efficiency last recorded, and the window it was recorded for, #: so that the same value for the same window is not recorded again on every schedule. _charging_efficiency_recorded: tuple[datetime, float] | None @@ -243,7 +243,8 @@ def __init__( ) self._fill_level_scale = fill_level_scale self._power_sensor_takes_consumption_as_positive = None - self._last_power_recorded = None + self._power_recorded_at = {} + self._soc_sensor_records_instants: bool | None = None if hard_bound_factor < 1: # Below one, the hard upper bound sits inside the declared range, # and FlexMeasures clips the band into the hard bounds, @@ -290,14 +291,17 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - # Floored as an instant, in UTC, and handed back in the moment's own time zone. - # Flooring the local clock instead took the repeated hour of an autumn DST change for its first pass, - # an hour earlier than meant. For Europe/Amsterdam and a resolution dividing an hour the two agree everywhere else. - seconds = int(moment.timestamp()) - floored = datetime.fromtimestamp( - seconds - seconds % resolution_seconds, tz=ZoneInfo("UTC") + # Floored on the moment's own clock, which is the grid FlexMeasures floors a series to, + # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. + # In the repeated hour of an autumn DST change, the pass the moment is in decides which of the two boundaries it is, + # where an earlier version took the first pass every time, an hour earlier than meant. + # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. + floored = pd.Timestamp(moment).floor( + self._resolution, + ambiguous=moment.fold == 0, + nonexistent="shift_forward", ) - return floored.astimezone(moment.tzinfo) + return floored.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. @@ -311,6 +315,34 @@ def schedule_start(self, valid_from: datetime) -> datetime: max(valid_from + self._valid_from_shift, self.now()) ) + async def _soc_duration(self) -> timedelta: + """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. + + Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. + FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, + so on any other sensor, or one that cannot be read, it is posted over a minute, as before. + The sensor is asked once, since its resolution is a property of how it was set up. + """ + if self._soc_sensor_records_instants is None: + try: + sensor = await self._fm_client.get_sensor(self._soc_sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not read the resolution of state-of-charge sensor %s, so a fill level is posted over a minute: %s", + self._soc_sensor_id, + exc, + ) + return timedelta(minutes=1) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + self._soc_sensor_records_instants = resolution is not None and pd.Timedelta( + resolution + ) == pd.Timedelta(0) + return ( + timedelta(0) if self._soc_sensor_records_instants else timedelta(minutes=1) + ) + async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): @@ -321,8 +353,7 @@ async def send_storage_status( # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, - # A reading at an instant, not over the minute after it, which FlexMeasures took for a forecast. - duration=timedelta(0), + duration=await self._soc_duration(), ) # The state of charge this status reports is what a schedule starts from, so this is @@ -513,15 +544,21 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # and this handler sends several at once, so that is the ordinary case rather than an edge. # The comparison is on the event as FlexMeasures keys it, # since two statuses seconds apart still land on the same resolution step. + # Kept per event, since statuses for different steps can interleave within a second, + # and a comparison with the latest status alone then missed the one before it about the same step. event = self.align_to_resolution(start) recorded_at = self.now() - if self._last_power_recorded is not None: - last_event, last_recorded_at = self._last_power_recorded - if last_event == event and recorded_at - last_recorded_at < timedelta( - seconds=1 - ): - recorded_at = last_recorded_at + timedelta(seconds=1) - self._last_power_recorded = (event, recorded_at) + last_recorded_at = self._power_recorded_at.get(event) + if last_recorded_at is not None and recorded_at - last_recorded_at < timedelta( + seconds=1 + ): + recorded_at = last_recorded_at + timedelta(seconds=1) + self._power_recorded_at[event] = recorded_at + # Steps more than a day older than this one are forgotten, to keep this bounded: a status that late is not expected. + for old_event in [ + e for e in self._power_recorded_at if e < event - timedelta(days=1) + ]: + del self._power_recorded_at[old_event] # The value and the unit it is in travel together, # so that what is reported below is a pair that agrees even where the conversion is what failed. @@ -1066,8 +1103,21 @@ async def _record_storage_efficiency( :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. """ leakage_behaviour = self._latest(self._leakage_behaviour_history) - if self._leakage_behaviour_sensor_id is None or leakage_behaviour is None: + if ( + self._leakage_behaviour_sensor_id is None + or leakage_behaviour is None + or not self._system_description_history + ): return False + # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate + # only at one fill level, and the loss scales with the stock away from it. + # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: + # from a near-empty start, a share of the start level would let almost nothing remain per step, + # and the loss would be overstated many times over as soon as the plan filled the storage. + storage_range = list(self._system_description_history.values())[ + -1 + ].storage.fill_level_range + middle = (storage_range.start_of_range + storage_range.end_of_range) / 2 # What remains after one step of this handler's resolution, # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. efficiency = leakage_behaviour_to_storage_efficiency( @@ -1075,7 +1125,7 @@ async def _record_storage_efficiency( resolution=self._resolution, fill_level_scale=self._fill_level_scale, fill_level=fill_level, - share_of=fill_level, + share_of=max(fill_level, middle), ) if self._storage_efficiency_recorded == (start, efficiency): return True diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index e17b7e37..ad7b646e 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -24,6 +24,8 @@ Duration, EnergyManagementRole, Handshake, + InstructionStatus, + InstructionStatusUpdate, NumberRange, PowerRange, ReceptionStatus, @@ -284,10 +286,24 @@ def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: self.factor = instruction.operation_mode_factor +async def send_instruction_status( + ws, instruction_id, status: InstructionStatus +) -> None: + """Tell the CEM where an instruction stands, which is what it revokes by: only one not yet carried out.""" + await ws.send_str( + InstructionStatusUpdate( + message_id=get_unique_id(), + instruction_id=instruction_id, + status_type=status, + timestamp=datetime.now(timezone.utc), + ).model_dump_json() + ) + + async def apply_at_execution_time( ws, instruction: FRBCInstruction, - previous_operation_mode_id=None, + applied: dict | None = None, buffer: Buffer | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -297,6 +313,9 @@ async def apply_at_execution_time( A resource manager that answers on arrival reports a mode it is not in yet, and a customer energy manager recording that status has a measurement of a power the device is not drawing, timestamped now. + + The mode it reports leaving is the one the last instruction carried out put it in, kept in applied, + rather than whatever the instruction before it asked for, which may have been revoked and never carried out. """ delay = (instruction.execution_time - datetime.now(timezone.utc)).total_seconds() @@ -312,13 +331,16 @@ async def apply_at_execution_time( actuator_id=instruction.actuator_id, active_operation_mode_id=instruction.operation_mode, operation_mode_factor=instruction.operation_mode_factor, - previous_operation_mode_id=previous_operation_mode_id, + previous_operation_mode_id=(applied or {}).get("mode"), # The moment the transition happened, so that what is recorded about it is placed there, # rather than wherever the message is read. transition_timestamp=now, ) + if applied is not None: + applied["mode"] = instruction.operation_mode print("SENDING: FRBC.ActuatorStatus") await ws.send_str(status.model_dump_json()) + await send_instruction_status(ws, instruction.id, InstructionStatus.SUCCEEDED) async def answer_instructions(ws, buffer: Buffer | None = None) -> None: @@ -328,7 +350,7 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. """ - active_operation_mode_id = None + applied: dict = {"mode": None} pending: dict[str, asyncio.Task] = {} async for msg in ws: payload = msg.json() @@ -337,14 +359,17 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: if message_type == "RevokeObject": revocation = RevokeObject.model_validate(payload) task = pending.pop(str(revocation.object_id), None) - if task is not None: - task.cancel() await ws.send_str( ReceptionStatus( subject_message_id=revocation.message_id, status=ReceptionStatusValues.OK, ).model_dump_json() ) + if task is not None: + task.cancel() + await send_instruction_status( + ws, revocation.object_id, InstructionStatus.REVOKED + ) continue if message_type != "FRBC.Instruction": continue @@ -355,9 +380,10 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: status=ReceptionStatusValues.OK, ).model_dump_json() ) + await send_instruction_status(ws, instruction.id, InstructionStatus.ACCEPTED) # In a task of its own, so that an instruction due later does not hold up the ones behind it. task = asyncio.create_task( - apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) + apply_at_execution_time(ws, instruction, applied, buffer) ) key = str(instruction.id) pending[key] = task @@ -366,7 +392,6 @@ def forget(done: asyncio.Task, key: str = key) -> None: pending.pop(key, None) task.add_done_callback(forget) - active_operation_mode_id = instruction.operation_mode async def report_fill_level( diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 9b1a8b5f..49d81d82 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -12,6 +12,11 @@ from s2python.common import ControlType from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ( + ContentTypeError, + JobFailedError, + WrongAPIVersionError, +) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple @@ -249,7 +254,13 @@ async def reseed(): await asyncio.sleep(SEED_PRICES_EVERY.total_seconds()) try: await seed_prices(app[FM_CLIENT], app[SENSORS]["price"]["id"]) - except (ConnectionError, ValueError): + except ( + ConnectionError, + ValueError, + ContentTypeError, + JobFailedError, + WrongAPIVersionError, + ): LOGGER.exception( "Could not seed the demo prices again; trying at the next interval." ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index bc4741e6..3ef96f46 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -577,3 +577,111 @@ async def test_a_failed_reseed_is_tried_again_at_the_next_interval(): await _until(lambda: fm.post_sensor_data.await_count >= 2) with pytest.raises(StopAsyncIteration): await seeding.__anext__() + + +def _instruction(mode, due_in: timedelta): + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + return FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=mode, + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + due_in, + abnormal_condition=False, + ) + + +@pytest.mark.asyncio +async def test_the_mode_reported_as_left_is_the_one_last_carried_out(): + """A revoked instruction was never carried out, so the next transition does not start from its mode.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + a, b = get_unique_id(), get_unique_id() + first = _instruction(a, timedelta(0)) + revoked = _instruction(b, timedelta(seconds=30)) + later = _instruction(a, timedelta(milliseconds=200)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [ + first.model_dump_json(), + revoked.model_dump_json(), + revoke.model_dump_json(), + later.model_dump_json(), + ] + ) + await answer_instructions(ws) + await _until( + lambda: len([m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"]) + >= 2 + ) + statuses = [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + assert statuses[1]["previous_operation_mode_id"] == str(a) + + +@pytest.mark.asyncio +async def test_the_example_tells_the_cem_where_each_instruction_stands(): + """Accepted on arrival, succeeded once carried out, revoked once revoked: the CEM revokes only what is not yet carried out.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + done = _instruction(get_unique_id(), timedelta(0)) + revoked = _instruction(get_unique_id(), timedelta(seconds=30)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [done.model_dump_json(), revoked.model_dump_json(), revoke.model_dump_json()] + ) + await answer_instructions(ws) + await _until( + lambda: any( + m.get("status_type") == "SUCCEEDED" + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + ) + ) + updates = { + (m["instruction_id"], m["status_type"]) + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + } + assert updates == { + (str(done.id), "ACCEPTED"), + (str(done.id), "SUCCEEDED"), + (str(revoked.id), "ACCEPTED"), + (str(revoked.id), "REVOKED"), + } + + +@pytest.mark.asyncio +async def test_a_failed_ingestion_of_the_reseeded_prices_does_not_end_the_reseeding(): + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.exceptions import JobFailedError + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [JobFailedError("ingestion failed"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 5909b4a1..992df47e 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -540,7 +540,60 @@ async def test_the_handler_records_leakage_at_the_latest_fill_level(): leakage, resolution=timedelta(minutes=15), fill_level=20.0, - share_of=20.0, + share_of=max(20.0, _middle_of_the_range(frbc)), ) ] ] + + +def _middle_of_the_range(frbc) -> float: + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + return (storage_range.start_of_range + storage_range.end_of_range) / 2 + + +@pytest.mark.asyncio +async def test_a_near_empty_start_takes_the_loss_as_a_share_of_the_middle_of_the_range(): + """A share of a near-empty start would leave almost nothing per step, and overstate the loss once the plan fills up.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + await frbc._record_storage_efficiency(start, 0.01) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + middle = _middle_of_the_range(frbc) + assert middle > 0.01 + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, + resolution=timedelta(minutes=15), + fill_level=0.01, + share_of=middle, + ) + assert posted > 99 + + +@pytest.mark.asyncio +async def test_a_start_above_the_middle_takes_the_loss_as_a_share_of_the_start(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + high = (storage_range.end_of_range + _middle_of_the_range(frbc)) / 2 + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), high + ) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=high, share_of=high + ) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index f8b9f8d3..6c8ce9db 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -389,9 +389,61 @@ async def recording(*args, **kwargs): @pytest.mark.asyncio async def test_a_state_of_charge_is_recorded_as_a_reading_at_an_instant(): + """On a sensor that records instants, and over a minute on one with a resolution, which refuses an instant.""" + for resolution, duration in ( + ("PT0M", timedelta(0)), + ("PT15M", timedelta(minutes=1)), + ): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": resolution} + await frbc.handle_message(storage_status()) + await settle(frbc) + posted = frbc._fm_client.post_measurements.await_args.kwargs + assert posted["duration"] == duration + + +@pytest.mark.asyncio +async def test_interleaved_statuses_about_one_step_get_belief_times_a_second_apart(): + """A status about step A, one about step B, then A again, all within a second: the two about A must not collide.""" + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + frbc = frbc_simple() - await frbc.handle_message(storage_status()) - await settle(frbc) - assert frbc._fm_client.post_measurements.await_args.kwargs["duration"] == timedelta( - 0 + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + mode = actuator.operation_modes[0].id + a = datetime(2022, 12, 2, 0, 5, tzinfo=timezone.utc) + b = datetime(2022, 12, 2, 0, 20, tzinfo=timezone.utc) + for when in (a, b, a): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + priors = [ + c.kwargs["prior"] + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("start") == a + ] + assert len(priors) == 2 + assert priors[1] - priors[0] >= timedelta(seconds=1) + + +def test_a_step_boundary_is_on_the_local_clock_where_the_zone_is_half_an_hour_off_utc(): + """FlexMeasures floors a series on the sensor's clock, so an hourly step in Kolkata starts on the hour there.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=1) + moment = datetime(2026, 11, 5, 14, 40, tzinfo=ZoneInfo("Asia/Kolkata")) + assert frbc.align_to_resolution(moment) == datetime( + 2026, 11, 5, 14, 0, tzinfo=ZoneInfo("Asia/Kolkata") ) From c7fe0a7fc9fa038e25a09ca15ce615ad677f69e1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:34:15 +0100 Subject: [PATCH 25/87] docs(changelog): describe the example round as it now stands Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d343b2ef..279a3356 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,7 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] Removed ------- @@ -48,8 +48,8 @@ Fixed ----- * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] -* A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] -* A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] +* A resolution boundary is found on the site's own clock, as FlexMeasures finds it, and in the repeated hour of an autumn DST change in the pass the moment is in, where it used to be taken for the first pass, an hour earlier, so a schedule could start in the past [`#259 `__] +* On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] * Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] @@ -62,7 +62,7 @@ Fixed * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] -* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] +* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind. Statuses about different steps may interleave, so the spacing is kept per step [`#252 `__, `#259 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] * ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] * The example server creates its power sensor with ``consumption_is_positive``, as S2 states power, so that an actuator's measured power and the schedule planned for it point the same way on the sensor that holds both [`#252 `__] From 41a8f637ac5c87c073d75014f430f6a4f59e5d67 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:58:03 +0100 Subject: [PATCH 26/87] fix(example, FRBC): what the third review of the example round found - In the repeated hour of an autumn DST change, a boundary is the later of the two candidates that is not after the moment. Taking the pass from the moment put 03:10 at two-hourly steps on 02:00 of the first pass, three hours back. The candidates are compared as pandas timestamps, which compare instants. - The changelog says what the floor does: on the clock of the moment, in the offset it carries, which is how FlexMeasures floors a posted series. "The site's own clock" was not true for series a resource manager anchors in UTC. - A state-of-charge sensor stating a resolution that is not a duration is posted over a minute and logged, where it aborted the status before planning. The docstring says the sensor is asked until it answers, which it is. - An instruction that has become due is no longer pending, so a revocation arriving while it is carried out is not answered as REVOKED for an instruction the heat pump is following. - The comment on the storage efficiency's reference says that below the middle of the range the loss is understated, and by how much at most. - Tests pin the forgetting of belief times more than a day old. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 40 +++++++---- .../s2/script/websockets_client.py | 16 +++-- tests/s2/test_example_end_to_end.py | 27 +++++++ tests/s2/test_frbc_simple_planning.py | 72 +++++++++++++++++++ 5 files changed, 139 insertions(+), 18 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 279a3356..f8c2a2a8 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -48,7 +48,7 @@ Fixed ----- * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] -* A resolution boundary is found on the site's own clock, as FlexMeasures finds it, and in the repeated hour of an autumn DST change in the pass the moment is in, where it used to be taken for the first pass, an hour earlier, so a schedule could start in the past [`#259 `__] +* A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a266a8cc..1f669a29 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -291,17 +291,22 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - # Floored on the moment's own clock, which is the grid FlexMeasures floors a series to, + # Floored on the moment's own clock, in the offset it carries, which is how FlexMeasures floors a series posted with it, # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. - # In the repeated hour of an autumn DST change, the pass the moment is in decides which of the two boundaries it is, - # where an earlier version took the first pass every time, an hour earlier than meant. + # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two + # that is not after the moment: taking the first pass every time put a boundary an hour or more too early. # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. - floored = pd.Timestamp(moment).floor( - self._resolution, - ambiguous=moment.fold == 0, - nonexistent="shift_forward", - ) - return floored.to_pydatetime() + # Compared as pandas timestamps, which compare instants: two datetimes on one local clock compare by wall time, + # whichever pass each is on. + at = pd.Timestamp(moment) + candidates = [ + at.floor( + self._resolution, ambiguous=first_pass, nonexistent="shift_forward" + ) + for first_pass in (True, False) + ] + boundary = max(c for c in candidates if c <= at) + return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. @@ -321,7 +326,7 @@ async def _soc_duration(self) -> timedelta: Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, so on any other sensor, or one that cannot be read, it is posted over a minute, as before. - The sensor is asked once, since its resolution is a property of how it was set up. + The sensor is asked until it answers, and its answer kept, since its resolution is a property of how it was set up. """ if self._soc_sensor_records_instants is None: try: @@ -336,9 +341,17 @@ async def _soc_duration(self) -> timedelta: resolution = ( sensor.get("event_resolution") if isinstance(sensor, dict) else None ) - self._soc_sensor_records_instants = resolution is not None and pd.Timedelta( - resolution - ) == pd.Timedelta(0) + try: + self._soc_sensor_records_instants = resolution is not None and ( + pd.Timedelta(resolution) == pd.Timedelta(0) + ) + except ValueError: + self._logger.warning( + "State-of-charge sensor %s states a resolution of %r, which is not a duration, so a fill level is posted over a minute.", + self._soc_sensor_id, + resolution, + ) + self._soc_sensor_records_instants = False return ( timedelta(0) if self._soc_sensor_records_instants else timedelta(minutes=1) ) @@ -1114,6 +1127,7 @@ async def _record_storage_efficiency( # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: # from a near-empty start, a share of the start level would let almost nothing remain per step, # and the loss would be overstated many times over as soon as the plan filled the storage. + # Below the middle the loss is understated instead, by at most the ratio of the middle to the stock, which is the milder error. storage_range = list(self._system_description_history.values())[ -1 ].storage.fill_level_range diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index ad7b646e..b73ce4c9 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -13,6 +13,7 @@ import asyncio import os +from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone @@ -305,6 +306,7 @@ async def apply_at_execution_time( instruction: FRBCInstruction, applied: dict | None = None, buffer: Buffer | None = None, + on_due: Callable[[], None] | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -323,6 +325,9 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) + # From here on it is being carried out, so a revocation arriving now is too late to stop it. + if on_due is not None: + on_due() now = datetime.now(timezone.utc) if buffer is not None: buffer.follow(instruction, now) @@ -382,15 +387,18 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: ) await send_instruction_status(ws, instruction.id, InstructionStatus.ACCEPTED) # In a task of its own, so that an instruction due later does not hold up the ones behind it. - task = asyncio.create_task( - apply_at_execution_time(ws, instruction, applied, buffer) - ) key = str(instruction.id) - pending[key] = task + + def due(key: str = key) -> None: + pending.pop(key, None) def forget(done: asyncio.Task, key: str = key) -> None: pending.pop(key, None) + task = asyncio.create_task( + apply_at_execution_time(ws, instruction, applied, buffer, on_due=due) + ) + pending[key] = task task.add_done_callback(forget) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 3ef96f46..0e6b680c 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -685,3 +685,30 @@ async def test_a_failed_ingestion_of_the_reseeded_prices_does_not_end_the_reseed await _until(lambda: fm.post_sensor_data.await_count >= 2) with pytest.raises(StopAsyncIteration): await seeding.__anext__() + + +@pytest.mark.asyncio +async def test_a_revocation_arriving_once_an_instruction_is_due_is_too_late_to_stop_it(): + """It is carried out and reported as succeeded, not reported as revoked while the heat pump follows it.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.script.websockets_client import apply_at_execution_time + from flexmeasures_client.s2.utils import get_unique_id + + instruction = _instruction(get_unique_id(), timedelta(0)) + became_due = [] + ws = _Collect() + await apply_at_execution_time( + ws, instruction, {"mode": None}, on_due=lambda: became_due.append(True) + ) + assert became_due == [True] + + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=instruction.id, + ) + ws = _Collect([instruction.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await _until(lambda: any(m.get("status_type") == "SUCCEEDED" for m in ws.sent)) + assert not [m for m in ws.sent if m.get("status_type") == "REVOKED"] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 6c8ce9db..92ca9b5b 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -447,3 +447,75 @@ def test_a_step_boundary_is_on_the_local_clock_where_the_zone_is_half_an_hour_of assert frbc.align_to_resolution(moment) == datetime( 2026, 11, 5, 14, 0, tzinfo=ZoneInfo("Asia/Kolkata") ) + + +def test_a_boundary_in_the_repeated_hour_is_the_later_one_not_after_the_moment(): + """At two-hourly steps, 03:10 on the night the clocks go back is past 02:00 of the second pass, which is the boundary meant.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=2) + amsterdam = ZoneInfo("Europe/Amsterdam") + moment = datetime(2026, 10, 25, 3, 10, tzinfo=amsterdam) + aligned = frbc.align_to_resolution(moment) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +def test_a_naive_moment_is_floored_on_its_clock(): + frbc = frbc_simple() + assert frbc.align_to_resolution(datetime(2026, 1, 1, 12, 7)) == datetime( + 2026, 1, 1, 12, 0 + ) + + +@pytest.mark.asyncio +async def test_a_state_of_charge_sensor_stating_no_duration_is_posted_over_a_minute( + caplog, +): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": "fortnightly"} + assert await frbc._soc_duration() == timedelta(minutes=1) + assert "not a duration" in caplog.text + + +@pytest.mark.asyncio +async def test_a_sensor_that_could_not_be_read_is_asked_again(): + frbc = frbc_simple() + frbc._fm_client.get_sensor.side_effect = [ + ConnectionError("down"), + {"event_resolution": "PT0M"}, + ] + assert await frbc._soc_duration() == timedelta(minutes=1) + assert await frbc._soc_duration() == timedelta(0) + + +@pytest.mark.asyncio +async def test_belief_times_are_forgotten_for_steps_more_than_a_day_old(): + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + + frbc = frbc_simple() + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + for when in ( + datetime(2022, 12, 1, 0, 0, tzinfo=timezone.utc), + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc), + ): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + assert list(frbc._power_recorded_at) == [ + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc) + ] From bc90b2bd1bcb1faa10ccd32aacf1abd84bbf7808 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 17:19:31 +0100 Subject: [PATCH 27/87] fix(FRBC): plan from the next boundary, not the start of the step under way The demo gate's second run, against a sandboxed FlexMeasures, heated the buffer in the quarter hour the plan chose, but every replan inside that quarter replanned the same quarter, at 0.72 kW, then 0.67, 0.63 and on down to 0.51 kW. A schedule asked for mid-step started at the boundary that step began at, while it was planned from the fill level reported now, which FlexMeasures takes to be the level at the schedule's start. So each replan counted on the whole step where only part of it was left, asked for less power over what remained, and the step delivered less than any plan meant it to. A schedule now starts at the next boundary, or at now where now is one. The step under way keeps the instruction it is already following, which a newer plan does not revoke, since it has been carried out. The end-to-end test stands the CEM's clock on a boundary, so that its schedules start there and the first instruction is due within the test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 19 +++++++++----- tests/s2/test_example_end_to_end.py | 14 ++++++++++- tests/s2/test_frbc_simple_planning.py | 25 +++++++++++++++++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index f8c2a2a8..243ee8d5 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -79,7 +79,7 @@ Fixed * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] * The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] * An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from a step before the current one, which a description days old made it slide into [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and from the next resolution boundary rather than the start of the step under way, which a description days old made it slide into the past, and which, planned from the fill level reported mid-step, counted on the whole step where only part was left, so each replan within it asked for less power than the step needed [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 1f669a29..8bce09af 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -309,16 +309,23 @@ def align_to_resolution(self, moment: datetime) -> datetime: return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. + """Where a schedule asked for at valid_from starts: shifted and floored to a boundary, but never before now. - Not into an earlier step, since planning happens on every storage status while a description may be days old: + Never before now, since planning happens on every storage status while a description may be days old: anchored to that description alone, every status in a long session was answered with the same window, which slid into the past, so its instructions were all due already and arrived as one burst. - The step now falls in has begun already, so its first instruction is due at once. + Where now is past that boundary, the schedule starts at the next one, not at the start of the step now is in. + A schedule is planned from the fill level reported now, and FlexMeasures takes that to be the level at its start, + so a schedule starting at a step already under way was planned as if the whole step were left, + and every replan within it asked for less power over what remained of it, delivering less than planned. + The step under way keeps the instruction it is already following. """ - return self.align_to_resolution( - max(valid_from + self._valid_from_shift, self.now()) - ) + now = self.now() + start = self.align_to_resolution(valid_from + self._valid_from_shift) + if start >= now: + return start + current = self.align_to_resolution(now) + return current if current == now else current + self._resolution async def _soc_duration(self) -> timedelta: """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 0e6b680c..c01016cf 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -118,7 +118,18 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): loop = asyncio.get_running_loop() loop.set_exception_handler(lambda _, context: unhandled.append(context)) began = datetime.now(timezone.utc) + # The CEM's clock stands on the boundary the step under way began at, so that is where its schedules start, + # and the first instruction is due at once rather than at the next boundary, up to a quarter of an hour away. + on_a_boundary = began.replace( + minute=began.minute - began.minute % 15, second=0, microsecond=0 + ) + + from unittest.mock import patch + clock = patch.object( + server.FRBCSimple, "now", lambda self: on_a_boundary.astimezone(self._timezone) + ) + clock.start() tasks = [ asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), asyncio.create_task( @@ -133,6 +144,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) + clock.stop() gc.collect() loop.set_exception_handler(None) @@ -150,7 +162,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): assert refused == [] start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] - assert began - timedelta(minutes=15) < start <= began + timedelta(seconds=5) + assert start == on_a_boundary assert unhandled == [] # Every declaration reached the sensor the server made for it, to be seen next to the schedule planned against it. diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 92ca9b5b..8b314168 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -284,12 +284,33 @@ def test_a_handler_turns_away_a_model_it_does_not_handle(): def test_a_schedule_never_starts_before_now(): - """A description days old still plans from now, on a boundary, rather than a window that has already passed.""" + """A description days old still plans from the next boundary, rather than a window that has already passed. + + Nor from the start of the step under way: the fill level reported now is taken as the level at the start, + so the plan would count on the whole step where only part of it is left. + """ frbc = frbc_simple() frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) assert frbc.schedule_start( datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) - ) == datetime(2022, 12, 10, 14, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_asked_for_on_a_boundary_starts_there(): + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_whose_shifted_start_is_in_the_step_under_way_starts_at_the_next_boundary(): + frbc = frbc_simple() + frbc._valid_from_shift = timedelta(0) + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 10, 14, 6, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) @pytest.mark.asyncio From 9824ea5dbb5e5ffc70f207c66c39dbbeb28ecdd8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 17:23:06 +0100 Subject: [PATCH 28/87] style(FRBC): keep the soc-at-start TODO off the line it annotates Black had split the expression around the trailing comment after the merge. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 6f21e10f..9805fc87 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -912,8 +912,8 @@ async def trigger_schedule( soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { "soc-unit": soc_unit, - "soc-at-start": soc_at_start - * to_soc_unit, # TODO: use forecast of the SOC instead + # TODO: use forecast of the SOC instead + "soc-at-start": soc_at_start * to_soc_unit, "soc-min": soc_min * to_soc_unit, "soc-max": soc_max * to_soc_unit, } From 93e3ae3ea77d4b8fbd2d280799c2f2c7ad01a5ad Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:06:28 +0100 Subject: [PATCH 29/87] feat(FRBC): a production price of its own, an opt-in relaxed site capacity, and assets without a location Carved from #188, which needed these for a community of homes steered by one CEM. - FRBCSimple takes production_price_sensor_id, for a site where feeding power back is priced differently from drawing it, as under a feed-in tariff. Without one, both directions are priced by price_sensor_id, as before. - FRBCSimple takes relax_site_capacity, off by default. For a home the connection is a physical limit; for a member of a community whose capacity is a target set by others, a breach at a price is what is meant. - A schedule overtaken by a newer storage status during its round trip is no longer translated into instructions: send_instruction_batch would drop them anyway, and translating follows every step of the schedule. - The instructions of a batch are logged once, at debug level, by mode name, rather than as a list growing on every step. - add_asset no longer requires a location, which FlexMeasures does not require either and an asset standing for a community has no use for. Arguments keep their positions, so generic_asset_type_id is optional in the signature only, and its absence is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 19 ++-- .../s2/control_types/FRBC/frbc_simple.py | 30 ++++++- .../s2/control_types/FRBC/utils.py | 14 +++ tests/client/test_asset.py | 47 ++++++++++ tests/s2/test_frbc_charging_efficiency.py | 88 +++++++++++++++++++ 5 files changed, 191 insertions(+), 7 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 698f5bc6..5aa6cb1f 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1511,9 +1511,9 @@ async def add_asset( self, name: str, account_id: int, - latitude: float, - longitude: float, - generic_asset_type_id: int, + latitude: float | None = None, + longitude: float | None = None, + generic_asset_type_id: int | None = None, parent_asset_id: int | None = None, sensors_to_show: list | None = None, flex_context: dict | None = None, @@ -1539,13 +1539,20 @@ async def add_asset( This function raises a ValueError when an unhandled status code is returned. """ - asset = dict( + if generic_asset_type_id is None: + # Optional in the signature only so that latitude and longitude, before it, can be left out, + # while every argument keeps its position. + raise ValueError("add_asset needs a generic_asset_type_id.") + asset: dict = dict( name=name, account_id=account_id, - latitude=latitude, - longitude=longitude, generic_asset_type_id=generic_asset_type_id, ) + # A location is optional to FlexMeasures, which many assets have no use for, such as one standing for a community. + if latitude is not None: + asset["latitude"] = latitude + if longitude is not None: + asset["longitude"] = longitude if parent_asset_id: asset["parent_asset_id"] = parent_asset_id if sensors_to_show: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 110b3f26..8e88ce64 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -122,6 +122,8 @@ def _operation_mode_power_bands( class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int + _production_price_sensor_id: int + _relax_site_capacity: bool _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None @@ -159,6 +161,13 @@ def __init__( soc_sensor_id: int, rm_discharge_sensor_id: int, price_sensor_id: int, + # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. + # Without one, both directions are priced by price_sensor_id. + production_price_sensor_id: int | None = None, + # Whether the site's connection capacity may be breached at a price rather than never. + # Off by default: for a home, the connection is a physical limit. + # A site whose capacity is a target set by others, as in a community steering its members, can turn it on. + relax_site_capacity: bool = False, # Optional so existing callers keep working; when absent the actuator's fill rate is # simply not recorded, rather than being posted onto the power sensor. fill_rate_sensor_id: int | None = None, @@ -204,6 +213,12 @@ def __init__( super().__init__(max_size) self._power_sensor_id = power_sensor_id self._price_sensor_id = price_sensor_id + self._production_price_sensor_id = ( + production_price_sensor_id + if production_price_sensor_id is not None + else price_sensor_id + ) + self._relax_site_capacity = relax_site_capacity self._schedule_duration = schedule_duration self._resolution = resolution self._soc_sensor_id = soc_sensor_id @@ -922,7 +937,7 @@ async def trigger_schedule( self._logger.debug("flex model: %s", flex_model) flex_context: dict = { - "production-price": {"sensor": self._price_sensor_id}, + "production-price": {"sensor": self._production_price_sensor_id}, "consumption-price": {"sensor": self._price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", } @@ -932,10 +947,23 @@ async def trigger_schedule( # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. flex_context["relax-soc-constraints"] = True + if self._relax_site_capacity: + flex_context["relax-site-capacity-constraints"] = True schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) self._warn_where_a_fallback_scheduler_answered(schedule, start) + # send_instruction_batch checks this once more, but translating a schedule follows every step of it, + # which is wasted on a request that is already out of date. + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not turning the schedule for planning request %s into instructions: a newer storage status arrived while it was being planned.", + request_number, + ) + return + # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor instructions = fm_schedule_to_instructions( schedule, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index a513d9eb..d30cfd3a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -189,6 +189,7 @@ def fm_schedule_to_instructions( ) actuator = system_description.actuators[0] operation_modes: list[FRBCOperationMode] = actuator.operation_modes + modes_by_id = {str(mode.id): mode for mode in operation_modes} # Held between the hard bounds the schedule was planned against, # so that each step is answered with the operation mode for the fill level the scheduler planned. @@ -295,6 +296,19 @@ def fm_schedule_to_instructions( bounds=(fill_level_min, fill_level_max), ) + if logger.isEnabledFor(logging.DEBUG): + # The whole batch once, by name, rather than as a growing list on every step. + logger.debug( + "Instructions: %s", + [ + ( + instruction.execution_time.isoformat(), + om_label(modes_by_id[str(instruction.operation_mode)]), + round(instruction.operation_mode_factor, 6), + ) + for instruction in instructions + ], + ) return instructions diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index 25101873..78f0f723 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -560,3 +560,50 @@ async def test_delete_asset_confirm_no(): (k[0], str(k[1])) for k in m.requests.keys() ] await client.close() + + +def _posted_asset(m) -> dict: + """The body of the one asset post aioresponses saw.""" + ((_, calls),) = [ + (key, calls) for key, calls in m.requests.items() if key[0] == "POST" + ] + return calls[0].kwargs["json"] + + +@pytest.mark.asyncio +async def test_an_asset_can_be_added_without_a_location(): + """FlexMeasures takes no location for an asset that has none, such as one standing for a community.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 27}) + await client.add_asset(name="community", account_id=1, generic_asset_type_id=5) + posted = _posted_asset(m) + assert "latitude" not in posted and "longitude" not in posted + await client.close() + + +@pytest.mark.asyncio +async def test_a_location_is_posted_as_numbers_where_given(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 28}) + await client.add_asset( + name="home", + account_id=1, + latitude=52.0, + longitude=4.0, + generic_asset_type_id=5, + ) + posted = _posted_asset(m) + assert (posted["latitude"], posted["longitude"]) == (52.0, 4.0) + await client.close() + + +@pytest.mark.asyncio +async def test_an_asset_needs_a_type(): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="generic_asset_type_id"): + await client.add_asset(name="untyped", account_id=1) + await client.close() diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 6baea1dd..dc4a3460 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -448,3 +448,91 @@ async def test_the_measured_state_of_charge_is_not_named_in_the_flex_model(): flex_model = await _asked_for(frbc, system_description) assert "state-of-charge" not in flex_model assert isinstance(flex_model["soc-at-start"], float) + + +@pytest.mark.asyncio +async def test_feeding_back_is_priced_by_its_own_sensor_where_there_is_one(): + system_description = _heater() + frbc = _frbc(system_description, production_price_sensor_id=42) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 42} + assert flex_context["consumption-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +async def test_without_its_own_sensor_feeding_back_is_priced_as_drawing(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("relax", [False, True]) +async def test_the_site_capacity_is_relaxed_only_where_asked(relax): + """A home's connection is a physical limit; a community member's may be a target it can breach at a price.""" + system_description = _heater() + frbc = _frbc(system_description, relax_site_capacity=relax) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context.get("relax-site-capacity-constraints", False) is relax + + +@pytest.mark.asyncio +async def test_a_schedule_overtaken_while_it_was_planned_is_not_translated(): + """A newer storage status arrived during the round trip, so turning this schedule into instructions is wasted.""" + from unittest.mock import patch + + from flexmeasures_client.s2.control_types.FRBC import STORAGE_STATUS_REQUESTS + + system_description = _heater() + frbc = _frbc(system_description) + request_number = frbc._count_request(STORAGE_STATUS_REQUESTS) + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def overtaken(**kwargs): + frbc._count_request(STORAGE_STATUS_REQUESTS) # a newer status, mid round trip + return planned + + frbc._fm_client.trigger_and_get_schedule = AsyncMock(side_effect=overtaken) + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions" + ) as translate: + await frbc.trigger_schedule( + system_description.message_id, request_number=request_number + ) + translate.assert_not_called() + frbc.send_instruction_batch.assert_not_awaited() + + +def test_a_batch_of_instructions_is_logged_once_and_by_name(caplog): + from flexmeasures_client.s2.control_types.FRBC.utils import ( + fm_schedule_to_instructions, + ) + + system_description = _heater() + with caplog.at_level(logging.DEBUG): + fm_schedule_to_instructions( + { + "start": "2024-01-01T00:00:00+00:00", + "duration": "PT1H", + "values": [0.0, HEATER_POWER, HEATER_POWER, 0.0], + }, + system_description, + initial_fill_level=FILL_LEVEL_AT_START, + ) + batches = [ + r.getMessage() + for r in caplog.records + if r.getMessage().startswith("Instructions: ") + ] + assert len(batches) == 1 + assert "'on'" in batches[0] and "'off'" in batches[0] From 6408b6fc8356b1cd9c94e71a9ebb740448128ede Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:13:07 +0100 Subject: [PATCH 30/87] fix(FRBC): what the review of the energy-unit change found - soc-min is converted like soc-at-start and soc-max, but nothing checked it: it is zero unless a storage is reported below its range, and zero converts to zero. A test now reports one below it, in joules. - A test docstring and a changelog line from #244 said that a state-of-charge sensor in another unit would hold readings a thousand times off. FlexMeasures converts what is posted into the sensor's own unit, so it does not. - A comment broke between its subject and its verb. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 5 +++-- tests/s2/test_frbc_charging_efficiency.py | 13 +++++++++++++ tests/s2/test_frbc_declared_profiles.py | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 61c953e5..9dcbd5a7 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -34,7 +34,7 @@ Changed * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] * ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, rather than as the hard ``soc-min`` and ``soc-max``, which become wide bounds at zero and half again above the range, each widened to take in a state of charge beyond them. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band [`#245 `__] -* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] +* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] * The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 9805fc87..fee4e462 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -906,8 +906,9 @@ async def trigger_schedule( # and the window a charging efficiency has to be recorded over to be planned against. start = self.schedule_start(system_description.valid_from) - # FlexMeasures takes a state of charge in kWh or MWh only, so a fill level recorded in another energy unit - # is converted here, at the edge, rather than refused, and the bounds read back below stay in the recorded unit. + # FlexMeasures takes a state of charge in kWh or MWh only. + # So a fill level recorded in another energy unit is converted here, at the edge, rather than refused, + # and the bounds read back below stay in the recorded unit. # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 6baea1dd..37c14be8 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -418,6 +418,19 @@ async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( assert flex_model["soc-max"] == pytest.approx(1.5 * RANGE_TOP * per_unit) +@pytest.mark.asyncio +async def test_a_lower_bound_below_zero_goes_out_in_the_same_unit(): + """Reported below its range, a storage widens the lower bound to take it in, which then needs the conversion too.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit="J") + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=-3.6e6 + ) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-at-start"] == pytest.approx(-1.0) + assert flex_model["soc-min"] == pytest.approx(-1.0) + + @pytest.mark.asyncio async def test_the_bounds_read_back_stay_in_the_recorded_unit(): """Only what goes to FlexMeasures is converted: the schedule is matched against the operation modes in their own units.""" diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index b1af0a3b..201ed323 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -409,7 +409,7 @@ async def test_the_energy_unit_is_the_deployment_s_to_choose(system_description) async def test_a_schedule_is_asked_for_in_the_unit_the_fill_level_is_recorded_in( system_description, ): - """soc-unit has to follow energy_unit, or the scheduler plans against bounds a thousand times its measurements.""" + """soc-unit follows energy_unit where FlexMeasures takes it, so the values stated inline are in the unit they are recorded in.""" frbc = _frbc(system_description, with_sensors=True) # A schedule is only asked for once a storage status has said where to start from. frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( From 11e9a397891bcb46c4838fc5b1509efa35217d51 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:20:33 +0100 Subject: [PATCH 31/87] test(FRBC): an overtaken plan sends nothing, whether dropped before or after translation Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_frbc_simple_planning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 8b314168..c35ec8b1 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -405,7 +405,8 @@ async def recording(*args, **kwargs): assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 sent = dict(batches) assert sent[2] > 0 # the description's plan, which is the newer request - assert sent[1] == 0 # the status's plan, overtaken while it was out + # The status's plan, overtaken while it was out, sends nothing, whether it is dropped before or after translation. + assert sent.get(1, 0) == 0 @pytest.mark.asyncio From 7c8ae45ff7bc497fdb14e5127288ea6b7dda263e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:21:24 +0100 Subject: [PATCH 32/87] docs(changelog): the handler options, the dropped translation, and assets without a location Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 ++++ CHANGELOG.rst | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 9dcbd5a7..ecd80cc9 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -22,10 +22,14 @@ Added * ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] +* ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``, off by default, which lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others rather than a physical limit [`#261 `__] +* An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] Changed ------- +* An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] * ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes, which a schedule's start is lined up with [`#240 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 05d31188..4a939f62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,7 @@ Added Changed ------- +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] Removed From 47d295dc3bd716be40240bf950e0ffdd62a9a4ed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:34:55 +0100 Subject: [PATCH 33/87] fix(FRBC): send relax_site_capacity either way, or leave it to FlexMeasures Review found that relax_site_capacity=False, the default, kept nothing hard. Leaving the key out lets FlexMeasures, since v1.0, read its relax-constraints umbrella, which relaxes by default at a 10000 EUR/kW breach price, so a home's connection was soft whichever way the option was set. relax_site_capacity is now True, False or unset. True and False are sent as they are; unset leaves the server's own default in place, which is what the handler has always done, so nothing changes for a caller that does not pass it. The test asserts what is sent, where it asserted a missing key, which passed on the bug. Also from the review: - add_asset refuses half a location, which FlexMeasures would store, and its docstring says when it raises before any request is made. - The comment on the batch log line said it replaced a growing per-step list that this branch never had. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- CHANGELOG.rst | 2 +- src/flexmeasures_client/client.py | 7 ++++++- .../s2/control_types/FRBC/frbc_simple.py | 16 ++++++++------- .../s2/control_types/FRBC/utils.py | 2 +- tests/client/test_asset.py | 11 ++++++++++ tests/s2/test_frbc_charging_efficiency.py | 20 ++++++++++++++++--- 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index ecd80cc9..b3f40ebe 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -23,7 +23,7 @@ Added * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] -* ``FRBCSimple`` takes ``relax_site_capacity``, off by default, which lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others rather than a physical limit [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] Changed diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a939f62..4d0d1f34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,7 +27,7 @@ Added Changed ------- -* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required [`#261 `__] +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required, and half a location is refused [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] Removed diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 5aa6cb1f..9252c477 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1537,12 +1537,17 @@ async def add_asset( 'status': 200 } - This function raises a ValueError when an unhandled status code is returned. + This function raises a ValueError when an unhandled status code is returned, + when no generic_asset_type_id is given, and when only one of latitude and longitude is. """ if generic_asset_type_id is None: # Optional in the signature only so that latitude and longitude, before it, can be left out, # while every argument keeps its position. raise ValueError("add_asset needs a generic_asset_type_id.") + if (latitude is None) != (longitude is None): + raise ValueError( + "add_asset takes a location as both a latitude and a longitude, or neither." + ) asset: dict = dict( name=name, account_id=account_id, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 735b29a7..694541e2 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -123,7 +123,7 @@ class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int _production_price_sensor_id: int - _relax_site_capacity: bool + _relax_site_capacity: bool | None _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None @@ -164,10 +164,12 @@ def __init__( # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. # Without one, both directions are priced by price_sensor_id. production_price_sensor_id: int | None = None, - # Whether the site's connection capacity may be breached at a price rather than never. - # Off by default: for a home, the connection is a physical limit. - # A site whose capacity is a target set by others, as in a community steering its members, can turn it on. - relax_site_capacity: bool = False, + # Whether the site's connection capacity may be breached at a price (True) or never (False). + # Unset, it is left to FlexMeasures, which since v1.0 relaxes it at a high default breach price, + # under its relax-constraints umbrella, and before that did not. + # A home whose connection is a physical limit can pass False; + # a site whose capacity is a target set by others, as in a community steering its members, can pass True. + relax_site_capacity: bool | None = None, # Optional so existing callers keep working; when absent the actuator's fill rate is # simply not recorded, rather than being posted onto the power sensor. fill_rate_sensor_id: int | None = None, @@ -1016,8 +1018,8 @@ async def trigger_schedule( # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. flex_context["relax-soc-constraints"] = True - if self._relax_site_capacity: - flex_context["relax-site-capacity-constraints"] = True + if self._relax_site_capacity is not None: + flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) self._warn_where_a_fallback_scheduler_answered(schedule, start) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 2dbfd0b7..cec7fa08 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -383,7 +383,7 @@ def fm_schedule_to_instructions( ) if logger.isEnabledFor(logging.DEBUG): - # The whole batch once, by name, rather than as a growing list on every step. + # The whole batch in one line, by mode name, which the per-step lines above do not give. logger.debug( "Instructions: %s", [ diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index 78f0f723..85c4b238 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -607,3 +607,14 @@ async def test_an_asset_needs_a_type(): with pytest.raises(ValueError, match="generic_asset_type_id"): await client.add_asset(name="untyped", account_id=1) await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("location", [dict(latitude=52.0), dict(longitude=4.0)]) +async def test_half_a_location_is_refused(location): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="both a latitude and a longitude"): + await client.add_asset( + name="half", account_id=1, generic_asset_type_id=5, **location + ) + await client.close() diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index bc12cbab..2a899d82 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -488,15 +488,29 @@ async def test_without_its_own_sensor_feeding_back_is_priced_as_drawing(): @pytest.mark.asyncio @pytest.mark.parametrize("relax", [False, True]) -async def test_the_site_capacity_is_relaxed_only_where_asked(relax): - """A home's connection is a physical limit; a community member's may be a target it can breach at a price.""" +async def test_the_site_capacity_is_relaxed_or_kept_hard_as_asked(relax): + """Sent either way: FlexMeasures reads a missing flag as its relax-constraints umbrella, which relaxes by default. + + So a home whose connection is a physical limit is only kept to it by an explicit False. + """ system_description = _heater() frbc = _frbc(system_description, relax_site_capacity=relax) await frbc.trigger_schedule(system_description.message_id) flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ "flex_context" ] - assert flex_context.get("relax-site-capacity-constraints", False) is relax + assert flex_context["relax-site-capacity-constraints"] is relax + + +@pytest.mark.asyncio +async def test_unset_the_site_capacity_is_left_to_flexmeasures(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert "relax-site-capacity-constraints" not in flex_context @pytest.mark.asyncio From bcdbd79c53d031f21cc4f0c27e65f7f9586642df Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:41:16 +0100 Subject: [PATCH 34/87] feat(example): the site's graphs show the buffer, its comfort band, and what fills and draws it The demo site's graphs page showed only the price and the power, which says nothing about whether the plan kept the buffer warm. The example server now sets the site's sensors_to_show: the price; the heat pump's planned and measured power; the buffer's fill level between the comfort band's minima and maxima; and the heat pump's fill rate next to the hot-water usage forecast, both in kWh/s. Checked against a sandboxed FlexMeasures v1.1.0rc3: all four graphs render, with the pre-heat before the morning showers visible as a raised minimum. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/script/websockets_server.py | 31 +++++++++++++++++++ tests/s2/test_example_end_to_end.py | 29 +++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 49d81d82..820808a3 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -217,10 +217,41 @@ async def configure_site( **({"attributes": attributes} if attributes else {}), ) + # The site's graphs page shows what the plan is about, not only its price and power: + # the buffer with the comfort band around it, what is drawn from it, and what the heat pump puts in. + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": site_graphs(sensors)}, + parse_json_fields=True, + ) + await seed_prices(fm_client, sensors["price"]["id"]) return sensors +def site_graphs(sensors: dict[str, dict]) -> list[dict]: + """The site's graphs, top to bottom, in the format FlexMeasures takes for an asset's sensors_to_show.""" + + def ids(*names: str) -> list[int]: + return [sensors[name]["id"] for name in names] + + return [ + {"title": "Price", "plots": [{"sensors": ids("price")}]}, + { + "title": "Heat pump: planned and measured power", + "plots": [{"sensors": ids("power")}], + }, + { + "title": "Buffer: fill level and comfort band", + "plots": [{"sensors": ids("state of charge", "soc minima", "soc maxima")}], + }, + { + "title": "Buffer: filled by the heat pump, drawn by hot-water use", + "plots": [{"sensors": ids("fill rate", "usage forecast")}], + }, + ] + + async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: """Demo prices, so that a site has something to schedule against. diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index c01016cf..1406388b 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -724,3 +724,32 @@ async def test_a_revocation_arriving_once_an_instruction_is_due_is_too_late_to_s await answer_instructions(ws) await _until(lambda: any(m.get("status_type") == "SUCCEEDED" for m in ws.sent)) assert not [m for m in ws.sent if m.get("status_type") == "REVOKED"] + + +@pytest.mark.asyncio +async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_draws_it(): + """The graphs page showed only price and power, which says nothing about whether the plan kept the buffer warm.""" + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[{"id": 5, "name": server.SITE_NAME, "sensors": []}] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + sensors = await server.configure_site(server.SITE_NAME, fm) + shown = [ + call.args[1]["sensors_to_show"] + for call in fm.update_asset.await_args_list + if "sensors_to_show" in call.args[1] + ] + assert len(shown) == 1 + plotted = {graph["title"]: set(graph["plots"][0]["sensors"]) for graph in shown[0]} + by_id = {sensor["id"]: name for name, sensor in sensors.items()} + named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} + assert {"state of charge", "soc minima", "soc maxima"} in named.values() + assert {"fill rate", "usage forecast"} in named.values() + assert {"power"} in named.values() and {"price"} in named.values() From cc54bb683f5c7db7bcc450c1205a1a31e289f52b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:41:47 +0100 Subject: [PATCH 35/87] docs(changelog): the example site's graphs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b3f40ebe..101f6b47 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,7 +41,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] Removed ------- From c46675257df5d1c25cdf4a613917586d09d0e110 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 19:16:38 +0100 Subject: [PATCH 36/87] fix(example): set the site graphs in the form every FlexMeasures reads Review found that the form with plots is refused before FlexMeasures v0.32, so the example server would not start against an older server. A title and a list of sensors per graph is read by every version and draws the same graphs (checked against v1.1.0rc3). The docstring now also says the server sets them on every start, so changes made in the UI do not outlive a restart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/script/websockets_server.py | 18 ++++++++++-------- tests/s2/test_example_end_to_end.py | 12 +++++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 820808a3..ee7d2152 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -230,24 +230,26 @@ async def configure_site( def site_graphs(sensors: dict[str, dict]) -> list[dict]: - """The site's graphs, top to bottom, in the format FlexMeasures takes for an asset's sensors_to_show.""" + """The site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + + In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, + rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. + The server sets them on every start, so changes made to them in the UI do not outlive a restart. + """ def ids(*names: str) -> list[int]: return [sensors[name]["id"] for name in names] return [ - {"title": "Price", "plots": [{"sensors": ids("price")}]}, - { - "title": "Heat pump: planned and measured power", - "plots": [{"sensors": ids("power")}], - }, + {"title": "Price", "sensors": ids("price")}, + {"title": "Heat pump: planned and measured power", "sensors": ids("power")}, { "title": "Buffer: fill level and comfort band", - "plots": [{"sensors": ids("state of charge", "soc minima", "soc maxima")}], + "sensors": ids("state of charge", "soc minima", "soc maxima"), }, { "title": "Buffer: filled by the heat pump, drawn by hot-water use", - "plots": [{"sensors": ids("fill rate", "usage forecast")}], + "sensors": ids("fill rate", "usage forecast"), }, ] diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 1406388b..952c2867 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -747,9 +747,19 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ if "sensors_to_show" in call.args[1] ] assert len(shown) == 1 - plotted = {graph["title"]: set(graph["plots"][0]["sensors"]) for graph in shown[0]} + plotted = {graph["title"]: set(graph["sensors"]) for graph in shown[0]} by_id = {sensor["id"]: name for name, sensor in sensors.items()} named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} assert {"state of charge", "soc minima", "soc maxima"} in named.values() assert {"fill rate", "usage forecast"} in named.values() assert {"power"} in named.values() and {"price"} in named.values() + + +def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): + """A title and a list of sensors per graph; the form with plots is refused before FlexMeasures v0.32.""" + graphs = server.site_graphs( + {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + ) + for graph in graphs: + assert set(graph) == {"title", "sensors"} + assert all(isinstance(i, int) for i in graph["sensors"]) From 00284949114efb2999fa83444b2f1c6a7f8e7865 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:46:20 +0100 Subject: [PATCH 37/87] fix(S2): a lost confirmation does not stall a session, and a closed session stops Carved from #188, rewritten for the library rather than transcribed. - A control type's message arriving while no control type is selected activates that control type, if the CEM has a handler for it, with a warning. A resource manager sends a control type's messages only after accepting the SelectControlType for it, so such a message is that acceptance; without this, a lost ReceptionStatus for the selection left every FRBC message answered with TEMPORARY_ERROR, which a resource manager retries forever, and a whole co-simulation hung at its first step. #188 did this for FRBC alone and also before a resource manager's details had arrived, when nothing can have been selected; here it covers each control type's message prefix, and only after the details. - Closing a control-type handler cancels the work it still has in flight, such as a schedule being planned, which otherwise went on posting and planning for a resource manager no longer connected. TUNES no longer overrides close() with a copy of the default. - The example server's session ends with its connection: the CEM is closed and the producer and the watchdog are stopped. Left running, they outlived the connection, and a resource manager that reconnected was served by two sessions planning for it at once. - A test now pins the case of a system description replaced while the plan it asked for is out: only the newer plan is sent. The behaviour came with the request counting in #259; #188 had no equivalent. Not carried from #188: the futures paired with each outgoing message, which nothing awaited and which changed get_message's public return type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 41 +++++++++++ .../s2/control_types/FRBC/frbc_tunes.py | 6 -- .../s2/control_types/__init__.py | 5 ++ .../s2/script/websockets_server.py | 22 ++++-- tests/s2/test_cem.py | 70 +++++++++++++++++++ tests/s2/test_example_end_to_end.py | 33 +++++++++ tests/s2/test_frbc_simple_planning.py | 37 ++++++++++ 7 files changed, 202 insertions(+), 12 deletions(-) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 72ef9b3d..ceb27934 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -42,6 +42,15 @@ _LOGGER = logging.getLogger(__name__) +#: The control type each family of control-type messages belongs to, by the prefix of its message type. +CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { + "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, + "PEBC.": ControlType.POWER_ENVELOPE_BASED_CONTROL, + "PPBC.": ControlType.POWER_PROFILE_BASED_CONTROL, + "OMBC.": ControlType.OPERATION_MODE_BASED_CONTROL, + "DDBC.": ControlType.DEMAND_DRIVEN_BASED_CONTROL, +} + class CEM(Handler): __version__ = "0.0.2-beta" @@ -188,6 +197,7 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): ) # what a message is from here on, whatever form it came in self._logger.debug(f"Received: {message}") + self._activate_implicitly(message) # try to handle the message with the control_type handle if ( @@ -220,6 +230,37 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if response is not None: await self._sending_queue.put(response) + def _activate_implicitly(self, message: dict) -> None: + """Take a control type's message as proof that the resource manager selected that control type. + + A resource manager sends a control type's messages only after accepting the SelectControlType for it. + So where one arrives while no control type is selected here, the confirmation was lost on the way, + and without this every such message was answered with a TEMPORARY_ERROR, which a resource manager retries forever. + Only a control type this CEM has a handler for is activated. + """ + if self._control_type != ControlType.NO_SELECTION: + return + message_type = message.get("message_type") + if not isinstance(message_type, str): + return + control_type = next( + ( + ct + for prefix, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() + if message_type.startswith(prefix) + ), + None, + ) + if control_type is None or control_type not in self._control_types_handlers: + return + self._logger.warning( + "Received %s while no control type is selected, which a resource manager sends only after accepting %s; " + "taking the confirmation as lost and activating it.", + message_type, + control_type.value, + ) + self._control_type = control_type + def update_control_type(self, control_type: ControlType): """ Callback function that is triggered when we receive diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index d13e5708..c6a33e50 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -664,12 +664,6 @@ async def update_flex_model(self, system_description: FRBCSystemDescription): parse_json_fields=True, ) - async def close(self): - """ - Can be used to stop recurring tasks. - """ - self._logger.debug(f"Closing {self.__class__.__name__} handler") - async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): """ Send FRBC.UsageForecast to FlexMeasures. diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index 09f526f3..70f32b72 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -53,6 +53,11 @@ async def close(self): killing the CEM's request handler and hanging the resource manager. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") + # Work still in flight, such as a schedule being planned, would otherwise go on after the session has ended: + # it posts to FlexMeasures and plans for a resource manager that is no longer there, + # racing the session that replaces it when the resource manager reconnects. + for task in list(self.background_tasks): + task.cancel() @register(InstructionStatusUpdate) def handle_instruction_status_update(self, message: InstructionStatusUpdate): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index ee7d2152..279e7605 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -162,12 +162,22 @@ async def serve_resource_manager( frbc.send_revocations = True cem.register_control_type(frbc) - # create "parallel" tasks for the message producer and consumer - await asyncio.gather( - websocket_consumer(ws, cem), - websocket_producer(ws, cem), - rm_details_watchdog(ws, cem), - ) + # The session lasts as long as the resource manager's messages do. + # When they end, the CEM is closed, which stops what its handlers still have in flight, + # and the producer and the watchdog stop with it: left running, they outlived the connection, + # and a resource manager that reconnected was served by two sessions planning for it at once. + consumer = asyncio.create_task(websocket_consumer(ws, cem)) + side_tasks = [ + asyncio.create_task(websocket_producer(ws, cem)), + asyncio.create_task(rm_details_watchdog(ws, cem)), + ] + try: + await consumer + finally: + await cem.close() + for task in side_tasks: + task.cancel() + await asyncio.gather(*side_tasks, return_exceptions=True) async def configure_site( diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 37f3aaff..190f1841 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -242,3 +242,73 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" assert response["status"] == "OK" + + +async def _details_received(cem, rm_handshake, resource_manager_details) -> None: + """Handshake and details, with every answer taken off the queue: the CEM selects no control type yet.""" + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + while not cem._sending_queue.empty(): + await cem.get_message() + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_control_types_message_activates_it_where_its_confirmation_was_lost( + frbc_system_description, resource_manager_details, rm_handshake +): + """A resource manager sends FRBC messages only once it has accepted FRBC, so one arriving is that acceptance. + + Without this, a lost ReceptionStatus for the SelectControlType left every FRBC message answered with a + TEMPORARY_ERROR, which the resource manager retried forever, and a whole co-simulation hung at its first step. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_control_type_without_a_handler_is_not_activated_implicitly( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + await _details_received(cem, rm_handshake, resource_manager_details) + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + + +@pytest.mark.asyncio +async def test_a_message_before_the_details_activates_nothing(frbc_system_description): + """Before its details, a resource manager has not been offered a control type, so it cannot have accepted one.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await cem.handle_message(frbc_system_description) + assert cem.control_type is None + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_what_its_handlers_have_in_flight(): + """A schedule still being planned for a session that has ended would post and plan for a resource manager no longer there.""" + import asyncio + + cem = CEM(fm_client=None) + frbc = FRBCTest() + cem.register_control_type(frbc) + planning = asyncio.create_task(asyncio.sleep(3600)) + frbc.background_tasks.add(planning) + + await cem.close() + await asyncio.wait([planning], timeout=1) + + assert planning.cancelled() + planning.cancel() # in case it was not, so the test does not leave it running diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 952c2867..50c65bd7 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -763,3 +763,36 @@ def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): for graph in graphs: assert set(graph) == {"title", "sensors"} assert all(isinstance(i, int) for i in graph["sensors"]) + + +@pytest.mark.asyncio +async def test_a_session_ends_with_its_connection(): + """Its producer, its watchdog and its planning stop, so a resource manager that reconnects is served by one session.""" + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0].is_closed() + leftover = [ + t + for t in asyncio.all_tasks() + if t is not asyncio.current_task() + and getattr(t.get_coro(), "__qualname__", "") + in ("websocket_producer", "rm_details_watchdog") + ] + assert leftover == [] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index c35ec8b1..046c4fb2 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -541,3 +541,40 @@ async def test_belief_times_are_forgotten_for_steps_more_than_a_day_old(): assert list(frbc._power_recorded_at) == [ datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc) ] + + +@pytest.mark.asyncio +async def test_a_description_replaced_while_its_plan_is_out_sends_only_the_newer_plan(): + """The plan for a description that a newer one replaced during the round trip is for a device that no longer is.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(storage_status()) + await settle(frbc) + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + newest = max(n for n, _ in batches) + assert sent[newest] > 0 + assert all(count == 0 for n, count in batches if n != newest) From 1cc6261a6c4cf15691a01d7b6c0e9acea5b1b1d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:49:15 +0100 Subject: [PATCH 38/87] test(S2): a message without a type activates nothing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_cem.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 190f1841..6a6fcb0a 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -312,3 +312,16 @@ async def test_closing_the_cem_stops_what_its_handlers_have_in_flight(): assert planning.cancelled() planning.cancel() # in case it was not, so the test does not leave it running + + +@pytest.mark.asyncio +async def test_a_message_without_a_type_activates_nothing( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + from flexmeasures_client.s2.utils import get_unique_id + + await cem.handle_message({"message_id": get_unique_id()}) + assert cem.control_type == ControlType.NO_SELECTION From 9053b861a40626771cfd1a0439932f47119b9f29 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:49:50 +0100 Subject: [PATCH 39/87] docs(changelog): implicit activation and session teardown Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 101f6b47..4ae271a1 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -51,6 +51,8 @@ Removed Fixed ----- +* A control type's message arriving while no control type is selected activates that control type, where the CEM has a handler for it, since a resource manager sends it only after accepting the selection. A lost confirmation used to leave every such message answered with a temporary error, which a resource manager retries forever [`#263 `__] +* Closing a control-type handler cancels the work it still has in flight, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] From 6b1d3eaa0a14f0db41eb61b1d7eb220d6f737593 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:03:50 +0100 Subject: [PATCH 40/87] fix(S2): take only the control type being selected as confirmed, and close any handler What the review of this PR found: - Implicit activation activated whichever control type a message belonged to, so a PEBC message while the CEM selected FRBC switched it to PEBC for good, and the watchdog waited for FRBC forever. Now only a message of the control type the CEM is selecting counts as the resource manager's confirmation, only from NO_SELECTION (a switch between control types is confirmed to the active handler, which is left to it), and the pending confirmation callback is then removed rather than left behind. - ControlTypeHandler.close() read background_tasks, which only FRBC handlers have, so closing any other handler raised AttributeError, which stopped CEM.close() and skipped the teardown of the producer and the watchdog: the very leak this PR set out to fix. A handler without them has nothing to stop. - The account of the hang is what #188 saw, not a lost confirmation it never established: a CEM left in NO_SELECTION while a co-simulation's reconnecting resource manager kept sending FRBC messages and retrying their temporary errors. The docstrings, the changelog and the close() comment now say so, and that closing stops recording in flight too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- src/flexmeasures_client/s2/cem.py | 41 +++++---- .../s2/control_types/__init__.py | 18 ++-- tests/s2/test_cem.py | 86 ++++++++++++++++++- 4 files changed, 119 insertions(+), 30 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 4ae271a1..d67516f4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -51,8 +51,8 @@ Removed Fixed ----- -* A control type's message arriving while no control type is selected activates that control type, where the CEM has a handler for it, since a resource manager sends it only after accepting the selection. A lost confirmation used to leave every such message answered with a temporary error, which a resource manager retries forever [`#263 `__] -* Closing a control-type handler cancels the work it still has in flight, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] +* A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] +* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index ceb27934..f46c4fdb 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -97,6 +97,8 @@ def __init__( self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() + #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. + self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type if not logger: @@ -231,35 +233,42 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): await self._sending_queue.put(response) def _activate_implicitly(self, message: dict) -> None: - """Take a control type's message as proof that the resource manager selected that control type. - - A resource manager sends a control type's messages only after accepting the SelectControlType for it. - So where one arrives while no control type is selected here, the confirmation was lost on the way, - and without this every such message was answered with a TEMPORARY_ERROR, which a resource manager retries forever. - Only a control type this CEM has a handler for is activated. + """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. + + A resource manager sends a control type's messages only once that control type is active on its side. + So where one arrives for the control type this CEM asked for, while the CEM still waits for the confirmation, + the resource manager is working in it already. #188 saw the CEM stay in NO_SELECTION while a resource manager, + reconnecting during setup in a co-simulation, kept sending FRBC messages, each answered with a TEMPORARY_ERROR + and retried; whether the confirmation was lost or never sent was not established. + Only the control type being selected is activated, never another one a message happens to belong to, + and its pending confirmation is then no longer waited for. """ - if self._control_type != ControlType.NO_SELECTION: + if self._control_type != ControlType.NO_SELECTION or self._selecting is None: return + control_type, selection_message_id = self._selecting message_type = message.get("message_type") if not isinstance(message_type, str): return - control_type = next( + prefix = next( ( - ct - for prefix, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() - if message_type.startswith(prefix) + p + for p, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() + if ct == control_type ), None, ) - if control_type is None or control_type not in self._control_types_handlers: + if prefix is None or not message_type.startswith(prefix): + return + if control_type not in self._control_types_handlers: return self._logger.warning( - "Received %s while no control type is selected, which a resource manager sends only after accepting %s; " - "taking the confirmation as lost and activating it.", + "Received %s before the resource manager confirmed the selection of %s; taking it as the confirmation.", message_type, control_type.value, ) - self._control_type = control_type + self.success_callbacks.pop(selection_message_id, None) + self.failure_callbacks.pop(selection_message_id, None) + self.update_control_type(control_type) def update_control_type(self, control_type: ControlType): """ @@ -267,6 +276,7 @@ def update_control_type(self, control_type: ControlType): a confirmation that the message has been received. """ self._control_type = control_type + self._selecting = None async def get_message(self) -> dict: """Call this function to get the messages to be sent to the RM @@ -317,6 +327,7 @@ async def activate_control_type( message_id, self.update_control_type, control_type=control_type ) + self._selecting = (control_type, message_id) await self._sending_queue.put( SelectControlType(message_id=message_id, control_type=control_type) ) diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index 70f32b72..c708c2b2 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -44,19 +44,19 @@ async def send_message(self, message: BaseModel) -> None: await self._sending_queue.put(message) async def close(self): - """Release any resources / stop recurring tasks for this handler. + """Stop this handler's work in flight, when the CEM closes as its websocket tears down. - Default no-op so CEM.close(), which calls close() on every registered handler when a - websocket tears down, works for any control-type handler. - Subclasses that own recurring tasks override this. - Without it, a handler lacking close() raised AttributeError inside CEM.close(), + Defined here so that CEM.close(), which calls close() on every registered handler, works for any of them: + a handler lacking close() raised AttributeError inside CEM.close(), killing the CEM's request handler and hanging the resource manager. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") - # Work still in flight, such as a schedule being planned, would otherwise go on after the session has ended: - # it posts to FlexMeasures and plans for a resource manager that is no longer there, - # racing the session that replaces it when the resource manager reconnects. - for task in list(self.background_tasks): + # Work still in flight would otherwise go on after the session has ended, + # planning for a resource manager that is no longer there and racing the session that replaces it. + # That includes recording what the resource manager last reported, which is stopped too: + # a reading cut off mid-post is reported again by a resource manager that reconnects. + # Not every handler keeps background tasks, so a handler without them has nothing to stop. + for task in list(getattr(self, "background_tasks", ())): task.cancel() @register(InstructionStatusUpdate) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 6a6fcb0a..752793da 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -4,6 +4,7 @@ from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types import ControlTypeHandler from flexmeasures_client.s2.control_types.FRBC import FRBCTest @@ -244,6 +245,12 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh assert response["status"] == "OK" +class _PlainPEBC(ControlTypeHandler): + """A handler for a second control type, which keeps no background tasks.""" + + _control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + + async def _details_received(cem, rm_handshake, resource_manager_details) -> None: """Handshake and details, with every answer taken off the queue: the CEM selects no control type yet.""" await cem.handle_message(rm_handshake) @@ -254,23 +261,81 @@ async def _details_received(cem, rm_handshake, resource_manager_details) -> None @pytest.mark.asyncio -async def test_a_control_types_message_activates_it_where_its_confirmation_was_lost( +async def test_a_message_of_the_control_type_being_selected_counts_as_its_confirmation( frbc_system_description, resource_manager_details, rm_handshake ): - """A resource manager sends FRBC messages only once it has accepted FRBC, so one arriving is that acceptance. + """A resource manager sends a control type's messages only once it is working in that control type. - Without this, a lost ReceptionStatus for the SelectControlType left every FRBC message answered with a - TEMPORARY_ERROR, which the resource manager retried forever, and a whole co-simulation hung at its first step. + #188 saw a CEM stay in NO_SELECTION while a reconnecting resource manager kept sending FRBC messages, + each answered with a temporary error and retried. """ cem = CEM(fm_client=None) cem.register_control_type(FRBCTest()) await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() await cem.handle_message(frbc_system_description) assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL response = await cem.get_message() assert response["status"] == ReceptionStatusValues.OK.value + assert selection["message_id"] not in cem.success_callbacks # no longer waited for + + +@pytest.mark.asyncio +async def test_a_message_of_another_control_type_is_not_taken_as_a_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """Selecting one control type and receiving another's message switches to neither.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.POWER_ENVELOPE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_switch_between_control_types_is_left_to_its_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """While one control type is active, a message of the one being switched to is still that of an unconfirmed switch. + + Its confirmation is awaited by the active handler, which this CEM would not clear, so it is left to arrive. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.POWER_ENVELOPE_BASED_CONTROL + + +@pytest.mark.asyncio +async def test_a_confirmed_selection_is_no_longer_pending( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], + status=ReceptionStatusValues.OK, + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + assert cem._selecting is None @pytest.mark.asyncio @@ -279,6 +344,7 @@ async def test_a_control_type_without_a_handler_is_not_activated_implicitly( ): cem = CEM(fm_client=None) await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") await cem.handle_message(frbc_system_description) @@ -325,3 +391,15 @@ async def test_a_message_without_a_type_activates_nothing( await cem.handle_message({"message_id": get_unique_id()}) assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_handler_without_background_tasks_closes(): + """Closing every handler is what ends a session, so one that keeps no tasks must close too.""" + cem = CEM(fm_client=None) + plain = _PlainPEBC() + cem.register_control_type(plain) + if hasattr(plain, "background_tasks"): + del plain.background_tasks + await cem.close() + assert cem.is_closed() From 69f5dbed16d616ec3aa1d5b2fe0be6eacf76d79b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:04:45 +0100 Subject: [PATCH 41/87] test(S2): a typeless message activates nothing while a selection is pending Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_cem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 752793da..4299fa8a 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -387,6 +387,7 @@ async def test_a_message_without_a_type_activates_nothing( cem = CEM(fm_client=None) cem.register_control_type(FRBCTest()) await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") from flexmeasures_client.s2.utils import get_unique_id await cem.handle_message({"message_id": get_unique_id()}) From 80ad945dfba7862fb6608e01535744acb62d7026 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:17:10 +0100 Subject: [PATCH 42/87] feat(S2): a clock the CEM and its handlers go by, which a co-simulation can set Carved from #188, rewritten for the library rather than transcribed. #188 let a co-simulation's resource manager wrap each S2 message with the simulated time, {"message": ..., "metadata": {"dt": ...}}, and patched a lambda returning that time onto the CEM and every handler, for every resource manager, live ones included. The lambda returned dt as it came, an ISO string, and it reached neither the CEM's timer, its measurement binning, nor TUNES' timer, which kept the wall clock. - flexmeasures_client.s2.clock holds a Clock protocol, a WallClock, and a SimulatedClock, which stands still between settings, refuses a time without a zone, and goes by the wall clock until it is first set: the handshake of each connection runs before any simulated time has arrived. - Handler.now() reads its handler's clock, in its time zone. The CEM gives its clock to every handler it registers, so they share one time. FRBCSimple and TUNES lose their copies of now(). The CEM's and TUNES' timers, and the CEM's measurement binning, go by now() instead of the wall clock; a timer handed in without a zone, as they all were, is read in the handler's time zone. - CEM(accept_simulation_metadata=True) takes the wrapper, sets its clock to the time it carries and handles the message inside. It needs a SimulatedClock and makes one where none is given. Off, which is the default, a wrapped message is refused with INVALID_MESSAGE and the reason, since the wrapper is not S2 and a live CEM should not let a resource manager set its time. A time that cannot be read is refused the same way. - The example server turns it on with S2_ACCEPT_SIMULATION_METADATA, which the co-simulation's stack will set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/__init__.py | 8 +- src/flexmeasures_client/s2/cem.py | 71 +++++- src/flexmeasures_client/s2/clock.py | 55 +++++ .../s2/control_types/FRBC/frbc_simple.py | 3 - .../s2/control_types/FRBC/frbc_tunes.py | 9 +- .../s2/script/websockets_server.py | 9 + tests/s2/test_clock.py | 208 ++++++++++++++++++ tests/s2/test_example_end_to_end.py | 27 +++ 8 files changed, 379 insertions(+), 11 deletions(-) create mode 100644 src/flexmeasures_client/s2/clock.py create mode 100644 tests/s2/test_clock.py diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index a1fcadc5..04a7cbcb 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -21,6 +21,7 @@ ) +from flexmeasures_client.s2.clock import Clock, WallClock from flexmeasures_client.s2.utils import SizeLimitOrderedDict, get_message_id @@ -126,6 +127,8 @@ def __init__(self, max_size: int = 100, timezone: str = "UTC") -> None: self.outgoing_messages_status = SizeLimitOrderedDict(max_size=max_size) self._timezone = ZoneInfo(timezone) + #: The time this handler goes by; a CEM gives each handler it registers its own clock. + self._clock: Clock = WallClock() self.discover() @@ -249,5 +252,6 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: subject_message_id=str(message.message_id), status=ReceptionStatusValues.OK ) - def now(self): - return datetime.now(self._timezone) + def now(self) -> datetime: + """The current moment by this handler's clock, in its time zone.""" + return self._clock.now().astimezone(self._timezone) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index f46c4fdb..5f2676d9 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -33,12 +33,14 @@ from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2 import Handler, register +from flexmeasures_client.s2.clock import Clock, SimulatedClock from flexmeasures_client.s2.control_types import ControlTypeHandler from flexmeasures_client.s2.utils import ( get_latest_compatible_version, get_reception_status, get_unique_id, ) +from flexmeasures_client.s2.wrapper import MetaData _LOGGER = logging.getLogger(__name__) @@ -85,12 +87,32 @@ def __init__( timers: dict[str, datetime] | None = None, datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, + clock: Clock | None = None, + accept_simulation_metadata: bool = False, **kwargs, ) -> None: """ Customer Energy Manager (CEM) + + :param clock: the time this CEM and every handler it registers go by; + the wall clock by default. + :param accept_simulation_metadata: whether to take messages wrapped with a simulated time + (see flexmeasures_client.s2.wrapper.S2Wrapper), + as a co-simulation sends them, and set the clock to that time. + Off by default, since the wrapper is not part of S2, + and a live CEM should not let a resource manager set its time. + On, the clock has to be a SimulatedClock, and one is made where none is given. """ super(CEM, self).__init__() + if accept_simulation_metadata: + clock = clock if clock is not None else SimulatedClock() + if not isinstance(clock, SimulatedClock): + raise ValueError( + "A CEM that accepts simulation metadata sets its clock from it, so it needs a SimulatedClock." + ) + if clock is not None: + self._clock = clock + self._accept_simulation_metadata = accept_simulation_metadata self._fm_client = fm_client self._sending_queue = Queue() @@ -111,8 +133,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. + due_time = due_time.replace(tzinfo=self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -169,6 +195,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): # add fm_client to control_type handler control_type_handler._fm_client = self._fm_client + # The handler goes by the CEM's time, which a co-simulation sets. + control_type_handler._clock = self._clock + # add sending queue control_type_handler._sending_queue = self._sending_queue @@ -198,6 +227,12 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): message, dict ) # what a message is from here on, whatever form it came in + if "metadata" in message and "message" in message: + unwrapped = self._unwrap(message) + if unwrapped is None: + return + message = unwrapped + self._logger.debug(f"Received: {message}") self._activate_implicitly(message) @@ -232,6 +267,38 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if response is not None: await self._sending_queue.put(response) + def _unwrap(self, wrapped: dict) -> dict | None: + """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. + + Where this CEM does not accept the wrapper, or the time cannot be read, the message is refused, + and None is returned so that it is not handled. + """ + inner = wrapped["message"] if isinstance(wrapped["message"], dict) else {} + reason = None + if not self._accept_simulation_metadata: + reason = ( + "This CEM does not accept messages wrapped with simulation metadata." + ) + else: + try: + moment = MetaData.model_validate(wrapped["metadata"]).dt + assert isinstance(self._clock, SimulatedClock) + self._clock.set(moment) + except (pydantic.ValidationError, ValueError, TypeError) as exc: + reason = f"The simulated time in this message's metadata cannot be used: {exc}" + if reason is None: + return inner + self._logger.warning("%s Refusing %s.", reason, inner.get("message_type")) + if inner.get("message_id") is not None: + self._sending_queue.put_nowait( + ReceptionStatus( + subject_message_id=inner["message_id"], + status=ReceptionStatusValues.INVALID_MESSAGE, + diagnostic_label=reason, + ) + ) + return None + def _activate_implicitly(self, message: dict) -> None: """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. @@ -399,7 +466,7 @@ async def handle_power_measurement(self, message: PowerMeasurement): ) # Compute bin - now = datetime.now(self._timezone) + now = self.now() m = self._minimum_measurement_period // pd.Timedelta(minutes=1) bin_end = now.replace( second=0, microsecond=0, minute=(now.minute // m) * m diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py new file mode 100644 index 00000000..15fdaa2f --- /dev/null +++ b/src/flexmeasures_client/s2/clock.py @@ -0,0 +1,55 @@ +"""The time an S2 CEM and its handlers go by. + +A CEM that runs live goes by the wall clock. +One that steers a co-simulation goes by the time the simulation says it is, +which a resource manager sends along with its messages (see CEM's accept_simulation_metadata), +so that what the CEM records and plans is placed at simulated moments rather than at the moment it ran. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Protocol + + +class Clock(Protocol): + def now(self) -> datetime: + """The current moment, timezone-aware.""" + ... + + +class WallClock: + """The time it is.""" + + def now(self) -> datetime: + return datetime.now(timezone.utc) + + +class SimulatedClock: + """The time a simulation says it is, as last set. + + It stands still between settings, as a simulation's time does between the messages that carry it. + Until it is first set it goes by the wall clock, which is the time the handshake of each connection runs on, + since a resource manager sends the simulated time only with the messages that follow it. + """ + + def __init__(self, start: datetime | None = None) -> None: + self._now: datetime | None = None + if start is not None: + self.set(start) + + def set(self, moment: datetime) -> None: + if moment.tzinfo is None or moment.utcoffset() is None: + raise ValueError( + f"A simulated time has to say which time zone it is in, and {moment.isoformat()} does not." + ) + self._now = moment + + @property + def is_set(self) -> bool: + return self._now is not None + + def now(self) -> datetime: + if self._now is None: + return datetime.now(timezone.utc) + return self._now diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 694541e2..ccd43d1f 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -284,9 +284,6 @@ def __init__( # of the FRBC.SystemDescription. self._valid_from_shift = valid_from_shift - def now(self): - return datetime.now(self._timezone) - def fill_rate_unit(self) -> str: """The unit a rate of change of the fill level is recorded in. diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index c6a33e50..ea11080a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -169,8 +169,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. + due_time = due_time.replace(tzinfo=self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -195,9 +199,6 @@ def _is_timer_due(self, name: str) -> bool: ) return False - def now(self): - return datetime.now(self._timezone) - async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 279e7605..9c3f9988 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -32,6 +32,13 @@ # Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, # with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 + +#: Whether resource managers may send messages wrapped with a simulated time, as a co-simulation does. +ACCEPT_SIMULATION_METADATA = os.getenv("S2_ACCEPT_SIMULATION_METADATA", "").lower() in ( + "1", + "true", + "yes", +) SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. @@ -141,6 +148,8 @@ async def serve_resource_manager( sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, + # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. + accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) frbc = FRBCSimple( power_sensor_id=sensors["power"]["id"], diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py new file mode 100644 index 00000000..f1394969 --- /dev/null +++ b/tests/s2/test_clock.py @@ -0,0 +1,208 @@ +"""The time an S2 CEM goes by, and a co-simulation's way of setting it.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +import pandas as pd +import pytest +from s2python.common import ReceptionStatusValues + +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.clock import SimulatedClock, WallClock +from flexmeasures_client.s2.control_types.FRBC import FRBCTest +from flexmeasures_client.s2.utils import get_unique_id + +SIMULATED = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) + + +def test_a_simulated_clock_stands_still_between_settings(): + clock = SimulatedClock() + clock.set(SIMULATED) + assert clock.now() == SIMULATED + assert clock.now() == SIMULATED + + +def test_a_simulated_clock_goes_by_the_wall_clock_until_it_is_set(): + clock = SimulatedClock() + assert not clock.is_set + assert abs(clock.now() - WallClock().now()) < timedelta(seconds=5) + + +def test_a_simulated_time_without_a_zone_is_refused(): + """#188 took naive times as Europe/Amsterdam, which a simulation in UTC shifts by an hour or two.""" + with pytest.raises(ValueError, match="time zone"): + SimulatedClock().set(datetime(2022, 3, 1, 12, 0)) + + +def test_a_handler_registered_with_a_cem_goes_by_its_clock(): + clock = SimulatedClock(SIMULATED) + cem = CEM(fm_client=None, clock=clock) + frbc = FRBCTest() + cem.register_control_type(frbc) + assert frbc.now() == SIMULATED + assert cem.now() == SIMULATED + later = SIMULATED + timedelta(hours=3) + clock.set(later) + assert frbc.now() == later # the same clock, not a copy of its time + + +def test_now_is_given_in_the_handlers_time_zone(): + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED)) + frbc = FRBCTest() + frbc._timezone = ZoneInfo("Europe/Amsterdam") + cem.register_control_type(frbc) + assert frbc.now().tzinfo == ZoneInfo("Europe/Amsterdam") + assert frbc.now() == SIMULATED + + +def test_accepting_simulation_metadata_needs_a_simulated_clock(): + with pytest.raises(ValueError, match="SimulatedClock"): + CEM(fm_client=None, clock=WallClock(), accept_simulation_metadata=True) + + +def _wrapped(message: dict, dt) -> dict: + return {"message": message, "metadata": {"dt": dt}} + + +@pytest.mark.asyncio +async def test_a_wrapped_message_sets_the_clock_and_is_handled(rm_handshake): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + _wrapped(rm_handshake.model_dump(mode="json"), "2022-03-01T12:00:00+00:00") + ) + assert cem.now() == SIMULATED + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_wrapped_message_is_refused_by_a_cem_that_does_not_accept_it( + rm_handshake, +): + """The wrapper is not S2, and a live CEM does not let a resource manager set its time.""" + cem = CEM(fm_client=None) + before = cem.now() + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, "2022-03-01T12:00:00+00:00")) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert reception["subject_message_id"] == message["message_id"] + assert cem._sending_queue.empty() # the handshake itself was not answered + assert cem.now() - before < timedelta(seconds=5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dt", ["2022-03-01T12:00:00", "not a time"]) +async def test_a_wrapped_message_with_an_unusable_time_is_refused(rm_handshake, dt): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, dt)) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert not cem._clock.is_set + + +@pytest.mark.asyncio +async def test_a_wrapper_around_no_message_is_refused_without_an_answer(): + """Nothing to answer: a reception status needs the id of the message it is about.""" + cem = CEM(fm_client=None) + await cem.handle_message({"message": "garbage", "metadata": {}}) + assert cem._sending_queue.empty() + + +def test_a_cem_timer_runs_on_simulated_time(): + # Off a period boundary: on one, the next due time rounds to the moment itself. + start = SIMULATED + timedelta(minutes=1) + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + assert cem._is_timer_due("power") + assert not cem._is_timer_due("power") + clock.set(start + timedelta(hours=1)) + assert cem._is_timer_due("power") + + +def test_a_timer_handed_in_without_a_zone_is_read_in_the_handlers_zone(): + cem = CEM( + fm_client=None, + clock=SimulatedClock(SIMULATED), + timers={"power": datetime(2022, 3, 1, 13, 0)}, + ) + assert not cem._is_timer_due("power") + + +def test_the_example_server_accepts_simulation_metadata_only_when_told_to(monkeypatch): + import importlib + + from flexmeasures_client.s2.script import websockets_server + + monkeypatch.setenv("S2_ACCEPT_SIMULATION_METADATA", "1") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is True + monkeypatch.delenv("S2_ACCEPT_SIMULATION_METADATA") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is False + + +def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) + + start = SIMULATED + timedelta(minutes=1) + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, + ) + cem.register_control_type(tunes) + tunes._timers["usage_forecast"] = datetime( + 2022, 3, 1, 12, 30 + ) # zoneless, as timers used to be + assert not tunes._is_timer_due("usage_forecast") + clock.set(start + timedelta(hours=1)) + assert tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +async def test_a_power_measurement_is_binned_by_simulated_time(): + """Binned by the wall clock, a co-simulation's measurements, stamped in 2022, fell outside every bin and were dropped.""" + from unittest.mock import AsyncMock + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.client import FlexMeasuresClient + + fm = AsyncMock(FlexMeasuresClient) + clock = SimulatedClock(datetime(2022, 3, 1, 12, 7, tzinfo=timezone.utc)) + cem = CEM(fm_client=fm, clock=clock, power_sensor_id={"ELECTRIC.POWER.L1": 7}) + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, 2, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + posted = fm.post_sensor_data.await_args + assert posted is not None + assert pd.Timestamp(posted.kwargs["start"]) == pd.Timestamp( + "2022-03-01T12:00:00+00:00" + ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 50c65bd7..e2eca6c4 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -796,3 +796,30 @@ def capture(**kwargs): in ("websocket_producer", "rm_details_watchdog") ] assert leftover == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("accept", [False, True]) +async def test_the_server_hands_its_simulation_flag_to_each_cem(accept): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with ( + patch.object(server, "CEM", side_effect=capture), + patch.object(server, "ACCEPT_SIMULATION_METADATA", accept), + ): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["accept_simulation_metadata"] is accept From 5b9793ec286ca1d4818db0337d04e312e68c85fc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:18:17 +0100 Subject: [PATCH 43/87] docs(changelog): the clock and the simulation metadata Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d67516f4..30f19ad3 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -25,6 +25,8 @@ Added * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] +* ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] Changed ------- From 47c695a262d74a71903bc410ac3ddb606dfd7d33 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:30:34 +0100 Subject: [PATCH 44/87] fix(S2): what the review of the clock found - On a period boundary, the timers' next due time rounded to the moment itself, so a clock standing still there, as a co-simulation's does between messages, found every timer due on every call. The next due time is now a whole period later. Both tests had started a minute off the boundary to dodge it; they now start on it. - A wrapper with no S2 message inside set the clock, and then crashed the consumer on a reception status for a message without an id. It is refused before the clock is touched. - A timer handed in without a zone was compared with the host's local time before, and is read as that again, where the first version read it in the handler's time zone, which is UTC by default. - The measurement binning was claimed as moved onto simulated time, but binning on now() never catches a co-simulation's measurements, each stamped at the start of a quarter hour that has passed by the time it arrives. #188 bypassed the binning instead. The claim and its test are dropped, and posting a co-simulation's measurements is left to the recorder that is planned next. - Docs: the co-simulation's resource manager wraps every message from its handshake on, so a CEM without accept_simulation_metadata refuses it from the first message, which the docstring now says; the simulated clock's docstring no longer claims the handshake runs on wall time, and says that background work reads the clock as it stands. The server's flag moved out of the middle of the price constants, and a docstring line breaks after punctuation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 28 ++++-- src/flexmeasures_client/s2/clock.py | 5 +- .../s2/control_types/FRBC/frbc_tunes.py | 8 +- .../s2/script/websockets_server.py | 2 +- tests/s2/test_clock.py | 92 +++++++++++-------- 5 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 5f2676d9..e1adf215 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -96,11 +96,12 @@ def __init__( :param clock: the time this CEM and every handler it registers go by; the wall clock by default. - :param accept_simulation_metadata: whether to take messages wrapped with a simulated time - (see flexmeasures_client.s2.wrapper.S2Wrapper), - as a co-simulation sends them, and set the clock to that time. + :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, + and set the clock to that time (see flexmeasures_client.s2.wrapper.S2Wrapper). Off by default, since the wrapper is not part of S2, and a live CEM should not let a resource manager set its time. + A resource manager that wraps its messages, from its handshake on, + is then refused with INVALID_MESSAGE and cannot connect. On, the clock has to be a SimulatedClock, and one is made where none is given. """ super(CEM, self).__init__() @@ -137,8 +138,8 @@ def _is_timer_due(self, name: str) -> bool: now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. - due_time = due_time.replace(tzinfo=self._timezone) + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -155,6 +156,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: @@ -270,12 +275,17 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): def _unwrap(self, wrapped: dict) -> dict | None: """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. - Where this CEM does not accept the wrapper, or the time cannot be read, the message is refused, - and None is returned so that it is not handled. + Where this CEM does not accept the wrapper, the time cannot be read, or there is no message inside, + the message is refused, the clock is left as it was, and None is returned so that nothing is handled. """ - inner = wrapped["message"] if isinstance(wrapped["message"], dict) else {} + inner = wrapped["message"] reason = None - if not self._accept_simulation_metadata: + if not isinstance(inner, dict) or not isinstance( + inner.get("message_type"), str + ): + reason = "There is no S2 message inside this wrapper." + inner = inner if isinstance(inner, dict) else {} + elif not self._accept_simulation_metadata: reason = ( "This CEM does not accept messages wrapped with simulation metadata." ) diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py index 15fdaa2f..55646905 100644 --- a/src/flexmeasures_client/s2/clock.py +++ b/src/flexmeasures_client/s2/clock.py @@ -29,8 +29,9 @@ class SimulatedClock: """The time a simulation says it is, as last set. It stands still between settings, as a simulation's time does between the messages that carry it. - Until it is first set it goes by the wall clock, which is the time the handshake of each connection runs on, - since a resource manager sends the simulated time only with the messages that follow it. + Until it is first set it goes by the wall clock, for a resource manager that does not wrap its first messages; + one that wraps every message from its handshake on, as the co-simulation's does, sets it with the first. + Background work, such as a schedule being planned, reads it as it stands, which may be a later message's time. """ def __init__(self, start: datetime | None = None) -> None: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index ea11080a..23babaf3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -173,8 +173,8 @@ def _is_timer_due(self, name: str) -> bool: now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. - due_time = due_time.replace(tzinfo=self._timezone) + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -191,6 +191,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 9c3f9988..21cdd974 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -32,6 +32,7 @@ # Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, # with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 +SEED_PRICES_EVERY = timedelta(hours=12) #: Whether resource managers may send messages wrapped with a simulated time, as a co-simulation does. ACCEPT_SIMULATION_METADATA = os.getenv("S2_ACCEPT_SIMULATION_METADATA", "").lower() in ( @@ -39,7 +40,6 @@ "true", "yes", ) -SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. #: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py index f1394969..946bae72 100644 --- a/tests/s2/test_clock.py +++ b/tests/s2/test_clock.py @@ -5,14 +5,12 @@ from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo -import pandas as pd import pytest from s2python.common import ReceptionStatusValues from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.clock import SimulatedClock, WallClock from flexmeasures_client.s2.control_types.FRBC import FRBCTest -from flexmeasures_client.s2.utils import get_unique_id SIMULATED = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) @@ -113,23 +111,22 @@ async def test_a_wrapper_around_no_message_is_refused_without_an_answer(): def test_a_cem_timer_runs_on_simulated_time(): - # Off a period boundary: on one, the next due time rounds to the moment itself. - start = SIMULATED + timedelta(minutes=1) - clock = SimulatedClock(start) + # On a period boundary, which a co-simulation's times are: the ceiling of the moment is the moment itself. + clock = SimulatedClock(SIMULATED) cem = CEM(fm_client=None, clock=clock) assert cem._is_timer_due("power") - assert not cem._is_timer_due("power") - clock.set(start + timedelta(hours=1)) + assert not cem._is_timer_due("power") # a clock standing still on a boundary + clock.set(SIMULATED + timedelta(hours=1)) assert cem._is_timer_due("power") -def test_a_timer_handed_in_without_a_zone_is_read_in_the_handlers_zone(): - cem = CEM( - fm_client=None, - clock=SimulatedClock(SIMULATED), - timers={"power": datetime(2022, 3, 1, 13, 0)}, - ) +def test_a_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + """As it was compared before, with a zoneless datetime.now(), which is the host's local time.""" + local = (SIMULATED + timedelta(minutes=30)).astimezone().replace(tzinfo=None) + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED), timers={"power": local}) assert not cem._is_timer_due("power") + cem._clock.set(SIMULATED + timedelta(minutes=31)) + assert cem._is_timer_due("power") def test_the_example_server_accepts_simulation_metadata_only_when_told_to(monkeypatch): @@ -148,7 +145,7 @@ def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): FillRateBasedControlTUNES, ) - start = SIMULATED + timedelta(minutes=1) + start = SIMULATED clock = SimulatedClock(start) cem = CEM(fm_client=None, clock=clock) tunes = FillRateBasedControlTUNES( @@ -170,39 +167,54 @@ def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): state_of_charge_sensor_id=18, ) cem.register_control_type(tunes) - tunes._timers["usage_forecast"] = datetime( - 2022, 3, 1, 12, 30 - ) # zoneless, as timers used to be - assert not tunes._is_timer_due("usage_forecast") + assert tunes._is_timer_due("usage_forecast") + assert not tunes._is_timer_due( + "usage_forecast" + ) # a clock standing still on a boundary clock.set(start + timedelta(hours=1)) assert tunes._is_timer_due("usage_forecast") @pytest.mark.asyncio -async def test_a_power_measurement_is_binned_by_simulated_time(): - """Binned by the wall clock, a co-simulation's measurements, stamped in 2022, fell outside every bin and were dropped.""" - from unittest.mock import AsyncMock +@pytest.mark.parametrize("inner", ["garbage", {"no": "type"}]) +async def test_a_wrapper_around_no_message_leaves_the_clock_alone(inner): + """With the flag on, too: the clock is set only for a message that will be handled.""" + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + {"message": inner, "metadata": {"dt": "2022-03-01T12:00:00+00:00"}} + ) + assert not cem._clock.is_set + assert cem._sending_queue.empty() - from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue - from flexmeasures_client.client import FlexMeasuresClient +def test_a_tunes_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) - fm = AsyncMock(FlexMeasuresClient) - clock = SimulatedClock(datetime(2022, 3, 1, 12, 7, tzinfo=timezone.utc)) - cem = CEM(fm_client=fm, clock=clock, power_sensor_id={"ELECTRIC.POWER.L1": 7}) - await cem.handle_message( - PowerMeasurement( - message_id=get_unique_id(), - measurement_timestamp=datetime(2022, 3, 1, 12, 2, tzinfo=timezone.utc), - values=[ - PowerValue( - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 - ) - ], - ) + start = SIMULATED + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, ) - posted = fm.post_sensor_data.await_args - assert posted is not None - assert pd.Timestamp(posted.kwargs["start"]) == pd.Timestamp( - "2022-03-01T12:00:00+00:00" + cem.register_control_type(tunes) + tunes._timers["usage_forecast"] = ( + (start + timedelta(minutes=30)).astimezone().replace(tzinfo=None) ) + assert not tunes._is_timer_due("usage_forecast") From 6a12ea4cb5088eb7551bda9932b12909e156ffac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:44:35 +0100 Subject: [PATCH 45/87] feat(S2): a site of its own for each resource manager, and the co-simulation stack Carved from #188, rewritten for the library rather than transcribed. #188 made every CEM find or create a FlexMeasures asset named after the resource manager that connected, with its sensors (config_utils.configure_site, a 12-tuple, a module-level id cache and logging configured at import), and registered an FRBCSimple wired to them, for every deployment. It also seeded constant prices from the wall clock's today with a belief time in 2026, which a simulation of 2022 never reads. - flexmeasures_client.s2.site holds FRBC_SITE_SENSORS, the sensors an FRBC site records on, which the example's table becomes; ensure_site, which finds or makes a site of a name with its sensors and graphs, and is safe to run again; and FRBCSiteConfigurator, which gives each resource manager a site named after it and an FRBCSimple wired to it, planning from now over 24 hours as #188 did, with the asset type, time zone, asset specs and the power sensor's attributes as options, and the site's id remembered for a reconnect. The sensor names the co-simulation reads by ("price", "production price", "power", "measured-power", "state of charge") are kept. - CEM(site_configurator=...) configures in the background once a resource manager's details arrive, and wires the site's measured-power sensor. Until the handler exists, the active control type's messages are answered with TEMPORARY_ERROR, which the resource manager retries; before, the CEM raised a KeyError on them. Closing the CEM stops a configuration in flight. - The example server takes S2_SITE_PER_RESOURCE_MANAGER: no fixed site and no demo prices, and a configurator for the server's lifetime, whose power sensors say the heat pumps cannot produce (is_strictly_non_positive), as #188's did. Its fixed site is set up with ensure_site, and gains the production price and measured-power sensors. - examples/S2-co-simulation holds the docker-compose overlay the co-simulation runs on, moved from #188's root docker-compose.override.yml, which compose applied to anyone running the repo's base file. It sets both switches, uses s2-python as this repo locks it (0.8.1, which the co-simulation's resource managers use too, where #188 pinned 0.8.2), runs one worker command for all three workers, logs at INFO, and drops the py-spy debugging setup. Its README says how to keep it apart from a development database: FlexMeasures keeps that database's data inside its checkout, so a project name does not. Not carried from #188: mirroring measured power onto the community's aggregate-power sensor (the maintainer dropped it, since nothing reads it back), and the price seeding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 30 ++ .../S2-co-simulation/docker-compose.cem.yml | 164 +++++++++++ src/flexmeasures_client/s2/cem.py | 60 +++- .../s2/script/websockets_server.py | 177 +++++------- src/flexmeasures_client/s2/site.py | 264 ++++++++++++++++++ tests/s2/test_example_end_to_end.py | 83 +++++- tests/s2/test_site.py | 264 ++++++++++++++++++ 7 files changed, 933 insertions(+), 109 deletions(-) create mode 100644 examples/S2-co-simulation/README.md create mode 100644 examples/S2-co-simulation/docker-compose.cem.yml create mode 100644 src/flexmeasures_client/s2/site.py create mode 100644 tests/s2/test_site.py diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md new file mode 100644 index 00000000..fa95fa8e --- /dev/null +++ b/examples/S2-co-simulation/README.md @@ -0,0 +1,30 @@ +# A FlexMeasures stack for an S2 co-simulation + +`docker-compose.cem.yml` runs FlexMeasures with one S2 CEM per simulated home, +for a co-simulation whose resource managers connect over websockets, such as a +community of homes steered at once. How to start it is at the top of the file. + +Each CEM is the example server (`src/flexmeasures_client/s2/script/websockets_server.py`) +with two switches on: + +- `S2_ACCEPT_SIMULATION_METADATA`: a resource manager wraps each message with the + simulated time, `{"message": ..., "metadata": {"dt": ...}}`, and the CEM, and every + handler it registers, go by that time (see `flexmeasures_client.s2.clock`). + `dt` has to name its time zone. Without the switch, a wrapped message is refused + as invalid, from the handshake on. +- `S2_SITE_PER_RESOURCE_MANAGER`: each resource manager gets a site of its own, + named after it, with the sensors in `flexmeasures_client.s2.site.FRBC_SITE_SENSORS`. + Messages are answered with a temporary error until its site is set up, which a + resource manager retries. Its schedules start from the simulated now and cover + 24 hours. The site's prices are for whoever runs the simulation to post. + +A co-simulation reads the sites by sensor name: `price`, `production price`, +`power` (the schedule), `measured-power` (what the home measured) and +`state of charge`. + +The stack uses FlexMeasures' own development database service, `dev-db`, whose +data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the +flexmeasures checkout. A compose project name of its own does not separate that, +so run the stack from a checkout of its own (a git worktree will do) to keep a +development database you care about out of it. Its host ports, 5000, 5433, 6380 +and 8080 to 8084, have to be free too. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml new file mode 100644 index 00000000..da7f58dd --- /dev/null +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -0,0 +1,164 @@ +# ------------------------------------------------------------------ +# A FlexMeasures stack with one S2 CEM per simulated home, for a co-simulation +# whose resource managers connect over websockets, such as the community +# simulations that steer several homes at once. +# +# It is an overlay on FlexMeasures' own docker-compose.yml, and runs the CEMs +# from this checkout. With the flexmeasures and flexmeasures-client repos side +# by side, run it from the flexmeasures folder (which holds the Dockerfile): +# +# docker compose \ +# -f docker-compose.yml \ +# -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \ +# up +# +# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws +# and so on, one per home. Each gets a site of its own, named after it, and the +# CEM goes by the simulated time its messages carry (see README.md). +# ------------------------------------------------------------------ + +# Shared definition for every CEM. Each listens on port 8080 inside its own +# container; only the host port differs. +x-cem: &cem + build: + context: . + dockerfile: Dockerfile + image: flexmeasures-client-cem + depends_on: + - server + restart: always + environment: + FLEXMEASURES_BASE_URL: http://server:5000 + FLEXMEASURES_USER: toy-user@flexmeasures.io + FLEXMEASURES_PASSWORD: toy-password + LOGGING_LEVEL: INFO + # A resource manager in a co-simulation wraps each message with the simulated time. + S2_ACCEPT_SIMULATION_METADATA: "1" + # And each home is a site of its own, rather than all of them sharing one. + S2_SITE_PER_RESOURCE_MANAGER: "1" + # The checkout is mounted without its git history, which is where its version would come from. + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLEXMEASURES_CLIENT: "0.0.0" + volumes: + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + command: + - | + # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`): + # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv, + # which ships without pip and cannot see what the system pip installs. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + # s2-python as this repo locks it, and as the co-simulation's resource managers use it. + python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py + +# Shared definition for every RQ worker: one per home simulated at once, +# like the CEMs and the server's gunicorn workers below, since every home's +# scheduling jobs otherwise queue behind one worker process. +x-worker: &worker + # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with. + build: + context: . + dockerfile: Dockerfile + depends_on: + - dev-db + - queue-db + - mailhog + restart: on-failure + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" + FLEXMEASURES_REDIS_URL: queue-db + FLEXMEASURES_REDIS_PASSWORD: fm-redis-pass + SECRET_KEY: notsecret + SECURITY_TOTP_SECRETS: '{"1": "something-secret"}' + FLEXMEASURES_ENV: development + MAIL_SERVER: mailhog + MAIL_PORT: 1025 + LOGGING_LEVEL: INFO + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Drop any lingering registration for the ingestion queue before this worker starts. + # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, + # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. + # No worker here serves that queue, so removing those entries on every start is safe, and idempotent. + python3 - <<'PYGUARD' + import os, redis + r = redis.Redis(host=os.environ.get("FLEXMEASURES_REDIS_URL", "queue-db"), + port=int(os.environ.get("FLEXMEASURES_REDIS_PORT", 6379)), + password=os.environ.get("FLEXMEASURES_REDIS_PASSWORD")) + try: + for name in r.smembers("rq:workers"): + key = name.decode() + queues = (r.hget(key, "queues") or b"").decode() + if "ingestion" in queues or not queues: + r.srem("rq:workers", name); r.delete(key) + r.delete("rq:workers:ingestion") + except redis.RedisError as exc: + print(f"ingestion-registration guard skipped: {exc}") + PYGUARD + # Named after the container, which is unique per worker. + flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting + +services: + dev-db: + ports: + - "5433:5432" + queue-db: + ports: + - "6380:6379" + server: + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install --break-system-packages -e /app/flexmeasures-client[s2] + flexmeasures db upgrade + if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then + flexmeasures add toy-account --name 'Docker Toy Account' + fi + # Worker processes rather than threads: a slow, CPU-bound request held its worker's GIL, + # and under several threads that starved the other requests into client-side timeouts. + # One worker per CEM below, since every home's CEM can be mid-request at once; + # scale the two together. + gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 5 --threads 1 --timeout 120 wsgi:application + # One worker per home simulated at once; add or remove them along with the CEMs. + worker: + <<: *worker + worker-2: + <<: *worker + worker-3: + <<: *worker + # One CEM per simulated home; add one by copying an entry and taking the next host port. + cem: + <<: *cem + ports: + - "8080:8080" + cem-2: + <<: *cem + ports: + - "8081:8080" + cem-3: + <<: *cem + ports: + - "8082:8080" + cem-4: + <<: *cem + ports: + - "8083:8080" + cem-5: + <<: *cem + ports: + - "8084:8080" diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index e1adf215..4b6749ce 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import logging import math @@ -7,7 +8,7 @@ from collections import defaultdict from datetime import datetime, timedelta from logging import Logger -from typing import Dict, Optional +from typing import Awaitable, Callable, Dict, Optional, Protocol import pandas as pd import pydantic @@ -44,6 +45,23 @@ _LOGGER = logging.getLogger(__name__) + +class ConfiguredHandler(Protocol): + """What a site configurator gives back: the handler for a resource manager, and its measured-power sensors.""" + + @property + def handler(self) -> ControlTypeHandler: ... + + @property + def power_sensor_ids(self) -> dict[str, int]: ... + + @property + def asset_id(self) -> int: ... + + +#: Gives a resource manager a site and a handler of its own, from its details (see flexmeasures_client.s2.site). +SiteConfigurator = Callable[[ResourceManagerDetails], Awaitable[ConfiguredHandler]] + #: The control type each family of control-type messages belongs to, by the prefix of its message type. CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, @@ -87,6 +105,7 @@ def __init__( timers: dict[str, datetime] | None = None, datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, + site_configurator: SiteConfigurator | None = None, clock: Clock | None = None, accept_simulation_metadata: bool = False, **kwargs, @@ -94,6 +113,10 @@ def __init__( """ Customer Energy Manager (CEM) + :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; + called with its details, in the background, while its messages are answered + with TEMPORARY_ERROR until the handler is registered (see FRBCSiteConfigurator). + Without one, the handlers registered up front serve every resource manager. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -120,6 +143,8 @@ def __init__( self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() + self._site_configurator = site_configurator + self._configuring: asyncio.Task | None = None #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type @@ -173,6 +198,8 @@ def supports_control_type(self, control_type: ControlType): async def close(self): self._is_closed = True + if self._configuring is not None: + self._configuring.cancel() for control_type, handler in self._control_types_handlers.items(): self._logger.debug(f"Closing handler for {control_type}") @@ -242,15 +269,17 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): self._activate_implicitly(message) # try to handle the message with the control_type handle + # A control type without a handler yet, as while a resource manager's site is being configured, + # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. + handler = self._control_types_handlers.get(self._control_type) if ( self._control_type is not None and ( self._control_type not in [ControlType.NO_SELECTION, ControlType.NOT_CONTROLABLE] ) - and self._control_types_handlers[self._control_type].supports_message( - message - ) + and handler is not None + and handler.supports_message(message) ): response = await self._control_types_handlers[ self._control_type @@ -431,6 +460,10 @@ async def handle_handshake(self, message: Handshake): @register(ResourceManagerDetails) async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message + if self._site_configurator is not None and self._configuring is None: + # In the background, since a site takes a few requests to FlexMeasures to set up, + # and the resource manager waits for this message's reception status meanwhile. + self._configuring = asyncio.create_task(self._configure_site(message)) if ( not self._control_type @@ -444,6 +477,25 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) + async def _configure_site(self, details: ResourceManagerDetails) -> None: + """Give the resource manager its site, and register the handler wired to it.""" + assert self._site_configurator is not None + try: + site = await self._site_configurator(details) + except (ValueError, ConnectionError) as exc: + # Its messages go on being answered with TEMPORARY_ERROR, which the resource manager sees and retries. + self._logger.error( + "Could not configure a site for resource manager %r: %s", + details.name, + exc, + ) + return + self.power_sensor_id = {**(self.power_sensor_id or {}), **site.power_sensor_ids} + self.register_control_type(site.handler) + self._logger.info( + "Resource manager %r is served on site %s.", details.name, site.asset_id + ) + @register(PowerMeasurement) async def handle_power_measurement(self, message: PowerMeasurement): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 21cdd974..3b8636eb 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -19,6 +19,11 @@ ) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + ensure_site, +) log_level = os.getenv("LOGGING_LEVEL", "WARNING").upper() logging.basicConfig( @@ -41,28 +46,27 @@ "yes", ) -#: Each sensor the site needs: its unit, its resolution, and any attributes. -#: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. -SITE_SENSORS: dict[str, tuple[str, str, dict]] = { - "price": ("EUR/kWh", "PT15M", {}), - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. - "power": ("kW", "PT15M", {"consumption_is_positive": True}), - "state of charge": ("kWh", "PT0M", {}), - "RM discharge": ("dimensionless", "PT15M", {}), - "fill rate": ("kWh/s", "PT15M", {}), - "soc minima": ("kWh", "PT15M", {}), - "soc maxima": ("kWh", "PT15M", {}), - "usage forecast": ("kWh/s", "PT15M", {}), - "storage efficiency": ("%", "PT15M", {}), - "charging efficiency": ("dimensionless", "PT15M", {}), -} +#: Each sensor the site needs, which is the library's table of what an FRBC site records. +SITE_SENSORS = FRBC_SITE_SENSORS + +#: What every site asset is made with: a building, whose connection takes three phases of 25 A at 230 V. +SITE_ASSET_SPECS: dict[str, Any] = dict( + generic_asset_type_id=6, + flex_model={"power-capacity": f"{3 * 25 * 230} VA"}, +) + +#: Whether each resource manager gets a site of its own, named after it, as in a co-simulation of a community, +#: rather than all of them sharing the one site this server sets up at startup. +SITE_PER_RESOURCE_MANAGER = os.getenv("S2_SITE_PER_RESOURCE_MANAGER", "").lower() in ( + "1", + "true", + "yes", +) # Keys under which the configured site is kept on the aiohttp application. FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) SENSORS = web.AppKey("sensors", dict) +CONFIGURATOR = web.AppKey("configurator", FRBCSiteConfigurator) async def rm_details_watchdog(ws, cem: CEM): @@ -136,21 +140,40 @@ async def websocket_handler(request): # calls, which takes about ten seconds against a local server, while s2-python's # reception-status timeout is five: a resource manager gave up before the CEM had read its # first message. - await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + if SITE_PER_RESOURCE_MANAGER: + await serve_resource_manager( + ws, request.app[FM_CLIENT], configurator=request.app[CONFIGURATOR] + ) + else: + await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) return ws async def serve_resource_manager( - ws, fm_client: FlexMeasuresClient, sensors: dict[str, dict] + ws, + fm_client: FlexMeasuresClient, + sensors: dict[str, dict] | None = None, + configurator: FRBCSiteConfigurator | None = None, ) -> None: - """Act as the CEM for one resource manager connected over ws, until it disconnects.""" + """Act as the CEM for one resource manager connected over ws, until it disconnects. + + On the site whose sensors are given, or, with a configurator, on a site of the resource manager's own, + which is set up once its details arrive. + """ cem = CEM( - sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, + site_configurator=configurator, # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) + if sensors is not None: + cem.register_control_type(fixed_site_handler(sensors)) + await serve(ws, cem) + + +def fixed_site_handler(sensors: dict[str, dict]) -> FRBCSimple: + """The handler for a resource manager on the one site this server sets up at startup.""" frbc = FRBCSimple( power_sensor_id=sensors["power"]["id"], price_sensor_id=sensors["price"]["id"], @@ -169,8 +192,11 @@ async def serve_resource_manager( ) # A newer plan revokes what is left of the one before, so the resource manager does not follow both. frbc.send_revocations = True - cem.register_control_type(frbc) + return frbc + +async def serve(ws, cem: CEM) -> None: + """Serve one resource manager with this CEM, for as long as its connection lasts.""" # The session lasts as long as the resource manager's messages do. # When they end, the CEM is closed, which stops what its handlers still have in flight, # and the producer and the watchdog stop with it: left running, they outlived the connection, @@ -192,87 +218,18 @@ async def serve_resource_manager( async def configure_site( site_name: str, fm_client: FlexMeasuresClient ) -> dict[str, dict]: - """Create or update the site asset and its sensors, and seed it with demo prices.""" - account = await fm_client.get_account() - if account is None: - raise ValueError( - "The demo user belongs to no organisation, so there is nowhere to create the site." - ) - assets = await fm_client.get_assets(parse_json_fields=True) - - site_asset = None - for asset in assets: - if asset["name"] == site_name: - site_asset = asset - break - - site_asset_specs: dict[str, Any] = dict( - latitude=0, - longitude=0, - generic_asset_type_id=6, # Building asset type - flex_model={ - "power-capacity": f"{3 * 25 * 230} VA", - }, - ) - - if not site_asset: - site_asset = await fm_client.add_asset( - name=site_name, account_id=account["id"], **site_asset_specs - ) - # Update site asset with the latest specs - await fm_client.update_asset( - site_asset["id"], site_asset_specs, parse_json_fields=True - ) - - existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} - sensors: dict[str, dict] = {} - for name, (unit, resolution, attributes) in SITE_SENSORS.items(): - sensors[name] = existing.get(name) or await fm_client.add_sensor( - name=name, - event_resolution=resolution, - unit=unit, - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - **({"attributes": attributes} if attributes else {}), - ) - - # The site's graphs page shows what the plan is about, not only its price and power: - # the buffer with the comfort band around it, what is drawn from it, and what the heat pump puts in. - await fm_client.update_asset( - site_asset["id"], - {"sensors_to_show": site_graphs(sensors)}, - parse_json_fields=True, + """Create or update the site asset, its sensors and its graphs, and seed it with demo prices.""" + _, sensors = await ensure_site( + fm_client, + site_name, + asset_specs=SITE_ASSET_SPECS, + sensors=SITE_SENSORS, + timezone=SITE_TIMEZONE, ) - await seed_prices(fm_client, sensors["price"]["id"]) return sensors -def site_graphs(sensors: dict[str, dict]) -> list[dict]: - """The site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. - - In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, - rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. - The server sets them on every start, so changes made to them in the UI do not outlive a restart. - """ - - def ids(*names: str) -> list[int]: - return [sensors[name]["id"] for name in names] - - return [ - {"title": "Price", "sensors": ids("price")}, - {"title": "Heat pump: planned and measured power", "sensors": ids("power")}, - { - "title": "Buffer: fill level and comfort band", - "sensors": ids("state of charge", "soc minima", "soc maxima"), - }, - { - "title": "Buffer: filled by the heat pump, drawn by hot-water use", - "sensors": ids("fill rate", "usage forecast"), - }, - ] - - async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: """Demo prices, so that a site has something to schedule against. @@ -299,7 +256,13 @@ async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> No async def keep_prices_seeded(app: web.Application): - """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them.""" + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them. + + Not where each resource manager has a site of its own, whose prices are whoever runs them to post. + """ + if SITE_PER_RESOURCE_MANAGER: + yield + return async def reseed(): while True: @@ -338,6 +301,20 @@ async def configure_site_once(app: web.Application) -> None: ssl=parsed.scheme == "https", ) app[FM_CLIENT] = fm_client + if SITE_PER_RESOURCE_MANAGER: + # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. + app[CONFIGURATOR] = FRBCSiteConfigurator( + fm_client, + asset_type_id=SITE_ASSET_SPECS["generic_asset_type_id"], + asset_specs={"flex_model": SITE_ASSET_SPECS["flex_model"]}, + timezone=SITE_TIMEZONE, + # The co-simulation's resource managers are heat pumps, which cannot feed power back: + # FlexMeasures then keeps their production at zero even where constraints are relaxed. + power_sensor_attributes={"is_strictly_non_positive": True}, + logger=LOGGER, + ) + LOGGER.info("Each resource manager gets a site of its own; ready for them.") + return app[SENSORS] = await configure_site(SITE_NAME, fm_client) LOGGER.info("Site '%s' is configured; ready for resource managers.", SITE_NAME) diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py new file mode 100644 index 00000000..0807c80f --- /dev/null +++ b/src/flexmeasures_client/s2/site.py @@ -0,0 +1,264 @@ +"""A FlexMeasures site for an S2 resource manager: its asset, its sensors, its graphs, and the handler wired to them. + +The example server keeps one site for whichever resource manager connects (see ensure_site). +A CEM serving many resource managers, as a co-simulation of a community does, gives each one a site of its own, +named after it, with FRBCSiteConfigurator. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import timedelta +from logging import Logger, getLogger +from typing import Any + +try: + from s2python.common import ResourceManagerDetails +except ImportError: + raise ImportError( + "The 's2-python' package is required for this functionality. " + "Install it using `pip install flexmeasures-client[s2]`." + ) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + +_LOGGER = getLogger(__name__) + + +@dataclass(frozen=True) +class SiteSensor: + """What a site's sensor is made with: its unit, its resolution, and any attributes.""" + + unit: str + resolution: str + attributes: dict = field(default_factory=dict) + + +#: The sensors an FRBC site needs, by name. +#: Their units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. +#: The names are a contract with whoever reads the site, a co-simulation among them, +#: which looks up "price", "production price", "power", "state of charge" and "measured-power" by name. +FRBC_SITE_SENSORS: dict[str, SiteSensor] = { + "price": SiteSensor("EUR/kWh", "PT15M"), + "production price": SiteSensor("EUR/kWh", "PT15M"), + # S2 states a power as positive when it is consumed, + # and this sensor holds both an actuator's power and the schedule planned for it. + # Left unsaid, FlexMeasures reads a power sensor the other way round, + # and flips a schedule's sign on the way in, which puts the two in opposite directions. + "power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + # What the resource manager measures its site drawing, kept apart from the schedule on "power". + "measured-power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + "state of charge": SiteSensor("kWh", "PT0M"), + "RM discharge": SiteSensor("dimensionless", "PT15M"), + "fill rate": SiteSensor("kWh/s", "PT15M"), + "soc minima": SiteSensor("kWh", "PT15M"), + "soc maxima": SiteSensor("kWh", "PT15M"), + "usage forecast": SiteSensor("kWh/s", "PT15M"), + "storage efficiency": SiteSensor("%", "PT15M"), + "charging efficiency": SiteSensor("dimensionless", "PT15M"), +} + + +def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: + """A site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + + In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, + rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. + A graph whose sensors the site does not have is left out. + """ + graphs = [ + ("Prices", ["price", "production price"]), + ("Power: planned and measured", ["power", "measured-power"]), + ( + "Buffer: fill level and comfort band", + ["state of charge", "soc minima", "soc maxima"], + ), + ( + "Buffer: filled by the actuator, drawn by usage", + ["fill rate", "usage forecast"], + ), + ] + shown = [] + for title, names in graphs: + ids = [sensors[name]["id"] for name in names if name in sensors] + if ids: + shown.append({"title": title, "sensors": ids}) + return shown + + +async def ensure_site( + fm_client: FlexMeasuresClient, + name: str, + *, + asset_specs: dict[str, Any], + sensors: dict[str, SiteSensor], + timezone: str, + known_asset_id: int | None = None, +) -> tuple[dict, dict[str, dict]]: + """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. + + It is safe to run again: what exists is kept, and what is missing is made. + + :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. + :returns: the asset, and its sensors by name. + """ + site_asset = None + if known_asset_id is not None: + try: + candidate = await fm_client.get_asset( + known_asset_id, parse_json_fields=True + ) + except (ValueError, ConnectionError): + candidate = None + if candidate is not None and candidate.get("name") == name: + site_asset = candidate + if site_asset is None: + for asset in await fm_client.get_assets(parse_json_fields=True): + if asset["name"] == name: + site_asset = asset + break + if site_asset is None: + account = await fm_client.get_account() + if account is None: + raise ValueError( + f"The user belongs to no organisation, so there is nowhere to create the site {name!r}." + ) + site_asset = await fm_client.add_asset( + name=name, account_id=account["id"], **asset_specs + ) + else: + await fm_client.update_asset( + site_asset["id"], asset_specs, parse_json_fields=True + ) + + existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} + made: dict[str, dict] = {} + for sensor_name, spec in sensors.items(): + made[sensor_name] = existing.get(sensor_name) or await fm_client.add_sensor( + name=sensor_name, + event_resolution=spec.resolution, + unit=spec.unit, + generic_asset_id=site_asset["id"], + timezone=timezone, + **({"attributes": spec.attributes} if spec.attributes else {}), + ) + + # Set on every run, so changes made to the graphs in the UI do not outlive a restart. + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": frbc_site_graphs(made)}, + parse_json_fields=True, + ) + return site_asset, made + + +@dataclass +class ConfiguredSite: + """What a CEM needs to serve one resource manager: its handler, and where its measured power goes.""" + + handler: FRBCSimple + #: The sensor each commodity quantity's measured power is recorded on, as the CEM's power_sensor_id takes it. + power_sensor_ids: dict[str, int] + asset_id: int + + +class FRBCSiteConfigurator: + """Gives each resource manager a site of its own, named after it, and an FRBC handler wired to its sensors. + + Made once per server and handed to each CEM it makes (see CEM's site_configurator), + so that a resource manager that reconnects finds its site by the id remembered here, + rather than by a listing of every asset on the server. + """ + + def __init__( + self, + fm_client: FlexMeasuresClient, + *, + asset_type_id: int, + timezone: str = "UTC", + asset_specs: dict[str, Any] | None = None, + sensors: dict[str, SiteSensor] | None = None, + power_sensor_attributes: dict[str, Any] | None = None, + handler_options: dict[str, Any] | None = None, + send_revocations: bool = True, + logger: Logger | None = None, + ) -> None: + """ + :param asset_type_id: the FlexMeasures asset type each site is made with. + :param timezone: the time zone of each site's sensors, and of its handler. + :param asset_specs: anything else each site asset is made or updated with, such as a flex model. + :param sensors: the sensors each site gets; FRBC_SITE_SENSORS by default. + :param power_sensor_attributes: attributes added to the "power" sensor's, which a device's nature decides, + such as that it cannot produce. + :param handler_options: keyword arguments for each FRBCSimple beyond its sensors. + By default it plans from now, over 24 hours, as a co-simulation steps through days. + :param send_revocations: whether a newer plan revokes what is left of the one before. + """ + self._fm_client = fm_client + self._timezone = timezone + self._asset_specs = { + "generic_asset_type_id": asset_type_id, + **(asset_specs or {}), + } + sensors = dict(sensors if sensors is not None else FRBC_SITE_SENSORS) + if power_sensor_attributes: + power = sensors["power"] + sensors["power"] = SiteSensor( + power.unit, + power.resolution, + {**power.attributes, **power_sensor_attributes}, + ) + self._sensors = sensors + self._handler_options = { + "valid_from_shift": timedelta(0), + "schedule_duration": timedelta(hours=24), + **(handler_options or {}), + } + self._send_revocations = send_revocations + self._asset_ids: dict[str, int] = {} + self._logger = logger or _LOGGER + + async def __call__(self, details: ResourceManagerDetails) -> ConfiguredSite: + name = details.name + if not name: + raise ValueError( + "A resource manager without a name cannot be given a site of its own, which is found by its name." + ) + asset, sensors = await ensure_site( + self._fm_client, + name, + asset_specs=self._asset_specs, + sensors=self._sensors, + timezone=self._timezone, + known_asset_id=self._asset_ids.get(name), + ) + self._asset_ids[name] = asset["id"] + self._logger.info("Resource manager %r has site %s.", name, asset["id"]) + + def sensor_id(sensor_name: str) -> int | None: + sensor = sensors.get(sensor_name) + return sensor["id"] if sensor else None + + handler = FRBCSimple( + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + production_price_sensor_id=sensor_id("production price"), + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + fill_rate_sensor_id=sensor_id("fill rate"), + soc_minima_sensor_id=sensor_id("soc minima"), + soc_maxima_sensor_id=sensor_id("soc maxima"), + usage_forecast_sensor_id=sensor_id("usage forecast"), + leakage_behaviour_sensor_id=sensor_id("storage efficiency"), + charging_efficiency_sensor_id=sensor_id("charging efficiency"), + timezone=self._timezone, + **self._handler_options, + ) + handler.send_revocations = self._send_revocations + measured = sensor_id("measured-power") + return ConfiguredSite( + handler=handler, + power_sensor_ids={"ELECTRIC.POWER.L1": measured} if measured else {}, + asset_id=asset["id"], + ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index e2eca6c4..3678e18b 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -534,9 +534,12 @@ def capture(**kwargs): def test_the_site_sensors_are_what_the_cem_records_in(): - assert server.SITE_SENSORS["state of charge"][:2] == ("kWh", "PT0M") - assert server.SITE_SENSORS["storage efficiency"][0] == "%" - assert server.SITE_SENSORS["usage forecast"][0] == "kWh/s" + assert ( + server.SITE_SENSORS["state of charge"].unit, + server.SITE_SENSORS["state of charge"].resolution, + ) == ("kWh", "PT0M") + assert server.SITE_SENSORS["storage efficiency"].unit == "%" + assert server.SITE_SENSORS["usage forecast"].unit == "kWh/s" @pytest.mark.asyncio @@ -752,12 +755,15 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} assert {"state of charge", "soc minima", "soc maxima"} in named.values() assert {"fill rate", "usage forecast"} in named.values() - assert {"power"} in named.values() and {"price"} in named.values() + assert {"power", "measured-power"} in named.values() + assert {"price", "production price"} in named.values() def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): """A title and a list of sensors per graph; the form with plots is refused before FlexMeasures v0.32.""" - graphs = server.site_graphs( + from flexmeasures_client.s2.site import frbc_site_graphs + + graphs = frbc_site_graphs( {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} ) for graph in graphs: @@ -823,3 +829,70 @@ def capture(**kwargs): await cem_end._inbox.put(_CLOSED) await asyncio.wait_for(serving, timeout=5) assert made[0]["accept_simulation_metadata"] is accept + + +@pytest.mark.asyncio +async def test_with_a_site_per_resource_manager_the_server_configures_none_at_startup(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "configure_site", AsyncMock()) as configure, + ): + await server.configure_site_once(app) + configure.assert_not_awaited() + assert isinstance(app[server.CONFIGURATOR], server.FRBCSiteConfigurator) + assert ( + app[server.CONFIGURATOR] + ._sensors["power"] + .attributes["is_strictly_non_positive"] + ) + app[server.SENSORS] = { + "price": {"id": 1} + } # so that seeding, were it to happen, would reach FlexMeasures + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)), + patch.object(server, "seed_prices", AsyncMock()) as seed, + ): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() # nothing to seed: the prices are the simulation's to post + await asyncio.sleep(0.05) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + seed.assert_not_awaited() + await app[server.FM_CLIENT].close() + + +@pytest.mark.asyncio +async def test_a_resource_manager_with_a_site_of_its_own_is_served_once_it_is_configured(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + async def configurator(details): + raise AssertionError("not called in this test") + + resource_manager, cem_end = _connected() + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), configurator=configurator + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]._site_configurator is configurator + assert ( + made[0]._control_types_handlers == {} + ) # no handler until its site is configured diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py new file mode 100644 index 00000000..54e81673 --- /dev/null +++ b/tests/s2/test_site.py @@ -0,0 +1,264 @@ +"""A site of its own for each resource manager, and a CEM that waits for it.""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from unittest.mock import AsyncMock + +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + SiteSensor, + ensure_site, + frbc_site_graphs, +) + + +def _flexmeasures(existing: list[dict] | None = None) -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock(return_value=existing or []) + fm.add_asset = AsyncMock(side_effect=lambda **kw: {"id": 50, "sensors": [], **kw}) + fm.made = {} + + def add_sensor(**kw): + sensor = {"id": 100 + len(fm.made), **kw} + fm.made[kw["name"]] = sensor + return sensor + + fm.add_sensor = AsyncMock(side_effect=add_sensor) + return fm + + +@pytest.fixture +def named_details(resource_manager_details): + return resource_manager_details.model_copy(update={"name": "home A"}) + + +@pytest.mark.asyncio +async def test_a_site_is_made_with_every_sensor_and_its_graphs(): + fm = _flexmeasures() + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 50 + assert set(sensors) == set(FRBC_SITE_SENSORS) + made = {c.kwargs["name"]: c.kwargs for c in fm.add_sensor.await_args_list} + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["state of charge"]["event_resolution"] == "PT0M" + graphs = fm.update_asset.await_args.args[1]["sensors_to_show"] + assert graphs == frbc_site_graphs(sensors) + + +@pytest.mark.asyncio +async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): + fm = _flexmeasures( + [ + {"id": 6, "name": "home B", "sensors": []}, + {"id": 7, "name": "home A", "sensors": [{"id": 70, "name": "price"}]}, + ] + ) + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 7 + assert sensors["price"]["id"] == 70 + assert "price" not in {c.kwargs["name"] for c in fm.add_sensor.await_args_list} + fm.add_asset.assert_not_awaited() + specs_update = fm.update_asset.await_args_list[0] + assert specs_update.args == (7, {"generic_asset_type_id": 6}) + + +@pytest.mark.asyncio +async def test_a_known_site_is_fetched_by_id_rather_than_found_in_a_listing(): + """A resource manager reconnects every round of a co-simulation; a listing of every asset each time is slow.""" + fm = _flexmeasures() + fm.get_asset = AsyncMock(return_value={"id": 7, "name": "home A", "sensors": []}) + await ensure_site( + fm, + "home A", + asset_specs={}, + sensors={"price": SiteSensor("EUR/kWh", "PT15M")}, + timezone="UTC", + known_asset_id=7, + ) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_known_id_that_now_names_another_asset_is_looked_up_again(): + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock( + return_value={"id": 7, "name": "someone else", "sensors": []} + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_site_needs_an_organisation_to_be_made_in(): + fm = _flexmeasures() + fm.get_account = AsyncMock(return_value=None) + with pytest.raises(ValueError, match="no organisation"): + await ensure_site(fm, "home A", asset_specs={}, sensors={}, timezone="UTC") + + +def test_a_graph_without_sensors_on_the_site_is_left_out(): + graphs = frbc_site_graphs({"price": {"id": 1}, "power": {"id": 2}}) + assert graphs == [ + {"title": "Prices", "sensors": [1]}, + {"title": "Power: planned and measured", "sensors": [2]}, + ] + + +@pytest.mark.asyncio +async def test_each_resource_manager_gets_a_site_named_after_it_and_a_handler_wired_to_it( + named_details, +): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator( + fm, asset_type_id=6, power_sensor_attributes={"is_strictly_non_positive": True} + ) + site = await configurator(named_details) + assert fm.add_asset.await_args.kwargs["name"] == "home A" + made = fm.made + assert made["power"]["attributes"] == { + "consumption_is_positive": True, + "is_strictly_non_positive": True, + } + handler = site.handler + assert handler._power_sensor_id == made["power"]["id"] + assert handler._price_sensor_id == made["price"]["id"] + assert handler._production_price_sensor_id == made["production price"]["id"] + assert handler._soc_sensor_id == made["state of charge"]["id"] + assert handler._valid_from_shift == timedelta(0) + assert handler._schedule_duration == timedelta(hours=24) + assert handler.send_revocations is True + assert site.power_sensor_ids == {"ELECTRIC.POWER.L1": made["measured-power"]["id"]} + + +@pytest.mark.asyncio +async def test_a_reconnecting_resource_manager_finds_its_site_by_id(named_details): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + await configurator(named_details) + fm.get_asset = AsyncMock( + return_value={"id": 50, "name": "home A", "sensors": list(fm.made.values())} + ) + fm.get_assets.reset_mock() + await configurator(named_details) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_resource_manager_without_a_name_gets_no_site(resource_manager_details): + configurator = FRBCSiteConfigurator(_flexmeasures(), asset_type_id=6) + with pytest.raises(ValueError, match="without a name"): + await configurator(resource_manager_details.model_copy(update={"name": None})) + + +# --- The CEM, while the site is being configured --- + + +async def _details_then_selection(cem, rm_handshake, resource_manager_details): + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + messages = [] + while not cem._sending_queue.empty(): + messages.append(await cem.get_message()) + return [m for m in messages if m["message_type"] == "SelectControlType"][0] + + +@pytest.mark.asyncio +async def test_messages_are_answered_with_a_temporary_error_until_the_site_is_configured( + rm_handshake, named_details, frbc_system_description +): + """The resource manager retries a temporary error, so it is served once its handler exists.""" + release = asyncio.Event() + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + + async def slow(details): + await release.wait() + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=slow) + selection = await _details_then_selection(cem, rm_handshake, named_details) + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], status=ReceptionStatusValues.OK + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + + await cem.handle_message(frbc_system_description) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + + release.set() + await cem._configuring + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + assert cem.power_sensor_id and "ELECTRIC.POWER.L1" in cem.power_sensor_id + await cem.handle_message(frbc_system_description) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_site_that_cannot_be_configured_leaves_the_resource_manager_retrying( + rm_handshake, resource_manager_details, caplog +): + async def failing(details): + raise ConnectionError("FlexMeasures is down") + + cem = CEM(fm_client=_flexmeasures(), site_configurator=failing) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + await cem._configuring + assert cem._control_types_handlers == {} + assert "Could not configure a site" in caplog.text + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_a_site_being_configured( + rm_handshake, resource_manager_details +): + async def forever(details): + await asyncio.sleep(3600) + + cem = CEM(fm_client=_flexmeasures(), site_configurator=forever) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + configuring = cem._configuring + await cem.close() + await asyncio.wait([configuring], timeout=1) + assert configuring.cancelled() + + +@pytest.mark.asyncio +async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): + """Deleted since, say: the listing finds it by name, or it is made again.""" + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock(side_effect=ValueError("404")) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 From bffe6c9df24dadb1f5ac886288ccb46806e9f5c9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:45:12 +0100 Subject: [PATCH 46/87] docs(changelog): sites per resource manager and the co-simulation stack Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 30f19ad3..8078ed8d 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -27,6 +27,9 @@ Added * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] * ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] +* ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] +* ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] Changed ------- From 65b359f5167d6fb443594f074937916e8303e067 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:02:38 +0100 Subject: [PATCH 47/87] fix(S2): what the review of the sites per resource manager found Against the co-simulation it is for: - A reconnect overwrote the apartment's flex model with the configurator's, which reset what the co-simulation's controller writes into it (the stock's value at the end), and replaced its graphs, dropping the charts the co-simulation adds. A flex model is now merged into the asset's, and graphs are set only where an asset has none (the demo's own fixed site resets them, as before). They carry #188's titles again, since the controller adds to the graph titled "Power". - #188 always relaxed the site capacity; the configured sites in the example server do so again, explicitly. - Of several assets with a resource manager's name, the newest is taken, as the co-simulation takes it. In the CEM: - A power measurement arriving before a resource manager's site was set up was recorded on sensor 357, a fallback from #115 that named whichever sensor of whichever site had that id, and answered OK, so it was never sent again. It is now answered with TEMPORARY_ERROR while the site is being set up, and without a sensor for it, it is not recorded at all. The fixed site's measured-power sensor is now wired too. - A setup that failed was not tried again, and only two exception types were logged; anything else surfaced as a task exception never retrieved. A failed setup is now logged, whatever failed, and tried again on the next message that needs it. In the co-simulation stack: - The overlay's workers serve the reporting queue too, as the base file's do. - Its README says what isolates the database: the overlay mounts ../flexmeasures and ../flexmeasures-client, so a folder of its own with checkouts of those names, not a FlexMeasures worktree on its own. Mailhog's ports are listed. - Comments break after punctuation. Still not here: recording a co-simulation's measured power, which arrives in bursts stamped at past quarter hours and which the CEM's binning on now() does not catch. That is the next package. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + examples/S2-co-simulation/README.md | 15 +- .../S2-co-simulation/docker-compose.cem.yml | 32 ++-- src/flexmeasures_client/s2/cem.py | 65 ++++--- .../s2/script/websockets_server.py | 10 ++ src/flexmeasures_client/s2/site.py | 51 +++--- tests/s2/test_example_end_to_end.py | 40 ++++- tests/s2/test_site.py | 158 ++++++++++++++++-- 8 files changed, 293 insertions(+), 79 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 8078ed8d..c8c5187d 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -56,6 +56,7 @@ Removed Fixed ----- +* A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index fa95fa8e..9e1b8e0f 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -22,9 +22,12 @@ A co-simulation reads the sites by sensor name: `price`, `production price`, `power` (the schedule), `measured-power` (what the home measured) and `state of charge`. -The stack uses FlexMeasures' own development database service, `dev-db`, whose -data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the -flexmeasures checkout. A compose project name of its own does not separate that, -so run the stack from a checkout of its own (a git worktree will do) to keep a -development database you care about out of it. Its host ports, 5000, 5433, 6380 -and 8080 to 8084, have to be free too. +The stack uses FlexMeasures' own development database service, `dev-db`, +whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` +inside the flexmeasures checkout it runs from. +A compose project name of its own does not separate that. +To keep a development database you care about out of it, +run it from a folder of its own holding checkouts (git worktrees will do) +named `flexmeasures` and `flexmeasures-client`: +the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. +Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index da7f58dd..79a5b364 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -1,24 +1,25 @@ # ------------------------------------------------------------------ -# A FlexMeasures stack with one S2 CEM per simulated home, for a co-simulation -# whose resource managers connect over websockets, such as the community -# simulations that steer several homes at once. +# A FlexMeasures stack with one S2 CEM per simulated home, +# for a co-simulation whose resource managers connect over websockets, +# such as the community simulations that steer several homes at once. # -# It is an overlay on FlexMeasures' own docker-compose.yml, and runs the CEMs -# from this checkout. With the flexmeasures and flexmeasures-client repos side -# by side, run it from the flexmeasures folder (which holds the Dockerfile): +# It is an overlay on FlexMeasures' own docker-compose.yml, +# and runs the CEMs from this checkout. +# With checkouts named flexmeasures and flexmeasures-client side by side, +# run it from the flexmeasures folder, which holds the Dockerfile: # # docker compose \ # -f docker-compose.yml \ # -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \ # up # -# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws -# and so on, one per home. Each gets a site of its own, named after it, and the -# CEM goes by the simulated time its messages carry (see README.md). +# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws and so on, one per home. +# Each gets a site of its own, named after it, +# and the CEM goes by the simulated time its messages carry (see README.md). # ------------------------------------------------------------------ -# Shared definition for every CEM. Each listens on port 8080 inside its own -# container; only the host port differs. +# Shared definition for every CEM. +# Each listens on port 8080 inside its own container; only the host port differs. x-cem: &cem build: context: . @@ -45,15 +46,15 @@ x-cem: &cem - | # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`): # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv, - # which ships without pip and cannot see what the system pip installs. + # which may come without pip, and cannot see what the system pip installs. python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade # s2-python as this repo locks it, and as the co-simulation's resource managers use it. python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py # Shared definition for every RQ worker: one per home simulated at once, -# like the CEMs and the server's gunicorn workers below, since every home's -# scheduling jobs otherwise queue behind one worker process. +# like the CEMs and the server's gunicorn workers below, +# since every home's scheduling jobs otherwise queue behind one worker process. x-worker: &worker # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with. build: @@ -105,7 +106,8 @@ x-worker: &worker print(f"ingestion-registration guard skipped: {exc}") PYGUARD # Named after the container, which is unique per worker. - flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting + # Every queue the base file's worker serves but ingestion, which is left to the server to do as it receives the data. + flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting\|reporting services: dev-db: diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 4b6749ce..679b3e6a 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -114,8 +114,8 @@ def __init__( Customer Energy Manager (CEM) :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; - called with its details, in the background, while its messages are answered - with TEMPORARY_ERROR until the handler is registered (see FRBCSiteConfigurator). + called with its details, in the background (see FRBCSiteConfigurator). + Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. @@ -272,6 +272,8 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): # A control type without a handler yet, as while a resource manager's site is being configured, # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. handler = self._control_types_handlers.get(self._control_type) + if handler is None and self._control_type is not None: + self._ensure_configuring() if ( self._control_type is not None and ( @@ -460,10 +462,7 @@ async def handle_handshake(self, message: Handshake): @register(ResourceManagerDetails) async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message - if self._site_configurator is not None and self._configuring is None: - # In the background, since a site takes a few requests to FlexMeasures to set up, - # and the resource manager waits for this message's reception status meanwhile. - self._configuring = asyncio.create_task(self._configure_site(message)) + self._ensure_configuring() if ( not self._control_type @@ -477,19 +476,39 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) + def _ensure_configuring(self) -> None: + """Set the resource manager's site up in the background, unless that is done or under way. + + In the background, since a site takes a few requests to FlexMeasures to set up, + and the resource manager waits for a reception status meanwhile. + A setup that failed is tried again on the next message that needs it, + while the resource manager's messages are answered with TEMPORARY_ERROR, which it retries. + """ + if ( + self._site_configurator is None + or self._control_types_handlers + or (self._configuring is not None and not self._configuring.done()) + or not hasattr(self, "_resource_manager_details") + ): + return + self._configuring = asyncio.create_task( + self._configure_site(self._resource_manager_details) + ) + self._configuring.add_done_callback(self._report_configuring) + + def _report_configuring(self, task: asyncio.Task) -> None: + if task.cancelled() or task.exception() is None: + return + self._logger.error( + "Could not set a site up for resource manager %r; trying again on its next message.", + self._resource_manager_details.name, + exc_info=task.exception(), + ) + async def _configure_site(self, details: ResourceManagerDetails) -> None: """Give the resource manager its site, and register the handler wired to it.""" assert self._site_configurator is not None - try: - site = await self._site_configurator(details) - except (ValueError, ConnectionError) as exc: - # Its messages go on being answered with TEMPORARY_ERROR, which the resource manager sees and retries. - self._logger.error( - "Could not configure a site for resource manager %r: %s", - details.name, - exc, - ) - return + site = await self._site_configurator(details) self.power_sensor_id = {**(self.power_sensor_id or {}), **site.power_sensor_ids} self.register_control_type(site.handler) self._logger.info( @@ -499,15 +518,17 @@ async def _configure_site(self, details: ResourceManagerDetails) -> None: @register(PowerMeasurement) async def handle_power_measurement(self, message: PowerMeasurement): + if self._site_configurator is not None and not self._control_types_handlers: + # Its site is still being set up, so there is nowhere to record it yet: the resource manager is asked to resend it. + self._ensure_configuring() + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + for power_measurement in message.values: commodity_quantity = power_measurement.commodity_quantity.value - if ( - self.power_sensor_id is None - and commodity_quantity == "ELECTRIC.POWER.L1" - ): - sensor_id = 357 - elif self.power_sensor_id: + # Without a sensor for it, a measurement is not recorded. + # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. + if self.power_sensor_id: s_id = self.power_sensor_id.get(commodity_quantity) if s_id is None: # TODO: create a new sensor or return ReceptionStatus diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 3b8636eb..0a7deeed 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -164,6 +164,12 @@ async def serve_resource_manager( fm_client=fm_client, logger=LOGGER, site_configurator=configurator, + # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. + power_sensor_id=( + {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} + if sensors is not None and "measured-power" in sensors + else None + ), # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) @@ -225,6 +231,8 @@ async def configure_site( asset_specs=SITE_ASSET_SPECS, sensors=SITE_SENSORS, timezone=SITE_TIMEZONE, + # The demo's own graphs, on every start, so changes made to them in the UI do not outlive a restart. + reset_graphs=True, ) await seed_prices(fm_client, sensors["price"]["id"]) return sensors @@ -311,6 +319,8 @@ async def configure_site_once(app: web.Application) -> None: # The co-simulation's resource managers are heat pumps, which cannot feed power back: # FlexMeasures then keeps their production at zero even where constraints are relaxed. power_sensor_attributes={"is_strictly_non_positive": True}, + # A community's capacity is a target set by its members' steering, breached at a price, as #188 asked for. + handler_options={"relax_site_capacity": True}, logger=LOGGER, ) LOGGER.info("Each resource manager gets a site of its own; ready for them.") diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py index 0807c80f..f94185f0 100644 --- a/src/flexmeasures_client/s2/site.py +++ b/src/flexmeasures_client/s2/site.py @@ -67,17 +67,12 @@ def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. A graph whose sensors the site does not have is left out. """ + # Titled as #188 titled them, since a co-simulation's controller adds to the graph titled "Power". graphs = [ ("Prices", ["price", "production price"]), - ("Power: planned and measured", ["power", "measured-power"]), - ( - "Buffer: fill level and comfort band", - ["state of charge", "soc minima", "soc maxima"], - ), - ( - "Buffer: filled by the actuator, drawn by usage", - ["fill rate", "usage forecast"], - ), + ("Power", ["power", "measured-power"]), + ("State of charge", ["state of charge", "soc minima", "soc maxima"]), + ("Fill rate and usage", ["fill rate", "usage forecast"]), ] shown = [] for title, names in graphs: @@ -95,12 +90,17 @@ async def ensure_site( sensors: dict[str, SiteSensor], timezone: str, known_asset_id: int | None = None, + reset_graphs: bool = False, ) -> tuple[dict, dict[str, dict]]: """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. It is safe to run again: what exists is kept, and what is missing is made. + A flex model in the specs is merged into the one the asset has, since others write to it too, + such as a co-simulation's controller setting what its stock is worth at the end. + Graphs are set where the asset has none, since others add to them too, unless asked to reset them. :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. + :param reset_graphs: whether to set the graphs even where the asset has some. :returns: the asset, and its sensors by name. """ site_asset = None @@ -114,10 +114,14 @@ async def ensure_site( if candidate is not None and candidate.get("name") == name: site_asset = candidate if site_asset is None: - for asset in await fm_client.get_assets(parse_json_fields=True): - if asset["name"] == name: - site_asset = asset - break + # The newest of any of that name, as a co-simulation picks it too. + named = [ + asset + for asset in await fm_client.get_assets(parse_json_fields=True) + if asset["name"] == name + ] + site_asset = max(named, key=lambda asset: asset["id"]) if named else None + created = site_asset is None if site_asset is None: account = await fm_client.get_account() if account is None: @@ -128,9 +132,12 @@ async def ensure_site( name=name, account_id=account["id"], **asset_specs ) else: - await fm_client.update_asset( - site_asset["id"], asset_specs, parse_json_fields=True - ) + specs = dict(asset_specs) + if isinstance(specs.get("flex_model"), dict) and isinstance( + site_asset.get("flex_model"), dict + ): + specs["flex_model"] = {**site_asset["flex_model"], **specs["flex_model"]} + await fm_client.update_asset(site_asset["id"], specs, parse_json_fields=True) existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} made: dict[str, dict] = {} @@ -144,12 +151,12 @@ async def ensure_site( **({"attributes": spec.attributes} if spec.attributes else {}), ) - # Set on every run, so changes made to the graphs in the UI do not outlive a restart. - await fm_client.update_asset( - site_asset["id"], - {"sensors_to_show": frbc_site_graphs(made)}, - parse_json_fields=True, - ) + if created or reset_graphs or not site_asset.get("sensors_to_show"): + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": frbc_site_graphs(made)}, + parse_json_fields=True, + ) return site_asset, made diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 3678e18b..8874af96 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -735,7 +735,14 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ fm = AsyncMock(FlexMeasuresClient) fm.get_account = AsyncMock(return_value={"id": 1}) fm.get_assets = AsyncMock( - return_value=[{"id": 5, "name": server.SITE_NAME, "sensors": []}] + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [], + "sensors_to_show": [{"title": "Edited in the UI", "sensors": [1]}], + } + ] ) fm.add_sensor = AsyncMock( side_effect=lambda **kwargs: { @@ -850,9 +857,9 @@ async def test_with_a_site_per_resource_manager_the_server_configures_none_at_st ._sensors["power"] .attributes["is_strictly_non_positive"] ) - app[server.SENSORS] = { - "price": {"id": 1} - } # so that seeding, were it to happen, would reach FlexMeasures + assert app[server.CONFIGURATOR]._handler_options["relax_site_capacity"] is True + # So that seeding, were it to happen, would reach FlexMeasures. + app[server.SENSORS] = {"price": {"id": 1}} with ( patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)), @@ -896,3 +903,28 @@ async def configurator(details): assert ( made[0]._control_types_handlers == {} ) # no handler until its site is configured + + +@pytest.mark.asyncio +async def test_on_the_fixed_site_measured_power_is_recorded_on_its_sensor(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["power_sensor_id"] == { + "ELECTRIC.POWER.L1": sensors["measured-power"]["id"] + } diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py index 54e81673..c67dbad9 100644 --- a/tests/s2/test_site.py +++ b/tests/s2/test_site.py @@ -64,7 +64,7 @@ async def test_a_site_is_made_with_every_sensor_and_its_graphs(): async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): fm = _flexmeasures( [ - {"id": 6, "name": "home B", "sensors": []}, + {"id": 8, "name": "home B", "sensors": []}, {"id": 7, "name": "home A", "sensors": [{"id": 70, "name": "price"}]}, ] ) @@ -123,7 +123,7 @@ def test_a_graph_without_sensors_on_the_site_is_left_out(): graphs = frbc_site_graphs({"price": {"id": 1}, "power": {"id": 2}}) assert graphs == [ {"title": "Prices", "sensors": [1]}, - {"title": "Power: planned and measured", "sensors": [2]}, + {"title": "Power", "sensors": [2]}, ] @@ -223,18 +223,95 @@ async def slow(details): @pytest.mark.asyncio -async def test_a_site_that_cannot_be_configured_leaves_the_resource_manager_retrying( - rm_handshake, resource_manager_details, caplog +async def test_a_site_that_could_not_be_set_up_is_tried_again_on_the_next_message( + rm_handshake, named_details, frbc_system_description, caplog ): - async def failing(details): - raise ConnectionError("FlexMeasures is down") + """A transient failure, such as a server error under load, would otherwise leave the resource manager retrying forever.""" + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + attempts = [] - cem = CEM(fm_client=_flexmeasures(), site_configurator=failing) + async def flaky(details): + attempts.append(details.name) + if len(attempts) == 1: + raise KeyError("a sensor the table lacks") + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=flaky) await cem.handle_message(rm_handshake) - await cem.handle_message(resource_manager_details) - await cem._configuring + await cem.handle_message(named_details) + await asyncio.wait([cem._configuring]) + assert "trying again on its next message" in caplog.text assert cem._control_types_handlers == {} - assert "Could not configure a site" in caplog.text + + cem._control_type = ControlType.FILL_RATE_BASED_CONTROL + await cem.handle_message(frbc_system_description) + await asyncio.wait([cem._configuring]) + assert attempts == ["home A", "home A"] + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + + +@pytest.mark.asyncio +async def test_a_measurement_before_the_site_is_set_up_is_asked_for_again( + rm_handshake, named_details +): + """It used to be recorded on sensor 357, whichever site that belonged to, and answered OK, so it was never resent.""" + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + release = asyncio.Event() + + async def slow(details): + await release.wait() + + fm = _flexmeasures() + cem = CEM(fm_client=fm, site_configurator=slow) + await cem.handle_message(rm_handshake) + await cem.handle_message(named_details) + while not cem._sending_queue.empty(): + await cem.get_message() + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + fm.post_sensor_data.assert_not_awaited() + release.set() + + +@pytest.mark.asyncio +async def test_without_a_sensor_for_it_a_measurement_is_not_recorded(): + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + fm = _flexmeasures() + cem = CEM(fm_client=fm) + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + fm.post_sensor_data.assert_not_awaited() @pytest.mark.asyncio @@ -262,3 +339,64 @@ async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 ) assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_reconnect_keeps_what_others_wrote_to_the_flex_model_and_the_graphs(): + """A co-simulation's controller writes its end-of-day stock value into the flex model, and adds to the graphs.""" + fm = _flexmeasures( + [ + { + "id": 7, + "name": "home A", + "sensors": [], + "flex_model": { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "1 kVA", + }, + "sensors_to_show": [{"title": "Gross vs net", "sensors": [1]}], + } + ] + ) + await ensure_site( + fm, + "home A", + asset_specs={"flex_model": {"power-capacity": "17250 VA"}}, + sensors={}, + timezone="UTC", + ) + specs = fm.update_asset.await_args_list[0].args[1] + assert specs["flex_model"] == { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "17250 VA", + } + assert all( + "sensors_to_show" not in call.args[1] + for call in fm.update_asset.await_args_list + ) + + +@pytest.mark.asyncio +async def test_the_demo_resets_its_graphs_where_asked(): + fm = _flexmeasures( + [{"id": 7, "name": "home A", "sensors": [], "sensors_to_show": [1]}] + ) + await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", reset_graphs=True + ) + assert "sensors_to_show" in fm.update_asset.await_args.args[1] + + +@pytest.mark.asyncio +async def test_of_several_sites_with_one_name_the_newest_is_taken(): + fm = _flexmeasures( + [ + {"id": 9, "name": "home A", "sensors": []}, + {"id": 12, "name": "home A", "sensors": []}, + {"id": 3, "name": "home A", "sensors": []}, + ] + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC" + ) + assert asset["id"] == 12 From ccab11bfe3275275e05e6cbc286ba0cf092dd608 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:14:40 +0100 Subject: [PATCH 48/87] feat(S2): record measured power by when it was measured, in watts, once per step Carved from #188, rewritten for the library rather than transcribed. The CEM averaged a resource manager's measured power into five-minute bins ending at its own now, posted one at most every five minutes, and dropped whatever fell outside that bin. A co-simulation's resource manager sends a window's measurements together once the window has passed, each stamped at the start of its quarter hour, so none of them ever fell in the bin, and nothing was recorded; the buffer kept them all, in a class attribute that every CEM in the process shared. The values, which S2 states in watts, also went out labelled kW, a thousand times too large. #188 bypassed the binning and posted each value at its own timestamp, deduplicated in module-level globals. flexmeasures_client.s2.measurements.MeasurementRecorder takes each measurement into the step of the sensor's resolution its timestamp falls in, and records a step once it is over by the CEM's clock, as the average of what was measured in it, in watts, with the step's end as the moment it became known. Consecutive steps go in one request, so a co-simulation's window of quarter hours costs one. A step that could not be posted is kept and tried again. A server makes one recorder and hands it to every CEM, so a resource manager that reconnects and sends its series again, as a co-simulation's does every round, is not recorded twice: the first value for a step stands, for a week. The CEM hands measurements to its recorder, its own by default, and asks the resource manager to resend one it could not take at all, such as where the sensor's resolution could not be read. get_commodity_unit, which labelled power kW, goes; nothing else used it. Not carried from #188: a barrier awaiting the posts before planning, since the co-simulation's resource manager sends its storage status before its measurements, so the barrier never covered the step being planned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 86 ++------ src/flexmeasures_client/s2/measurements.py | 132 ++++++++++++ .../s2/script/websockets_server.py | 18 +- tests/s2/test_example_end_to_end.py | 37 ++++ tests/s2/test_measurements.py | 195 ++++++++++++++++++ 5 files changed, 402 insertions(+), 66 deletions(-) create mode 100644 src/flexmeasures_client/s2/measurements.py create mode 100644 tests/s2/test_measurements.py diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 679b3e6a..c079bd40 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -5,7 +5,6 @@ import logging import math from asyncio import Queue -from collections import defaultdict from datetime import datetime, timedelta from logging import Logger from typing import Awaitable, Callable, Dict, Optional, Protocol @@ -33,9 +32,11 @@ from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError from flexmeasures_client.s2 import Handler, register from flexmeasures_client.s2.clock import Clock, SimulatedClock from flexmeasures_client.s2.control_types import ControlTypeHandler +from flexmeasures_client.s2.measurements import MeasurementRecorder from flexmeasures_client.s2.utils import ( get_latest_compatible_version, get_reception_status, @@ -93,10 +94,6 @@ class CEM(Handler): _datastore: dict _minimum_measurement_period: pd.Timedelta = pd.Timedelta(minutes=5) - _power_buffer: defaultdict = defaultdict( - list - ) # {commodity_quantity: [(timestamp, value), ...]} - def __init__( self, fm_client: FlexMeasuresClient, @@ -106,6 +103,7 @@ def __init__( datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, site_configurator: SiteConfigurator | None = None, + measurement_recorder: MeasurementRecorder | None = None, clock: Clock | None = None, accept_simulation_metadata: bool = False, **kwargs, @@ -117,6 +115,9 @@ def __init__( called with its details, in the background (see FRBCSiteConfigurator). Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. + :param measurement_recorder: where the resource manager's measured power is recorded; + one of its own by default. A server that makes a CEM per connection + hands each the same one, so a resource manager that reconnects is not recorded twice. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -144,6 +145,9 @@ def __init__( self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() self._site_configurator = site_configurator + self._measurement_recorder = measurement_recorder or MeasurementRecorder( + fm_client, logger=logger + ) self._configuring: asyncio.Task | None = None #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. self._selecting: tuple[ControlType, str] | None = None @@ -543,60 +547,24 @@ async def handle_power_measurement(self, message: PowerMeasurement): ) continue - # Store the value in the buffer - self._power_buffer[commodity_quantity].append( - (message.measurement_timestamp, power_measurement.value) - ) - - # Compute bin - now = self.now() - m = self._minimum_measurement_period // pd.Timedelta(minutes=1) - bin_end = now.replace( - second=0, microsecond=0, minute=(now.minute // m) * m - ) # e.g. 10:15:00 - bin_start = bin_end - self._minimum_measurement_period - - # If timer not due, just collect values - if not self._is_timer_due(f"power_measurement_{commodity_quantity}"): - self._logger.debug( - f"Collecting 5-minute average for {commodity_quantity} ({bin_start.isoformat()} – {bin_end.isoformat()})" - ) - continue - - # Compute average of all buffered values in last 5 minutes - buffer = self._power_buffer[commodity_quantity] - period_values = [v for (t, v) in buffer if bin_start <= t < bin_end] - - if not period_values: - self._logger.debug( - f"No samples found for {commodity_quantity} in {bin_start}–{bin_end}, skipping." - ) - continue - - avg_value = sum(period_values) / len(period_values) - self._logger.debug( - f"Posting 5-minute average for {commodity_quantity}: " - f"{avg_value} ({bin_start.isoformat()} – {bin_end.isoformat()})" - ) - - # Send measurement try: - await self._fm_client.post_sensor_data( + await self._measurement_recorder.add( sensor_id, - start=bin_start.isoformat(), - duration=self._minimum_measurement_period.isoformat(), # TODO: not specified in S2 Protocol - values=[avg_value], - unit=get_commodity_unit(commodity_quantity), + message.measurement_timestamp, + power_measurement.value, + now=self.now(), ) - except Exception as e: # noqa: B902 - intentional safety net + except (ValueError, ConnectionError, ContentTypeError) as exc: + # Not taken at all, as when the sensor's resolution could not be read: the resource manager is asked to resend it. self._logger.warning( - f"POSTing power measurement failed with error: {e}" + "Could not take measured power for %s at %s: %s", + commodity_quantity, + message.measurement_timestamp, + exc, + ) + return get_reception_status( + message, ReceptionStatusValues.TEMPORARY_ERROR ) - - # Keep only samples newer than this bin (for next period) - self._power_buffer[commodity_quantity] = [ - (t, v) for (t, v) in buffer if t >= bin_end - ] return get_reception_status(message) @@ -625,13 +593,3 @@ def handle_revoke_object(self, message: RevokeObject): async def send_message(self, message): self._logger.debug(f"Sent: {message}") await self._sending_queue.put(message) - - -def get_commodity_unit(commodity_quantity) -> str: - if "POWER" in commodity_quantity: - return "kW" - if "FLOW_RATE" in commodity_quantity: - return "m³/h" - if "TEMPERATURE" in commodity_quantity: - return "°C" - return "" diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py new file mode 100644 index 00000000..492cb0fc --- /dev/null +++ b/src/flexmeasures_client/s2/measurements.py @@ -0,0 +1,132 @@ +"""Recording what a resource manager measures on FlexMeasures sensors, one value per step of each sensor's resolution.""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime, timedelta +from logging import Logger, getLogger + +import pandas as pd + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError, JobFailedError + +_LOGGER = getLogger(__name__) + +#: The unit S2 states every power in, measured ones included. +S2_POWER_UNIT = "W" + +#: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. +REMEMBERED_FOR = timedelta(days=7) + + +class MeasurementRecorder: + """Records measured power on a sensor as the average of what was measured in each step of its resolution. + + A step is recorded once it is over by the CEM's clock, with the moment it ended as the moment it became known, + so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. + Steps are taken by the time each measurement is stamped with, not by when it arrives, + since a co-simulation's resource manager sends a window's measurements together once the window has passed, + each stamped at the start of its quarter hour, while a live one sends them as they are taken. + Consecutive steps that are over are recorded in one request. + + Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, + which a co-simulation makes every round, and so remembers what was recorded: + a step sent again is not recorded again, and the first value recorded for a step stands. + """ + + def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): + self._fm_client = fm_client + self._logger = logger or _LOGGER + self._resolutions: dict[int, timedelta] = {} + self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) + self._recorded: dict[int, set[datetime]] = defaultdict(set) + + async def add( + self, sensor_id: int, measured_at: datetime, watts: float, now: datetime + ) -> None: + """Take one measurement, and record every step of this sensor that is over by now.""" + resolution = await self._resolution(sensor_id) + step = self._step_of(measured_at, resolution) + if step in self._recorded[sensor_id]: + self._logger.debug( + "Not recording %s W on sensor %s for %s again.", watts, sensor_id, step + ) + else: + self._pending[sensor_id].setdefault(step, []).append(watts) + await self.record(sensor_id, now) + + async def record(self, sensor_id: int, now: datetime) -> None: + """Record every pending step of this sensor that is over by now, consecutive ones in one request each.""" + resolution = await self._resolution(sensor_id) + over = sorted( + step for step in self._pending[sensor_id] if step + resolution <= now + ) + for run in _consecutive(over, resolution): + values = [ + sum(samples) / len(samples) + for samples in (self._pending[sensor_id][step] for step in run) + ] + ended = run[-1] + resolution + try: + await self._fm_client.post_sensor_data( + sensor_id, + start=run[0], + duration=resolution * len(run) if resolution else timedelta(0), + values=values, + unit=S2_POWER_UNIT, + prior=ended, + ) + except ( + ValueError, + ConnectionError, + ContentTypeError, + JobFailedError, + ) as exc: + # Kept, and tried again with the next measurement. + self._logger.warning( + "Could not record measured power on sensor %s from %s to %s; trying again later: %s", + sensor_id, + run[0], + ended, + exc, + ) + continue + for step in run: + del self._pending[sensor_id][step] + self._recorded[sensor_id].add(step) + self._recorded[sensor_id] = { + step for step in self._recorded[sensor_id] if step >= now - REMEMBERED_FOR + } + + async def _resolution(self, sensor_id: int) -> timedelta: + """The sensor's resolution, asked once, since it is a property of how the sensor was set up.""" + if sensor_id not in self._resolutions: + sensor = await self._fm_client.get_sensor(sensor_id) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + if resolution is None: + raise ValueError( + f"Sensor {sensor_id} states no resolution, so its steps are unknown." + ) + self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() + return self._resolutions[sensor_id] + + @staticmethod + def _step_of(moment: datetime, resolution: timedelta) -> datetime: + """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it.""" + if not resolution: + return moment + return pd.Timestamp(moment).floor(resolution).to_pydatetime() + + +def _consecutive(steps: list[datetime], resolution: timedelta) -> list[list[datetime]]: + """Sorted steps, in runs of steps that follow one another; with no resolution, each on its own.""" + runs: list[list[datetime]] = [] + for step in steps: + if runs and resolution and step == runs[-1][-1] + resolution: + runs[-1].append(step) + else: + runs.append([step]) + return runs diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 0a7deeed..591540bd 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -19,6 +19,7 @@ ) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.measurements import MeasurementRecorder from flexmeasures_client.s2.site import ( FRBC_SITE_SENSORS, FRBCSiteConfigurator, @@ -67,6 +68,7 @@ FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) SENSORS = web.AppKey("sensors", dict) CONFIGURATOR = web.AppKey("configurator", FRBCSiteConfigurator) +RECORDER = web.AppKey("recorder", MeasurementRecorder) async def rm_details_watchdog(ws, cem: CEM): @@ -142,10 +144,18 @@ async def websocket_handler(request): # first message. if SITE_PER_RESOURCE_MANAGER: await serve_resource_manager( - ws, request.app[FM_CLIENT], configurator=request.app[CONFIGURATOR] + ws, + request.app[FM_CLIENT], + configurator=request.app[CONFIGURATOR], + recorder=request.app[RECORDER], ) else: - await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + await serve_resource_manager( + ws, + request.app[FM_CLIENT], + request.app[SENSORS], + recorder=request.app[RECORDER], + ) return ws @@ -154,6 +164,7 @@ async def serve_resource_manager( fm_client: FlexMeasuresClient, sensors: dict[str, dict] | None = None, configurator: FRBCSiteConfigurator | None = None, + recorder: MeasurementRecorder | None = None, ) -> None: """Act as the CEM for one resource manager connected over ws, until it disconnects. @@ -164,6 +175,8 @@ async def serve_resource_manager( fm_client=fm_client, logger=LOGGER, site_configurator=configurator, + # The server's, which outlives a connection, so a resource manager that reconnects is not recorded twice. + measurement_recorder=recorder, # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. power_sensor_id=( {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} @@ -309,6 +322,7 @@ async def configure_site_once(app: web.Application) -> None: ssl=parsed.scheme == "https", ) app[FM_CLIENT] = fm_client + app[RECORDER] = MeasurementRecorder(fm_client, logger=LOGGER) if SITE_PER_RESOURCE_MANAGER: # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. app[CONFIGURATOR] = FRBCSiteConfigurator( diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 8874af96..8a7ed3b9 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -928,3 +928,40 @@ def capture(**kwargs): assert made[0]["power_sensor_id"] == { "ELECTRIC.POWER.L1": sensors["measured-power"]["id"] } + + +@pytest.mark.asyncio +async def test_every_cem_the_server_makes_records_with_the_servers_recorder(): + """It outlives a connection, so what a reconnecting resource manager sends again is not recorded twice.""" + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + with patch.object(server, "SITE_PER_RESOURCE_MANAGER", True): + await server.configure_site_once(app) + assert isinstance(app[server.RECORDER], MeasurementRecorder) + await app[server.FM_CLIENT].close() + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + recorder = MeasurementRecorder(_flexmeasures()) + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), sensors, recorder=recorder + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["measurement_recorder"] is recorder diff --git a/tests/s2/test_measurements.py b/tests/s2/test_measurements.py new file mode 100644 index 00000000..697bd7bc --- /dev/null +++ b/tests/s2/test_measurements.py @@ -0,0 +1,195 @@ +"""Recording a resource manager's measured power, live and in a co-simulation.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.measurements import MeasurementRecorder + +T = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) +Q = timedelta(minutes=15) + + +def _recorder(resolution: str = "PT15M") -> tuple[MeasurementRecorder, AsyncMock]: + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": resolution}) + return MeasurementRecorder(fm), fm + + +def _posts(fm) -> list[dict]: + return [ + c.kwargs | {"sensor_id": c.args[0]} for c in fm.post_sensor_data.await_args_list + ] + + +@pytest.mark.asyncio +async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_request(): + """Its resource manager sends a window's measurements once the window has passed, each stamped at its quarter hour's start. + + Binned by the moment they arrived, none of them fell in the step the CEM's clock was in, and none was recorded. + """ + recorder, fm = _recorder() + window_end = T + 4 * Q + for i, watts in enumerate([1000, 2000, 0, 500]): + await recorder.add(7, T + i * Q, watts, now=window_end) + (post,) = _posts(fm)[-1:] + recorded = {} + for p in _posts(fm): + for k, v in enumerate(p["values"]): + recorded[p["start"] + k * Q] = v + assert recorded == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} + assert ( + post["unit"] == "W" + ) # as S2 states power; it used to go out labelled kW, a thousand times too large + assert ( + post["prior"] == post["start"] + len(post["values"]) * Q + ) # known at the end of what it measures + + +@pytest.mark.asyncio +async def test_a_live_resource_managers_samples_are_averaged_per_step_once_it_is_over(): + recorder, fm = _recorder() + await recorder.add(7, T + timedelta(minutes=1), 1000, now=T + timedelta(minutes=1)) + await recorder.add(7, T + timedelta(minutes=8), 2000, now=T + timedelta(minutes=8)) + fm.post_sensor_data.assert_not_awaited() # the step is not over + await recorder.add( + 7, T + Q + timedelta(minutes=1), 800, now=T + Q + timedelta(minutes=1) + ) + (post,) = _posts(fm) + assert (post["start"], post["values"], post["duration"]) == (T, [1500], Q) + + +@pytest.mark.asyncio +async def test_a_step_sent_again_is_not_recorded_again(): + """A co-simulation's resource manager reconnects every round and sends its series again.""" + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add( + 7, T, 999, now=T + Q + ) # the first value recorded for a step stands + assert len(_posts(fm)) == 1 + + +@pytest.mark.asyncio +async def test_a_step_that_could_not_be_recorded_is_tried_again(caplog): + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None] + await recorder.add(7, T, 1000, now=T + Q) + assert "trying again later" in caplog.text + await recorder.add(7, T + Q, 2000, now=T + 2 * Q) + retried = _posts(fm)[-1] + assert (retried["start"], retried["values"]) == (T, [1000, 2000]) + + +@pytest.mark.asyncio +async def test_steps_that_do_not_follow_one_another_go_in_requests_of_their_own(): + recorder, fm = _recorder() + await recorder.add(7, T, 1, now=T) + await recorder.add(7, T + 2 * Q, 3, now=T) + await recorder.record(7, now=T + 3 * Q) + assert [(p["start"], p["values"]) for p in _posts(fm)] == [ + (T, [1]), + (T + 2 * Q, [3]), + ] + + +@pytest.mark.asyncio +async def test_each_sensor_is_recorded_at_its_own_resolution(): + recorder, fm = _recorder("PT1H") + await recorder.add(7, T + timedelta(minutes=20), 10, now=T + timedelta(minutes=20)) + await recorder.add(7, T + timedelta(minutes=40), 30, now=T + timedelta(hours=1)) + (post,) = _posts(fm) + assert (post["start"], post["values"], post["duration"]) == ( + T, + [20], + timedelta(hours=1), + ) + fm.get_sensor.assert_awaited_once() # asked once + + +@pytest.mark.asyncio +async def test_on_an_instantaneous_sensor_each_measurement_is_its_own_reading(): + recorder, fm = _recorder("PT0M") + await recorder.add(7, T, 10, now=T) + await recorder.add(7, T + timedelta(seconds=30), 20, now=T + timedelta(seconds=30)) + assert [(p["start"], p["values"], p["duration"]) for p in _posts(fm)] == [ + (T, [10], timedelta(0)), + (T + timedelta(seconds=30), [20], timedelta(0)), + ] + + +@pytest.mark.asyncio +async def test_a_sensor_stating_no_resolution_cannot_take_measurements(): + recorder, fm = _recorder() + fm.get_sensor = AsyncMock(return_value={}) + with pytest.raises(ValueError, match="no resolution"): + await recorder.add(7, T, 10, now=T + Q) + + +@pytest.mark.asyncio +async def test_what_was_recorded_is_forgotten_after_a_week(): + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.record(7, now=T + timedelta(days=8)) + await recorder.add(7, T, 1000, now=T + timedelta(days=8)) + assert len(_posts(fm)) == 2 + + +@pytest.mark.asyncio +async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where_it_cannot(): + from s2python.common import ( + CommodityQuantity, + PowerMeasurement, + PowerValue, + ReceptionStatusValues, + ) + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + from flexmeasures_client.s2.utils import get_unique_id + + recorder, fm = _recorder() + clock = SimulatedClock(T + timedelta(minutes=5)) + cem = CEM( + fm_client=fm, + clock=clock, + power_sensor_id={"ELECTRIC.POWER.L1": 7}, + measurement_recorder=recorder, + ) + + def measurement() -> PowerMeasurement: + return PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=T, + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=750.0 + ) + ], + ) + + await cem.handle_message(measurement()) + assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + fm.post_sensor_data.assert_not_awaited() # its step is not over by the simulated clock + clock.set(T + Q) + await cem.handle_message(measurement()) + assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + assert _posts(fm)[0]["values"] == [750.0] + + fm.get_sensor = AsyncMock(side_effect=ConnectionError("down")) + recorder._resolutions.clear() + await cem.handle_message(measurement()) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + + +def test_a_cem_has_a_recorder_of_its_own_unless_given_one(): + from flexmeasures_client.s2.cem import CEM + + assert isinstance(CEM(fm_client=None)._measurement_recorder, MeasurementRecorder) From 809b1362024858690cc7ec5ecf3ce3e98a90bbec Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:15:03 +0100 Subject: [PATCH 49/87] docs(changelog): the measurement recorder Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c8c5187d..584bb264 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -30,6 +30,7 @@ Added * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] * ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] +* ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed ------- @@ -51,11 +52,13 @@ Changed Removed ------- +* ``flexmeasures_client.s2.cem.get_commodity_unit``, which stated power in kW where S2 states it in W [`#266 `__] * ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] Fixed ----- +* A CEM records measured power by the moment it was measured, once each step of its sensor's resolution is over, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] From ab56606346da7743f7d12e160551fd29871d1eb7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:29:14 +0100 Subject: [PATCH 50/87] fix(S2): what the review of the measurement recorder found - A step the server kept refusing blocked its sensor for good: steps were never dropped, and consecutive ones went in one request, so every later request carried the bad step and failed with it. Each step now goes in a request of its own, is tried a few times with a wait in between, and is given up on, with an error naming its values, after that. - One request for several steps gave the earlier ones a belief time after their own end. With a request per step, each is known at its own end, as the docstring said. The batching the first version claimed only ever happened after a failure, and its test checked nothing about it; both are gone. - Two measurements for one sensor at once, from two CEMs or a sample landing while its step's post was in flight, posted the step twice and then raised a KeyError the CEM did not catch, so no reception status went out. Recording is now one post at a time per sensor. - Posting held up each measurement's reception status, which #188 had kept off the co-simulation's critical path by posting in the background. Recording happens in the background again, a defect in it is logged, and a failed post is tried again a while later without waiting for another measurement, which a co-simulation's final window never gets. - A moment in the repeated hour of an autumn DST change could not be floored to its step, so such a measurement was refused on every resend. Its step is now the later boundary not after it, as align_to_resolution finds it. - Remembering recorded steps cost a rebuild of the whole set per measurement, which on an instantaneous sensor grew large; the oldest are now dropped from the front. - A message with measurements for several sensors was taken in part where one sensor could not be read, and, asked for again, had that part taken twice. Every sensor is now made ready before any value is taken. - The server's request handler is tested for handing each CEM its recorder, and the changelog says what changes for a live resource manager: one value per step of the sensor's resolution, recorded with the first measurement after it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- src/flexmeasures_client/s2/cem.py | 60 +++--- src/flexmeasures_client/s2/measurements.py | 221 ++++++++++++++------- tests/s2/test_example_end_to_end.py | 33 +++ tests/s2/test_measurements.py | 216 ++++++++++++++------ 5 files changed, 363 insertions(+), 169 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 584bb264..d0e74592 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -58,7 +58,7 @@ Removed Fixed ----- -* A CEM records measured power by the moment it was measured, once each step of its sensor's resolution is over, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process [`#266 `__] +* A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index c079bd40..6d1f70e4 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -115,9 +115,9 @@ def __init__( called with its details, in the background (see FRBCSiteConfigurator). Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. - :param measurement_recorder: where the resource manager's measured power is recorded; - one of its own by default. A server that makes a CEM per connection - hands each the same one, so a resource manager that reconnects is not recorded twice. + :param measurement_recorder: where the resource manager's measured power is recorded; one of its own by default. + A server that makes a CEM per connection hands each the same one, + so a resource manager that reconnects is not recorded twice. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -527,44 +527,38 @@ async def handle_power_measurement(self, message: PowerMeasurement): self._ensure_configuring() return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + taken: list[tuple[int, float]] = [] for power_measurement in message.values: commodity_quantity = power_measurement.commodity_quantity.value # Without a sensor for it, a measurement is not recorded. # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. - if self.power_sensor_id: - s_id = self.power_sensor_id.get(commodity_quantity) - if s_id is None: - # TODO: create a new sensor or return ReceptionStatus - self._logger.debug( - f"No power sensor set up for {commodity_quantity}. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue - sensor_id = s_id - else: - self._logger.warning( - f"No power sensor IDs set up. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue - - try: - await self._measurement_recorder.add( - sensor_id, - message.measurement_timestamp, - power_measurement.value, - now=self.now(), - ) - except (ValueError, ConnectionError, ContentTypeError) as exc: - # Not taken at all, as when the sensor's resolution could not be read: the resource manager is asked to resend it. - self._logger.warning( - "Could not take measured power for %s at %s: %s", + sensor_id = (self.power_sensor_id or {}).get(commodity_quantity) + if sensor_id is None: + self._logger.debug( + "No power sensor set up for %s. Ignoring measurement %s at %s.", commodity_quantity, + power_measurement.value, message.measurement_timestamp, - exc, - ) - return get_reception_status( - message, ReceptionStatusValues.TEMPORARY_ERROR ) + continue + taken.append((sensor_id, power_measurement.value)) + + # Every sensor is made ready before any value is taken, so that a message asked for again is not taken in part twice. + try: + for sensor_id, _ in taken: + await self._measurement_recorder.resolution(sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not take the measured power at %s, so the resource manager is asked to send it again: %s", + message.measurement_timestamp, + exc, + ) + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + for sensor_id, watts in taken: + await self._measurement_recorder.add( + sensor_id, message.measurement_timestamp, watts, now=self.now() + ) return get_reception_status(message) diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py index 492cb0fc..5f154514 100644 --- a/src/flexmeasures_client/s2/measurements.py +++ b/src/flexmeasures_client/s2/measurements.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections import defaultdict from datetime import datetime, timedelta from logging import Logger, getLogger @@ -9,7 +10,12 @@ import pandas as pd from flexmeasures_client.client import FlexMeasuresClient -from flexmeasures_client.exceptions import ContentTypeError, JobFailedError +from flexmeasures_client.exceptions import ( + ContentTypeError, + InsufficientServerVersionError, + JobFailedError, + WrongAPIVersionError, +) _LOGGER = getLogger(__name__) @@ -19,20 +25,40 @@ #: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. REMEMBERED_FOR = timedelta(days=7) +#: How often a step is tried before it is given up on, and how long to wait between tries. +ATTEMPTS = 3 +RETRY_AFTER = timedelta(seconds=30) + +#: What a post can fail with, short of a defect in this code. +POST_FAILURES = ( + ValueError, + ConnectionError, + ContentTypeError, + JobFailedError, + InsufficientServerVersionError, + WrongAPIVersionError, +) + class MeasurementRecorder: """Records measured power on a sensor as the average of what was measured in each step of its resolution. - A step is recorded once it is over by the CEM's clock, with the moment it ended as the moment it became known, + A step is recorded once it is over by the CEM's clock, as it stands when the step's last measurement arrives, + with the moment it ended as the moment it became known, so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. Steps are taken by the time each measurement is stamped with, not by when it arrives, since a co-simulation's resource manager sends a window's measurements together once the window has passed, - each stamped at the start of its quarter hour, while a live one sends them as they are taken. - Consecutive steps that are over are recorded in one request. + each stamped at the start of its quarter hour, while a live one sends them as they are taken, + in which case a step is recorded with the first measurement after it is over. + + Recording happens in the background, so that a resource manager is not kept waiting for FlexMeasures + before its measurement is acknowledged, one post at a time per sensor. + A step that could not be recorded is tried again a while later, and given up on after a few tries. Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, which a co-simulation makes every round, and so remembers what was recorded: a step sent again is not recorded again, and the first value recorded for a step stands. + A sensor is meant to be one resource manager's: one that several share has their measurements averaged together. """ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): @@ -40,93 +66,136 @@ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): self._logger = logger or _LOGGER self._resolutions: dict[int, timedelta] = {} self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) - self._recorded: dict[int, set[datetime]] = defaultdict(set) + self._attempts: dict[int, dict[datetime, int]] = defaultdict(dict) + #: By sensor, the steps recorded, in the order they were, which is roughly that of time. + self._recorded: dict[int, dict[datetime, None]] = defaultdict(dict) + self._locks: dict[int, asyncio.Lock] = defaultdict(asyncio.Lock) + self._tasks: set[asyncio.Task] = set() + + async def resolution(self, sensor_id: int) -> timedelta: + """The sensor's resolution, asked once, since it is a property of how the sensor was set up. + + A caller taking several measurements at once asks this for each sensor first, + so that none is taken where one of them cannot be. + """ + if sensor_id not in self._resolutions: + sensor = await self._fm_client.get_sensor(sensor_id) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + if resolution is None: + raise ValueError( + f"Sensor {sensor_id} states no resolution, so its steps are unknown." + ) + self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() + return self._resolutions[sensor_id] async def add( self, sensor_id: int, measured_at: datetime, watts: float, now: datetime ) -> None: - """Take one measurement, and record every step of this sensor that is over by now.""" - resolution = await self._resolution(sensor_id) - step = self._step_of(measured_at, resolution) + """Take one measurement, and record in the background every step of this sensor that is over by now.""" + resolution = await self.resolution(sensor_id) + step = step_of(measured_at, resolution) if step in self._recorded[sensor_id]: self._logger.debug( "Not recording %s W on sensor %s for %s again.", watts, sensor_id, step ) else: self._pending[sensor_id].setdefault(step, []).append(watts) - await self.record(sensor_id, now) + self._in_background(self.record(sensor_id, now)) async def record(self, sensor_id: int, now: datetime) -> None: - """Record every pending step of this sensor that is over by now, consecutive ones in one request each.""" - resolution = await self._resolution(sensor_id) - over = sorted( - step for step in self._pending[sensor_id] if step + resolution <= now - ) - for run in _consecutive(over, resolution): - values = [ - sum(samples) / len(samples) - for samples in (self._pending[sensor_id][step] for step in run) - ] - ended = run[-1] + resolution - try: - await self._fm_client.post_sensor_data( - sensor_id, - start=run[0], - duration=resolution * len(run) if resolution else timedelta(0), - values=values, - unit=S2_POWER_UNIT, - prior=ended, - ) - except ( - ValueError, - ConnectionError, - ContentTypeError, - JobFailedError, - ) as exc: - # Kept, and tried again with the next measurement. - self._logger.warning( - "Could not record measured power on sensor %s from %s to %s; trying again later: %s", - sensor_id, - run[0], - ended, - exc, - ) - continue - for step in run: + """Record every pending step of this sensor that is over by now, each in a request of its own. + + Each with its own end as the moment it became known, which one request for several steps cannot say. + One at a time per sensor, so a step is not posted twice, nor taken from under a post in flight. + """ + async with self._locks[sensor_id]: + resolution = await self.resolution(sensor_id) + over = sorted( + step for step in self._pending[sensor_id] if step + resolution <= now + ) + failed = False + for step in over: + samples = self._pending[sensor_id][step] + try: + await self._fm_client.post_sensor_data( + sensor_id, + start=step, + duration=resolution, + values=[sum(samples) / len(samples)], + unit=S2_POWER_UNIT, + prior=step + resolution, + ) + except POST_FAILURES as exc: + failed = True + attempts = self._attempts[sensor_id].get(step, 0) + 1 + if attempts < ATTEMPTS: + self._attempts[sensor_id][step] = attempts + self._logger.warning( + "Could not record measured power on sensor %s for %s; trying again later: %s", + sensor_id, + step, + exc, + ) + continue + self._logger.error( + "Giving up recording %s W of measured power on sensor %s for %s after %s tries: %s", + samples, + sensor_id, + step, + attempts, + exc, + ) del self._pending[sensor_id][step] - self._recorded[sensor_id].add(step) - self._recorded[sensor_id] = { - step for step in self._recorded[sensor_id] if step >= now - REMEMBERED_FOR - } + self._attempts[sensor_id].pop(step, None) + self._recorded[sensor_id][step] = None + self._forget_before(sensor_id, now - REMEMBERED_FOR) + if failed: + self._in_background(self._retry(sensor_id, now)) + + async def _retry(self, sensor_id: int, now: datetime) -> None: + await asyncio.sleep(RETRY_AFTER.total_seconds()) + await self.record(sensor_id, now) - async def _resolution(self, sensor_id: int) -> timedelta: - """The sensor's resolution, asked once, since it is a property of how the sensor was set up.""" - if sensor_id not in self._resolutions: - sensor = await self._fm_client.get_sensor(sensor_id) - resolution = ( - sensor.get("event_resolution") if isinstance(sensor, dict) else None + async def settle(self) -> None: + """Wait for every recording in the background to finish, retries included.""" + while self._tasks: + await asyncio.gather(*list(self._tasks), return_exceptions=True) + + def _in_background(self, recording) -> None: + task = asyncio.create_task(recording) + self._tasks.add(task) + task.add_done_callback(self._done) + + def _done(self, task: asyncio.Task) -> None: + self._tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + self._logger.error( + "Recording measured power failed.", exc_info=task.exception() ) - if resolution is None: - raise ValueError( - f"Sensor {sensor_id} states no resolution, so its steps are unknown." - ) - self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() - return self._resolutions[sensor_id] - @staticmethod - def _step_of(moment: datetime, resolution: timedelta) -> datetime: - """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it.""" - if not resolution: - return moment - return pd.Timestamp(moment).floor(resolution).to_pydatetime() + def _forget_before(self, sensor_id: int, cutoff: datetime) -> None: + """Forget recorded steps from before the cutoff, oldest first, so that this costs little when nothing is old.""" + recorded = self._recorded[sensor_id] + while recorded: + oldest = next(iter(recorded)) + if oldest >= cutoff: + break + del recorded[oldest] -def _consecutive(steps: list[datetime], resolution: timedelta) -> list[list[datetime]]: - """Sorted steps, in runs of steps that follow one another; with no resolution, each on its own.""" - runs: list[list[datetime]] = [] - for step in steps: - if runs and resolution and step == runs[-1][-1] + resolution: - runs[-1].append(step) - else: - runs.append([step]) - return runs +def step_of(moment: datetime, resolution: timedelta) -> datetime: + """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it. + + In the repeated hour of an autumn DST change a boundary exists twice, and the step is the later of the two + that is not after the moment; taken from the moment's pass alone, flooring cannot always be done at all. + """ + if not resolution: + return moment + at = pd.Timestamp(moment) + candidates = [ + at.floor(resolution, ambiguous=first_pass, nonexistent="shift_forward") + for first_pass in (True, False) + ] + return max(c for c in candidates if c <= at).to_pydatetime() diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 8a7ed3b9..12a64aa3 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -965,3 +965,36 @@ def capture(**kwargs): await cem_end._inbox.put(_CLOSED) await asyncio.wait_for(serving, timeout=5) assert made[0]["measurement_recorder"] is recorder + + +@pytest.mark.asyncio +@pytest.mark.parametrize("per_resource_manager", [False, True]) +async def test_a_connection_is_served_with_the_servers_recorder(per_resource_manager): + from unittest.mock import patch + + from aiohttp import web + from aiohttp.test_utils import TestClient, TestServer + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + app.add_routes([web.get("/ws", server.websocket_handler)]) + app[server.FM_CLIENT] = _flexmeasures() + app[server.RECORDER] = MeasurementRecorder(app[server.FM_CLIENT]) + app[server.SENSORS] = {"power": {"id": 1}} + app[server.CONFIGURATOR] = server.FRBCSiteConfigurator( + app[server.FM_CLIENT], asset_type_id=6 + ) + served = AsyncMock() + with ( + patch.object(server, "serve_resource_manager", served), + patch.object(server, "SITE_PER_RESOURCE_MANAGER", per_resource_manager), + ): + async with TestClient(TestServer(app)) as client: + async with client.ws_connect("/ws"): + await _until(lambda: served.await_count) + assert served.await_args.kwargs["recorder"] is app[server.RECORDER] + if per_resource_manager: + assert served.await_args.kwargs["configurator"] is app[server.CONFIGURATOR] + else: + assert served.await_args.args[2] is app[server.SENSORS] diff --git a/tests/s2/test_measurements.py b/tests/s2/test_measurements.py index 697bd7bc..1fb3182e 100644 --- a/tests/s2/test_measurements.py +++ b/tests/s2/test_measurements.py @@ -2,13 +2,16 @@ from __future__ import annotations +import asyncio from datetime import datetime, timedelta, timezone -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch +from zoneinfo import ZoneInfo import pytest from flexmeasures_client.client import FlexMeasuresClient -from flexmeasures_client.s2.measurements import MeasurementRecorder +from flexmeasures_client.s2 import measurements +from flexmeasures_client.s2.measurements import MeasurementRecorder, step_of T = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) Q = timedelta(minutes=15) @@ -26,8 +29,12 @@ def _posts(fm) -> list[dict]: ] +def _recorded(fm) -> dict[datetime, float]: + return {p["start"]: p["values"][0] for p in _posts(fm)} + + @pytest.mark.asyncio -async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_request(): +async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded(): """Its resource manager sends a window's measurements once the window has passed, each stamped at its quarter hour's start. Binned by the moment they arrived, none of them fell in the step the CEM's clock was in, and none was recorded. @@ -36,18 +43,16 @@ async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_r window_end = T + 4 * Q for i, watts in enumerate([1000, 2000, 0, 500]): await recorder.add(7, T + i * Q, watts, now=window_end) - (post,) = _posts(fm)[-1:] - recorded = {} - for p in _posts(fm): - for k, v in enumerate(p["values"]): - recorded[p["start"] + k * Q] = v - assert recorded == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} - assert ( - post["unit"] == "W" - ) # as S2 states power; it used to go out labelled kW, a thousand times too large - assert ( - post["prior"] == post["start"] + len(post["values"]) * Q - ) # known at the end of what it measures + await recorder.settle() + assert _recorded(fm) == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} + for post in _posts(fm): + assert ( + post["unit"] == "W" + ) # as S2 states power; it went out labelled kW, a thousand times too large + assert post["duration"] == Q + assert ( + post["prior"] == post["start"] + Q + ) # each known at the end of what it measures @pytest.mark.asyncio @@ -55,12 +60,13 @@ async def test_a_live_resource_managers_samples_are_averaged_per_step_once_it_is recorder, fm = _recorder() await recorder.add(7, T + timedelta(minutes=1), 1000, now=T + timedelta(minutes=1)) await recorder.add(7, T + timedelta(minutes=8), 2000, now=T + timedelta(minutes=8)) + await recorder.settle() fm.post_sensor_data.assert_not_awaited() # the step is not over await recorder.add( 7, T + Q + timedelta(minutes=1), 800, now=T + Q + timedelta(minutes=1) ) - (post,) = _posts(fm) - assert (post["start"], post["values"], post["duration"]) == (T, [1500], Q) + await recorder.settle() + assert _recorded(fm) == {T: 1500} @pytest.mark.asyncio @@ -68,34 +74,67 @@ async def test_a_step_sent_again_is_not_recorded_again(): """A co-simulation's resource manager reconnects every round and sends its series again.""" recorder, fm = _recorder() await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() await recorder.add(7, T, 1000, now=T + Q) await recorder.add( 7, T, 999, now=T + Q ) # the first value recorded for a step stands + await recorder.settle() assert len(_posts(fm)) == 1 @pytest.mark.asyncio -async def test_a_step_that_could_not_be_recorded_is_tried_again(caplog): +async def test_a_step_that_could_not_be_recorded_is_tried_again_without_another_measurement( + caplog, +): + """The final window of a co-simulation has no measurement after it to try again with.""" recorder, fm = _recorder() fm.post_sensor_data.side_effect = [ConnectionError("down"), None] - await recorder.add(7, T, 1000, now=T + Q) + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() assert "trying again later" in caplog.text - await recorder.add(7, T + Q, 2000, now=T + 2 * Q) - retried = _posts(fm)[-1] - assert (retried["start"], retried["values"]) == (T, [1000, 2000]) + assert [p["start"] for p in _posts(fm)] == [T, T] @pytest.mark.asyncio -async def test_steps_that_do_not_follow_one_another_go_in_requests_of_their_own(): +async def test_a_step_that_keeps_failing_is_given_up_on_and_does_not_hold_up_the_rest( + caplog, +): + """Posted together, one step the server always refused failed every request after it, and nothing was recorded again.""" recorder, fm = _recorder() - await recorder.add(7, T, 1, now=T) - await recorder.add(7, T + 2 * Q, 3, now=T) - await recorder.record(7, now=T + 3 * Q) - assert [(p["start"], p["values"]) for p in _posts(fm)] == [ - (T, [1]), - (T + 2 * Q, [3]), - ] + + async def refuse_the_first_step(sensor_id, **kwargs): + if kwargs["start"] == T: + raise ValueError("422") + + fm.post_sensor_data.side_effect = refuse_the_first_step + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add(7, T + Q, 2000, now=T + 2 * Q) + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == measurements.ATTEMPTS + assert T + Q in [p["start"] for p in _posts(fm)] + assert "Giving up" in caplog.text + assert recorder._pending[7] == {} + + +@pytest.mark.asyncio +async def test_two_measurements_for_one_sensor_at_once_post_each_step_once(): + """Two CEMs on one sensor, or a sample arriving while its step's post is in flight, posted it twice and then raised.""" + recorder, fm = _recorder() + release = asyncio.Event() + + async def slow(sensor_id, **kwargs): + await release.wait() + + fm.post_sensor_data.side_effect = slow + await recorder.add(7, T, 100, now=T + Q) + await asyncio.sleep(0) + await recorder.add(7, T + timedelta(minutes=5), 200, now=T + Q) + release.set() + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 1 @pytest.mark.asyncio @@ -103,6 +142,7 @@ async def test_each_sensor_is_recorded_at_its_own_resolution(): recorder, fm = _recorder("PT1H") await recorder.add(7, T + timedelta(minutes=20), 10, now=T + timedelta(minutes=20)) await recorder.add(7, T + timedelta(minutes=40), 30, now=T + timedelta(hours=1)) + await recorder.settle() (post,) = _posts(fm) assert (post["start"], post["values"], post["duration"]) == ( T, @@ -117,6 +157,7 @@ async def test_on_an_instantaneous_sensor_each_measurement_is_its_own_reading(): recorder, fm = _recorder("PT0M") await recorder.add(7, T, 10, now=T) await recorder.add(7, T + timedelta(seconds=30), 20, now=T + timedelta(seconds=30)) + await recorder.settle() assert [(p["start"], p["values"], p["duration"]) for p in _posts(fm)] == [ (T, [10], timedelta(0)), (T + timedelta(seconds=30), [20], timedelta(0)), @@ -135,23 +176,56 @@ async def test_a_sensor_stating_no_resolution_cannot_take_measurements(): async def test_what_was_recorded_is_forgotten_after_a_week(): recorder, fm = _recorder() await recorder.add(7, T, 1000, now=T + Q) - await recorder.record(7, now=T + timedelta(days=8)) + await recorder.settle() + await recorder.add(7, T + timedelta(days=8), 1, now=T + timedelta(days=8)) + await recorder.settle() + assert T not in recorder._recorded[7] await recorder.add(7, T, 1000, now=T + timedelta(days=8)) - assert len(_posts(fm)) == 2 + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 2 -@pytest.mark.asyncio -async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where_it_cannot(): - from s2python.common import ( - CommodityQuantity, - PowerMeasurement, - PowerValue, - ReceptionStatusValues, +def test_a_step_in_the_repeated_autumn_hour_is_found_in_the_pass_the_moment_is_in(): + """pandas cannot floor such a moment by its pass alone, and the CEM answered every resend with a temporary error.""" + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 37, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + step = step_of(second_pass, Q) + assert step.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 30, tzinfo=timezone.utc + ) + + +def test_steps_are_on_the_moments_own_clock(): + kolkata = ZoneInfo("Asia/Kolkata") + assert step_of( + datetime(2026, 11, 5, 14, 40, tzinfo=kolkata), timedelta(hours=1) + ) == datetime(2026, 11, 5, 14, 0, tzinfo=kolkata) + + +def _measurement(*values: tuple[str, float]): + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + return PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=T, + values=[ + PowerValue(commodity_quantity=CommodityQuantity(q), value=v) + for q, v in values + ], ) + +@pytest.mark.asyncio +async def test_the_cem_hands_measured_power_to_its_recorder_on_its_own_clock(): + from s2python.common import ReceptionStatusValues + from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.clock import SimulatedClock - from flexmeasures_client.s2.utils import get_unique_id recorder, fm = _recorder() clock = SimulatedClock(T + timedelta(minutes=5)) @@ -161,35 +235,59 @@ async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where power_sensor_id={"ELECTRIC.POWER.L1": 7}, measurement_recorder=recorder, ) - - def measurement() -> PowerMeasurement: - return PowerMeasurement( - message_id=get_unique_id(), - measurement_timestamp=T, - values=[ - PowerValue( - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=750.0 - ) - ], - ) - - await cem.handle_message(measurement()) + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + await recorder.settle() fm.post_sensor_data.assert_not_awaited() # its step is not over by the simulated clock clock.set(T + Q) - await cem.handle_message(measurement()) - assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value - assert _posts(fm)[0]["values"] == [750.0] + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) + await cem.get_message() + await recorder.settle() + assert _recorded(fm) == {T: 750.0} + + +@pytest.mark.asyncio +async def test_a_message_the_cem_cannot_take_in_full_is_taken_not_at_all(): + """Asked for again, a message taken in part had the part taken twice, which skewed its average.""" + from s2python.common import ReceptionStatusValues + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + + recorder, fm = _recorder() + + async def readable(sensor_id, **kwargs): + if sensor_id == 8: + raise ConnectionError("down") + return {"event_resolution": "PT15M"} - fm.get_sensor = AsyncMock(side_effect=ConnectionError("down")) - recorder._resolutions.clear() - await cem.handle_message(measurement()) + fm.get_sensor = AsyncMock(side_effect=readable) + cem = CEM( + fm_client=fm, + clock=SimulatedClock(T + timedelta(minutes=5)), + power_sensor_id={"ELECTRIC.POWER.L1": 7, "ELECTRIC.POWER.L2": 8}, + measurement_recorder=recorder, + ) + await cem.handle_message( + _measurement(("ELECTRIC.POWER.L1", 750.0), ("ELECTRIC.POWER.L2", 10.0)) + ) assert (await cem.get_message())[ "status" ] == ReceptionStatusValues.TEMPORARY_ERROR.value + assert recorder._pending[7] == {} def test_a_cem_has_a_recorder_of_its_own_unless_given_one(): from flexmeasures_client.s2.cem import CEM assert isinstance(CEM(fm_client=None)._measurement_recorder, MeasurementRecorder) + + +@pytest.mark.asyncio +async def test_a_defect_while_recording_is_logged_rather_than_lost(caplog): + """An exception no post is expected to raise surfaces in the log, not as a task exception never retrieved.""" + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = TypeError("a malformed error payload") + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + assert "Recording measured power failed" in caplog.text From b8aff1dc53f37141490e04232d330ace4d9f73ea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:11:19 +0100 Subject: [PATCH 51/87] fix(examples): the co-simulation stack runs on a current FlexMeasures image Found by running the stack, from checkouts of its own, against FlexMeasures main (1.1.0) with the community scheduler plugin, and a two-home, one-day co-simulation through it (no community steering): every step ran, every home's measured power was recorded for every quarter hour, and no CEM error remained. It took four fixes to the overlay first. - The server and the workers installed this client with the system pip, while FlexMeasures' image runs FlexMeasures and its plugins from its own virtualenv, so the plugin could not import flexmeasures_client and the server never booted. They now install with that interpreter's pip, as the CEMs already did, and no longer try to install /app itself, which the current image has no project file in. - A checkout mounted without git history it can read, as a git worktree is, gives hatch-vcs no version, so the editable install failed. The install is given one, in the plain SETUPTOOLS_SCM_PRETEND_VERSION, since hatch-vcs does not match the per-distribution variable the CEMs had. - A development server trusts only loopback names, and the CEMs reach it as "server", so every request of theirs was refused as from an untrusted host. The server now trusts that name too. - The README says how to build FlexMeasures' image from a worktree (its version as a build argument) and how to move the ports where they are taken. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 12 ++++++--- .../S2-co-simulation/docker-compose.cem.yml | 27 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 9e1b8e0f..02d76fef 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -27,7 +27,13 @@ whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the flexmeasures checkout it runs from. A compose project name of its own does not separate that. To keep a development database you care about out of it, -run it from a folder of its own holding checkouts (git worktrees will do) -named `flexmeasures` and `flexmeasures-client`: +run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`: the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. -Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too. +Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, +which it cannot through a worktree, so give it as a build argument, +`FLEXMEASURES_VERSION`, for each service the overlay builds. +Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too; +where they are not, an extra overlay can move them with compose's `!override`. + +It runs FlexMeasures as its image installs it, with the plugins and this client installed into the image's virtualenv, +and expects a development server, so the server trusts the name the CEMs reach it by, `server`, alongside the loopback names. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index 79a5b364..07a48049 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -37,8 +37,6 @@ x-cem: &cem S2_ACCEPT_SIMULATION_METADATA: "1" # And each home is a site of its own, rather than all of them sharing one. S2_SITE_PER_RESOURCE_MANAGER: "1" - # The checkout is mounted without its git history, which is where its version would come from. - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLEXMEASURES_CLIENT: "0.0.0" volumes: - ../flexmeasures-client:/app/flexmeasures-client:rw entrypoint: ["/bin/sh", "-c"] @@ -49,7 +47,9 @@ x-cem: &cem # which may come without pip, and cannot see what the system pip installs. python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade # s2-python as this repo locks it, and as the co-simulation's resource managers use it. - python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py # Shared definition for every RQ worker: one per home simulated at once, @@ -83,9 +83,12 @@ x-worker: &worker LOGGING_LEVEL: INFO command: - | - pip install --break-system-packages -e /app - pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt - pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" # Drop any lingering registration for the ingestion queue before this worker starts. # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. @@ -117,6 +120,9 @@ services: ports: - "6380:6379" server: + environment: + # The CEMs reach the server by its service name, which a development server does not trust by default. + TRUSTED_HOSTS: "localhost,.localhost,127.0.0.1,[::1],server" volumes: - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw - ./flexmeasures-instance/:/app/instance/:rw @@ -124,9 +130,12 @@ services: - ../flexmeasures-client:/app/flexmeasures-client:rw command: - | - pip install --break-system-packages -e /app - pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt - pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" flexmeasures db upgrade if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then flexmeasures add toy-account --name 'Docker Toy Account' From 40288e8b442ad87768c8e0ced865d634bcaf0332 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:11:48 +0100 Subject: [PATCH 52/87] docs(changelog): the co-simulation stack on FlexMeasures 1.1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d0e74592..c2bf6885 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -29,7 +29,7 @@ Added * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] -* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets, run on FlexMeasures 1.1 [`#265 `__, `#267 `__] * ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed From 48073881ce143519af677e7e7aa1dc1571cb9e74 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:05 +0100 Subject: [PATCH 53/87] docs(examples): what the co-simulation stack runs, and how to build it from a worktree Review found the README saying the stack runs FlexMeasures as its image installs it, while the mounted checkout is what runs, and an s2-python pin on the server and workers, which speak no S2 and whose locked environment it should not shift. The README now shows the build-argument overlay a worktree needs, says the UI may lack its fetched libraries, and the changelog no longer claims a FlexMeasures version the overlay does not pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- examples/S2-co-simulation/README.md | 28 +++++++++++++++---- .../S2-co-simulation/docker-compose.cem.yml | 6 ++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c2bf6885..06623ba4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -29,7 +29,7 @@ Added * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] -* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets, run on FlexMeasures 1.1 [`#265 `__, `#267 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__, `#267 `__] * ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 02d76fef..13b364dd 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -29,11 +29,29 @@ A compose project name of its own does not separate that. To keep a development database you care about out of it, run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`: the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. -Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, -which it cannot through a worktree, so give it as a build argument, -`FLEXMEASURES_VERSION`, for each service the overlay builds. +Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, which it cannot through a worktree. +Give it as the build argument `FLEXMEASURES_VERSION` to each service built from FlexMeasures' Dockerfile, in one more overlay: + +```yaml +x-version: &version + build: + context: . + dockerfile: Dockerfile + args: + FLEXMEASURES_VERSION: "1.1.0" + +services: + server: + <<: *version + worker: + <<: *version + # and likewise worker-2, worker-3, automation-runner and each cem +``` + Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too; where they are not, an extra overlay can move them with compose's `!override`. -It runs FlexMeasures as its image installs it, with the plugins and this client installed into the image's virtualenv, -and expects a development server, so the server trusts the name the CEMs reach it by, `server`, alongside the loopback names. +It runs FlexMeasures from the flexmeasures checkout, which is mounted over the code the image installed, +with this client installed into the image's virtualenv, which is what FlexMeasures and its plugins run from. +The UI's third-party libraries are fetched into the installed package when the image is built, so a UI served from the mounted checkout may lack them; the API does not need them. +The server trusts the name the CEMs reach it by, `server`, alongside the loopback names a development server trusts by default. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index 07a48049..9304b049 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -4,7 +4,7 @@ # such as the community simulations that steer several homes at once. # # It is an overlay on FlexMeasures' own docker-compose.yml, -# and runs the CEMs from this checkout. +# and runs FlexMeasures from the flexmeasures checkout and the CEMs from this one, both mounted. # With checkouts named flexmeasures and flexmeasures-client side by side, # run it from the flexmeasures folder, which holds the Dockerfile: # @@ -88,7 +88,7 @@ x-worker: &worker python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt # The checkout may be mounted without git history to read its version from, as a git worktree is, # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. - SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" # Drop any lingering registration for the ingestion queue before this worker starts. # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. @@ -135,7 +135,7 @@ services: python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt # The checkout may be mounted without git history to read its version from, as a git worktree is, # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. - SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" flexmeasures db upgrade if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then flexmeasures add toy-account --name 'Docker Toy Account' From f0c4d4e67a60d4a5ecfe8ad9ea51dcc7c05c11d7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:21 +0100 Subject: [PATCH 54/87] fix(FRBC): a state of charge is known as of the handler's clock Every other value FRBCSimple records says when it became known; the state of charge did not, so FlexMeasures took the moment the request reached it. In the co-simulation gate that recorded readings of 2022 as known in 2026, 1436 days after they were taken. It is now posted with the handler's time as its prior, which in a co-simulation is the simulated time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 7 ++++- tests/s2/test_clock.py | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ccd43d1f..908a314b 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -387,14 +387,19 @@ async def _soc_duration(self) -> timedelta: async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): + # Read by this handler's clock, and known as of then, which in a co-simulation is the simulated time: + # without a prior, FlexMeasures took the moment the request reached it, + # and a simulated reading was recorded as known years after it was taken. + now = self.now() await self._fm_client.post_measurements( self._soc_sensor_id, - start=self.now(), + start=now, # Scaled like every other fill level this handler records, or the level and the # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, duration=await self._soc_duration(), + prior=now, ) # The state of charge this status reports is what a schedule starts from, so this is diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py index 946bae72..dd3a4361 100644 --- a/tests/s2/test_clock.py +++ b/tests/s2/test_clock.py @@ -218,3 +218,32 @@ def test_a_tunes_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time( (start + timedelta(minutes=30)).astimezone().replace(tzinfo=None) ) assert not tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_known_as_of_the_simulated_time(): + """Without a prior, FlexMeasures stamped a simulated reading with the moment the request arrived, years later.""" + from unittest.mock import AsyncMock + + from s2python.frbc import FRBCStorageStatus + + from flexmeasures_client.client import FlexMeasuresClient + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + from flexmeasures_client.s2.utils import get_unique_id + + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": "PT0M"}) + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + ) + cem = CEM(fm_client=fm, clock=SimulatedClock(SIMULATED)) + cem.register_control_type(frbc) + await frbc.send_storage_status( + FRBCStorageStatus(message_id=get_unique_id(), present_fill_level=5.0) + ) + posted = fm.post_measurements.await_args.kwargs + assert posted["start"] == SIMULATED + assert posted["prior"] == SIMULATED From 2b5d48159d8bf71f73c6712399fdc598c58dd85b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:40 +0100 Subject: [PATCH 55/87] docs(changelog): the state of charge on the clock Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d0e74592..057fa2ae 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -25,7 +25,7 @@ Added * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] -* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers and the moment a state of charge is known as of included [`#264 `__, `#268 `__] * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] From 00d3676be5e40205942bcb6a7462c019eeca0270 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:21:33 +0100 Subject: [PATCH 56/87] feat(FRBC): plan one schedule at a time, for the newest request Package D4 of the plan to finish #188. Each storage status is a planning request, and one arriving while a schedule was being planned started a second solve beside it, whose answer replaced the first's. A resource manager reporting faster than FlexMeasures plans kept several solves running on the server for nothing. FRBCSimple now plans one schedule at a time: a request waits for the one in flight and, when its turn comes, is dropped where a newer one arrived meanwhile, as a system description's plan already was where a status overtook it. A burst of statuses costs at most the solve in flight and one more, for the newest. No clock is involved, so a co-simulation's retrigger round, which shares a simulated time with the round before it, is planned as before, and the newest request is never left without a plan. The plan's revision 1 proposed a minimum replanning interval instead; its review showed that a deferred plan could strand the last status of a session, and that on a simulated clock a retrigger would always fall inside the interval. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 24 +++++++- tests/s2/test_frbc_simple_planning.py | 56 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ccd43d1f..7e359c74 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -263,6 +263,7 @@ def __init__( self._power_sensor_takes_consumption_as_positive = None self._power_recorded_at = {} self._soc_sensor_records_instants: bool | None = None + self._planning = asyncio.Lock() if hard_bound_factor < 1: # Below one, the hard upper bound sits inside the declared range, # and FlexMeasures clips the band into the hard bounds, @@ -849,12 +850,33 @@ def _warn_where_a_fallback_scheduler_answered( async def trigger_schedule( self, system_description_id: str, request_number: int | None = None ): - """Translates S2 System Description into FM API calls. + """Plan, one schedule at a time, and only for the newest request once it is this one's turn. + + Each storage status is a planning request, and one arriving while a schedule is being planned + used to start a second solve beside it, whose answer replaced the first's. + A resource manager reporting faster than FlexMeasures plans then kept several solves running for nothing. + Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile, + so a burst of statuses costs at most the solve in flight and one more, for the newest. :param system_description_id: system description to base the schedule on :param request_number: which planning request this schedule answers, which is checked once more before the instructions go out """ + async with self._planning: + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not planning for request %s: a newer one arrived while the schedule before it was being planned.", + request_number, + ) + return + await self._plan(system_description_id, request_number) + + async def _plan( + self, system_description_id: str, request_number: int | None = None + ): + """Translates S2 System Description into FM API calls.""" system_description: FRBCSystemDescription = self._system_description_history[ system_description_id diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 046c4fb2..9628410f 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -578,3 +578,59 @@ async def recording(*args, **kwargs): newest = max(n for n, _ in batches) assert sent[newest] > 0 assert all(count == 0 for n, count in batches if n != newest) + + +@pytest.mark.asyncio +async def test_a_burst_of_statuses_costs_the_solve_in_flight_and_one_more(): + """Each status used to start a solve beside the one in flight, all but the last for nothing.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + running = 0 + most = 0 + + async def solve(**kwargs): + nonlocal running, most + running += 1 + most = max(most, running) + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + running -= 1 + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + for _ in range(4): + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert most == 1 + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_overtaken_while_it_waits_for_the_solve_in_flight_is_dropped(): + """It was the newest when it began to wait, and is not when its turn comes.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # waits for the solve in flight + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # overtakes the one waiting + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 From 401599bd160049693c1505a9f66cda911ef18879 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:22:21 +0100 Subject: [PATCH 57/87] docs(changelog): one schedule at a time Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b75342d6..6247fa97 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -35,6 +35,7 @@ Added Changed ------- +* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile. Each status used to start a solve of its own beside the one in flight [`#269 `__] * An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] From 29845ea71969ba0300b4a136960c6f24ea61773d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:37:19 +0100 Subject: [PATCH 58/87] fix(FRBC): a request out of date makes no further attempt, so the newest waits less Review found that a request overtaken while its schedule was being asked for kept the planning lock through every retry of a failed attempt, up to five attempts with a wait between each, and that the schedule they finally got was thrown away; only then was the newest request planned, where before this PR it would have started at once. Between attempts the request now checks whether a newer one has arrived, and stands down if so. The docstring, the PR's cost bound and the changelog line now say what holds: a burst costs at most the attempt in flight and one plan more, for the newest, which starts after that attempt rather than beside it. The docstring breaks after punctuation, and _plan documents its parameters again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 48 +++++++++++++----- tests/s2/test_frbc_simple_planning.py | 50 +++++++++++++++++++ 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 6247fa97..cce3da96 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -35,7 +35,7 @@ Added Changed ------- -* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile. Each status used to start a solve of its own beside the one in flight [`#269 `__] +* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile, and a request out of date makes no further attempt after a failed one. Each status used to start a solve of its own beside the one in flight, so the newest is now planned after the attempt in flight rather than beside it [`#269 `__] * An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index dde08a0c..ffa491a6 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -770,8 +770,12 @@ def _is_worth_asking_again(self, failure: Exception) -> bool: return any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS) async def _ask_for_a_schedule( - self, start: datetime, flex_context: dict, flex_model: dict - ) -> dict | list[dict]: + self, + start: datetime, + flex_context: dict, + flex_model: dict, + request_number: int | None = None, + ) -> dict | list[dict] | None: """Ask FlexMeasures for a schedule, and ask again where the attempt is worth making again. One failed scheduling job otherwise leaves the storage without an instruction for its window. @@ -781,7 +785,9 @@ async def _ask_for_a_schedule( a worker crashed inside its solver, and the resource manager it belonged to polled until its round timed out, every round, for the rest of the run. - :param start: the window the schedule is asked for, which every attempt asks for again. + :param start: the window the schedule is asked for, which every attempt asks for again. + :param request_number: the planning request this answers; where a newer one arrives between attempts, + no further attempt is made and None is returned, so that the newer one is planned sooner. :raises JobFailedError: where the last attempt worth making failed inside the server. :raises ValueError: where the trigger itself was refused. """ @@ -823,6 +829,14 @@ async def _ask_for_a_schedule( failure, ) await asyncio.sleep(self._schedule_retry_interval.total_seconds()) + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not asking again for planning request %s: a newer one arrived, and is planned instead.", + request_number, + ) + return None def _warn_where_a_fallback_scheduler_answered( self, schedule: dict | list[dict], start: datetime @@ -857,15 +871,17 @@ async def trigger_schedule( ): """Plan, one schedule at a time, and only for the newest request once it is this one's turn. - Each storage status is a planning request, and one arriving while a schedule is being planned - used to start a second solve beside it, whose answer replaced the first's. - A resource manager reporting faster than FlexMeasures plans then kept several solves running for nothing. - Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile, - so a burst of statuses costs at most the solve in flight and one more, for the newest. + Each storage status is a planning request. + One arriving while a schedule was being planned used to start a second solve beside it, whose answer replaced the first's, + so a resource manager reporting faster than FlexMeasures plans kept several solves running for nothing. + Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile; + the one in flight, once out of date, makes no further attempt after a failed one. + So a burst of statuses costs at most the attempt in flight and one plan more, for the newest, + which starts once that attempt is over rather than beside it. :param system_description_id: system description to base the schedule on - :param request_number: which planning request this schedule answers, which is - checked once more before the instructions go out + :param request_number: which planning request this schedule answers, which is checked when its turn comes, + between attempts, and before the schedule is translated and sent """ async with self._planning: if request_number is not None and not self._is_latest_request( @@ -881,7 +897,11 @@ async def trigger_schedule( async def _plan( self, system_description_id: str, request_number: int | None = None ): - """Translates S2 System Description into FM API calls.""" + """Translates S2 System Description into FM API calls. + + :param system_description_id: system description to base the schedule on + :param request_number: which planning request this schedule answers + """ system_description: FRBCSystemDescription = self._system_description_history[ system_description_id @@ -1045,7 +1065,11 @@ async def _plan( if self._relax_site_capacity is not None: flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity - schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) + schedule = await self._ask_for_a_schedule( + start, flex_context, flex_model, request_number=request_number + ) + if schedule is None: + return self._warn_where_a_fallback_scheduler_answered(schedule, start) # send_instruction_batch checks this once more, but translating a schedule follows every step of it, diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 9628410f..428c76ee 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -634,3 +634,53 @@ async def solve(**kwargs): release.set() await settle(frbc) assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_out_of_date_makes_no_further_attempt_while_the_newest_waits( + caplog, +): + """It kept the newest waiting through every one of its retries, and the schedule they got was thrown away.""" + import logging + from datetime import timedelta as td + + caplog.set_level(logging.INFO) + + from flexmeasures_client.exceptions import JobFailedError + + frbc = frbc_simple() + frbc._schedule_attempts = 5 + frbc._schedule_retry_interval = td(milliseconds=10) + frbc._is_worth_asking_again = lambda failure: True + first_attempt = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await first_attempt.wait() + raise JobFailedError("the worker crashed") + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + batches: list[int] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + batches.append(kwargs["request_number"]) + return await send(*args, **kwargs) + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # newer, while the first attempt is out + await asyncio.sleep(0.05) + first_attempt.set() + await settle(frbc) + # The failed attempt, then the newest. + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + assert "Not asking again for planning request" in caplog.text + assert not [ + r for r in caplog.records if r.levelname == "ERROR" + ] # nothing translated for it + assert batches == [2] # nothing sent for the request that stood down From 5564983963fa2644905ec5806f6c1d3a1636677a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:37:50 +0100 Subject: [PATCH 59/87] docs(examples): the instance settings a co-simulation needs Found by running a steered two-home co-simulation through the stack: a steering round replans a window at the same simulated moment, so without FLEXMEASURES_ALLOW_DATA_OVERWRITE the scheduler could not save the second plan and the homes got no instructions for their window; and every home triggers a schedule each round from one account, far above the default trigger limit, so every trigger past it waited minutes. Neither is read from the environment, so the README says what to put in the instance config, with the solver time limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 13b364dd..6e95fa06 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -18,6 +18,20 @@ with two switches on: resource manager retries. Its schedules start from the simulated now and cover 24 hours. The site's prices are for whoever runs the simulation to post. +FlexMeasures reads three settings a co-simulation needs from its instance config +(`flexmeasures-instance/flexmeasures.cfg` in the flexmeasures checkout), not from the environment: + +```python +# A steering round replans a window at the same simulated moment, and that plan replaces the one before. +# Without this, the scheduler cannot save the second plan, and the home gets no instructions for its window. +FLEXMEASURES_ALLOW_DATA_OVERWRITE = True +# Every home triggers a schedule each round, all from one account, far above the default of 10 per 5 minutes, +# and every trigger past the limit waits minutes before it is retried. +FLEXMEASURES_API_TRIGGER_RATE_LIMIT = "1000 per minute" +# A bound on each solve, since a tail of degenerate problems otherwise proves optimality past the job timeout. +FLEXMEASURES_LP_SOLVER_OPTIONS = {"time_limit": "15"} +``` + A co-simulation reads the sites by sensor name: `price`, `production price`, `power` (the schedule), `measured-power` (what the home measured) and `state of charge`. From 309e2834208ef22783bb9912291fe04645aec197 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:52:20 +0100 Subject: [PATCH 60/87] fix(FRBC): the declared fill-level range reaches the server, as a band, with or without sensors Package P2.1 of the plan to finish #188, for review findings B0, B11 and B12. B0: since #245 the declared fill-level range reached FlexMeasures only through the band sensors, and only where a fill-level target profile had been recorded on them. Without the sensors, or past the profile's end, a storage was planned with no band at all, between zero and half again above its range: a buffer declaring [20, 80] could be planned to empty on a cheap night. The band now goes out with every request, as soc-minima and soc-maxima time segments over the whole window: the declared range for every step, the latest target profile, held inside the range, where it covers a step, clipped at zero, which FlexMeasures requires, and in the unit FlexMeasures takes. relax-soc-constraints is always set, since the band always goes out. The sensors stay, for recording and the graphs. Checked against FlexMeasures 1.1.0rc3, which planned against it. B11: FlexMeasures refuses a negative soc-min, so widening the lower bound to take in a storage reported below zero got it no schedule. Such a storage is now planned from zero, with a warning. B12: a negative usage forecast is a gain FlexMeasures cannot relax, and past the hard ceiling it made every schedule for the window infeasible. It is now recorded as no usage, with a warning that counts the steps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 128 +++++++++--- tests/s2/test_frbc_charging_efficiency.py | 9 +- tests/s2/test_frbc_soft_band.py | 183 ++++++++++++++++-- 4 files changed, 265 insertions(+), 57 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index cce3da96..8f6448f9 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -44,7 +44,7 @@ Changed * The ``operation-modes`` field is asked for only of a FlexMeasures at v1.0.0 or newer, which is the release that added it. An older server is asked for a schedule without the power bands, and told so in the log, since a flex model naming a field the server does not know is refused outright and leaves the resource manager with no schedule at all [`#248 `__] * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] -* ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, rather than as the hard ``soc-min`` and ``soc-max``, which become wide bounds at zero and half again above the range, each widened to take in a state of charge beyond them. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band [`#245 `__] +* ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and the band clipped at zero, which FlexMeasures requires. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ffa491a6..33cd43d5 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -506,6 +506,68 @@ def _latest(history: dict): """The message a history holds that arrived last, or None while it holds none.""" return list(history.values())[-1] if history else None + def _fill_level_band( + self, + start: datetime, + system_description: FRBCSystemDescription, + soc_unit: str, + to_soc_unit: float, + ) -> tuple[list[dict], list[dict]]: + """The fill levels to keep the storage between over the window planned, as time segments FlexMeasures takes. + + The declared fill-level range applies to every step, and the latest fill-level target profile, + held inside that range, applies where it covers a step. + The range used to reach FlexMeasures only through the band sensors, and only where a target profile was recorded on them, + so a storage without the sensors, or beyond the profile's end, was planned with no band at all, + down to empty and half again above its range. + Clipped at zero, since FlexMeasures takes no negative minima. + """ + steps = int( + pd.Timedelta(self._schedule_duration) / pd.Timedelta(self._resolution) + ) + index = pd.date_range(start, periods=steps, freq=self._resolution) + range_bottom, range_top = get_soc_min_max( + system_description, self._fill_level_scale + ) + minima = pd.Series(range_bottom, index=index, dtype=float) + maxima = pd.Series(range_top, index=index, dtype=float) + profile = self._latest(self._fill_level_target_profile_history) + if profile is not None: + profile_minima, profile_maxima = translate_fill_level_target_profile( + fill_level_target_profile=profile, + resolution=self._resolution, + fill_level_scale=self._fill_level_scale, + ) + profile_minima, profile_maxima, _ = clip_fill_level_target_profile( + profile_minima, profile_maxima, range_bottom, range_top + ) + minima.update(profile_minima) + maxima.update(profile_maxima) + return ( + self._as_segments(minima.clip(lower=0) * to_soc_unit, soc_unit), + self._as_segments(maxima.clip(lower=0) * to_soc_unit, soc_unit), + ) + + def _as_segments(self, series: pd.Series, unit: str) -> list[dict]: + """A series on this handler's resolution as FlexMeasures' time segments, one per run of equal values.""" + segments: list[dict] = [] + for moment, value in series.items(): + end = moment + pd.Timedelta(self._resolution) + if segments and segments[-1]["_value"] == value: + segments[-1]["end"] = end.isoformat() + continue + segments.append( + { + "start": moment.isoformat(), + "end": end.isoformat(), + "value": f"{value} {unit}", + "_value": value, + } + ) + for segment in segments: + del segment["_value"] + return segments + def _soc_unit(self) -> tuple[str, float]: """The unit a state of charge goes to FlexMeasures in, and what a fill level is multiplied by to be in it.""" if self.energy_unit in ("kWh", "MWh"): @@ -945,13 +1007,9 @@ async def _plan( # The declared range is where a resource manager wants its storage kept, not where its storage can be: # S2 calls it the range in which the fill level should remain, # and has a resource manager free to ignore instructions while the fill level is outside it. - # So the range goes out as a band whose breach FlexMeasures prices, + # So the range goes out as a band whose breach FlexMeasures prices (see _fill_level_band), # rather than as the hard soc-min and soc-max, which made states a storage really reaches infeasible, - # and an infeasible job is answered by the fallback scheduler, which ignores the band altogether. - # Hard bounds are still sent, but wide ones: - # a fill-level target profile need not cover the whole schedule, - # and over the part it does not cover there is nothing to price a breach of, - # so nothing else keeps a plan within reach of the storage it is planned for. + # and hard bounds are still sent, but wide ones. _, range_top = get_soc_min_max(system_description, self._fill_level_scale) # Zero below, since a stock cannot be negative and FlexMeasures accepts no lower floor. soc_min = 0.0 @@ -960,7 +1018,14 @@ async def _plan( # so the bounds widen to take it in. # A storage outside its declared range is not an error: # a resource manager is free to report one, and a schedule still has to be planned from where it says its storage is. - soc_min = min(soc_min, soc_at_start) + # Upward only, since FlexMeasures takes no negative soc-min: + # a storage reported below zero is planned from zero, and said so, rather than refused with no schedule. + if soc_at_start < 0: + self._logger.warning( + "The storage is reported at %s, below zero, which FlexMeasures cannot plan from; planning from zero instead.", + soc_at_start, + ) + soc_at_start = 0.0 soc_max = max(soc_max, soc_at_start) # Where this schedule starts, which is both what it is asked for, @@ -979,26 +1044,13 @@ async def _plan( "soc-min": soc_min * to_soc_unit, "soc-max": soc_max * to_soc_unit, } - # What the resource manager declared about its storage, each entry only where a sensor holds it: - # an entry naming a sensor that was never configured has the scheduler plan against nothing at all. - # Both band sensors or neither, as when the band is recorded: - # one side alone is an open-ended push in one direction, not the band the resource manager declared. - has_soc_band = ( - self._soc_minima_sensor_id is not None - and self._soc_maxima_sensor_id is not None + # The band the resource manager declared, sent with the request itself rather than named by sensor, + # so that it is there with or without the sensors it is also recorded on. + soc_minima, soc_maxima = self._fill_level_band( + start, system_description, soc_unit, to_soc_unit ) - if has_soc_band: - flex_model["soc-minima"] = {"sensor": self._soc_minima_sensor_id} - flex_model["soc-maxima"] = {"sensor": self._soc_maxima_sensor_id} - elif ( - self._soc_minima_sensor_id is not None - or self._soc_maxima_sensor_id is not None - ): - self._logger.info( - "Not asking for a schedule against a fill-level band: a band needs both a soc-minima and a soc-maxima sensor, and this handler was given %s and %s.", - self._soc_minima_sensor_id, - self._soc_maxima_sensor_id, - ) + flex_model["soc-minima"] = soc_minima + flex_model["soc-maxima"] = soc_maxima if self._usage_forecast_sensor_id is not None: flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] if await self._record_storage_efficiency(start, fill_level_at_start): @@ -1056,12 +1108,11 @@ async def _plan( "consumption-price": {"sensor": self._price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", } - if has_soc_band: - # Without this, the minima and maxima above are hard bounds again, - # and the band is back to making reachable states infeasible. - # The breach price is left to FlexMeasures' own default, - # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. - flex_context["relax-soc-constraints"] = True + # The band always goes out, and without this its minima and maxima are hard bounds, + # which makes states a storage really reaches infeasible. + # The breach price is left to FlexMeasures' own default, + # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. + flex_context["relax-soc-constraints"] = True if self._relax_site_capacity is not None: flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity @@ -1184,6 +1235,19 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): strategy="mean", fill_level_scale=self._fill_level_scale, ) + # A negative usage is a gain the storage is forced to take, which FlexMeasures cannot relax: + # past the hard ceiling it makes every schedule for the window infeasible, so the storage gets no instruction at all. + # S2 allows the sign, but no storage in reach of this handler fills by being drawn from, + # so a negative value is taken as an artefact of the forecast, counted, and recorded as no usage. + negative = translated < 0 + if negative.any(): + self._logger.warning( + "An FRBC.UsageForecast expects negative usage on %s of %s steps, down to %s; recorded as no usage there, since FlexMeasures would plan it as a gain the storage cannot refuse.", + int(negative.sum()), + len(translated), + float(translated.min()), + ) + translated = translated.clip(lower=0) await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 2a899d82..4a74057f 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -419,16 +419,17 @@ async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( @pytest.mark.asyncio -async def test_a_lower_bound_below_zero_goes_out_in_the_same_unit(): - """Reported below its range, a storage widens the lower bound to take it in, which then needs the conversion too.""" +async def test_a_storage_reported_below_zero_is_planned_from_zero(caplog): + """FlexMeasures refuses a negative soc-min, so widening the bound to take the storage in got it no schedule at all.""" system_description = _heater() frbc = _frbc(system_description, energy_unit="J") frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( message_id=get_unique_id(), present_fill_level=-3.6e6 ) flex_model = await _asked_for(frbc, system_description) - assert flex_model["soc-at-start"] == pytest.approx(-1.0) - assert flex_model["soc-min"] == pytest.approx(-1.0) + assert flex_model["soc-at-start"] == 0.0 + assert flex_model["soc-min"] == 0.0 + assert "below zero" in caplog.text @pytest.mark.asyncio diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 64055d78..24e17c88 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -88,6 +88,23 @@ def system_description() -> FRBCSystemDescription: ) +def _system_description_with_range(bottom: float, top: float) -> FRBCSystemDescription: + """The fixture's description, declaring another fill-level range.""" + sd = system_description.__wrapped__() + return sd.model_copy( + update={ + "message_id": get_unique_id(), + "storage": sd.storage.model_copy( + update={ + "fill_level_range": NumberRange( + start_of_range=bottom, end_of_range=top + ) + } + ), + } + ) + + def _frbc(system_description, *, with_sensors: bool = True, **kwargs) -> FRBCSimple: """A handler that has had a system description and a storage status, ready to plan.""" sensors = ( @@ -139,13 +156,22 @@ async def _asked_for(frbc: FRBCSimple, system_description) -> tuple[dict, dict]: async def test_the_declared_band_is_what_a_schedule_is_asked_against( system_description, ): - """Recording the band changed nothing until the flex model named the sensors holding it.""" + """The declared range goes out with the request, over the whole window, as a band whose breach is priced.""" frbc = _frbc(system_description) flex_model, flex_context = await _asked_for(frbc, system_description) - assert flex_model["soc-minima"] == {"sensor": SOC_MINIMA_SENSOR_ID} - assert flex_model["soc-maxima"] == {"sensor": SOC_MAXIMA_SENSOR_ID} + assert [segment["value"] for segment in flex_model["soc-minima"]] == [ + f"{RANGE_BOTTOM} kWh" + ] + assert [segment["value"] for segment in flex_model["soc-maxima"]] == [ + f"{RANGE_TOP} kWh" + ] + (whole,) = flex_model["soc-minima"] + assert ( + pd.Timestamp(whole["end"]) - pd.Timestamp(whole["start"]) + == frbc._schedule_duration + ) # A minimum or maximum FlexMeasures may not breach is a hard bound by another name, # and the whole point of the band is that it is soft. assert flex_context["relax-soc-constraints"] is True @@ -254,24 +280,21 @@ async def test_how_far_the_hard_bounds_sit_from_the_range_is_the_deployments_to_ @pytest.mark.asyncio -async def test_one_side_of_a_band_is_not_a_band(system_description): - """One side alone is an open-ended push, not what the resource manager declared. - - A band needs both sides, which is also the rule the recording half goes by: - a fill-level target profile is recorded only where both sensors are there to hold it. - """ - frbc = _frbc( - system_description, - with_sensors=False, - soc_minima_sensor_id=SOC_MINIMA_SENSOR_ID, - ) +async def test_the_band_goes_out_without_the_sensors_it_is_recorded_on( + system_description, +): + """It reached FlexMeasures only through those sensors, so without them a storage was planned with no band at all.""" + frbc = _frbc(system_description, with_sensors=False) flex_model, flex_context = await _asked_for(frbc, system_description) - assert "soc-minima" not in flex_model - assert "soc-maxima" not in flex_model - # And nothing is relaxed either, since there is no band for a breach price to apply to. - assert "relax-soc-constraints" not in flex_context + assert [segment["value"] for segment in flex_model["soc-minima"]] == [ + f"{RANGE_BOTTOM} kWh" + ] + assert [segment["value"] for segment in flex_model["soc-maxima"]] == [ + f"{RANGE_TOP} kWh" + ] + assert flex_context["relax-soc-constraints"] is True def test_a_hard_bound_inside_the_declared_range_is_refused(): @@ -299,9 +322,8 @@ async def test_nothing_is_asked_against_a_sensor_that_was_never_configured( flex_model, flex_context = await _asked_for(frbc, system_description) - for absent in ("soc-minima", "soc-maxima", "soc-usage", "storage-efficiency"): + for absent in ("soc-usage", "storage-efficiency"): assert absent not in flex_model - assert "relax-soc-constraints" not in flex_context # The hard bounds, the start state and the unit are not a resource manager's to declare, # so they are asked for whatever the deployment configured. assert flex_model["soc-min"] == 0.0 @@ -309,6 +331,58 @@ async def test_nothing_is_asked_against_a_sensor_that_was_never_configured( assert flex_model["soc-at-start"] == pytest.approx(FILL_LEVEL_AT_START) +@pytest.mark.asyncio +async def test_a_target_profile_applies_where_it_covers_the_window_and_the_range_elsewhere( + system_description, +): + """Beyond the profile's end the band used to be whatever the sensors last held, or nothing.""" + from datetime import timedelta + + from s2python.common import Duration + from s2python.frbc import ( + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, + ) + + frbc = _frbc(system_description) + start = frbc.schedule_start(system_description.valid_from) + profile = FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration.from_timedelta(timedelta(hours=1)), + fill_level_range=NumberRange(start_of_range=50, end_of_range=60), + ) + ], + ) + frbc._fill_level_target_profile_history[str(profile.message_id)] = profile + + flex_model, _ = await _asked_for(frbc, system_description) + + minima = flex_model["soc-minima"] + assert [segment["value"] for segment in minima] == [ + "50.0 kWh", + f"{RANGE_BOTTOM} kWh", + ] + assert pd.Timestamp(minima[0]["end"]) - pd.Timestamp( + minima[0]["start"] + ) == timedelta(hours=1) + assert [segment["value"] for segment in flex_model["soc-maxima"]] == [ + "60.0 kWh", + f"{RANGE_TOP} kWh", + ] + + +@pytest.mark.asyncio +async def test_a_band_below_zero_is_clipped_at_zero(): + """FlexMeasures takes no negative minima, and refused the whole request over one.""" + sd = _system_description_with_range(-10.0, 80.0) + frbc = _frbc(sd) + flex_model, _ = await _asked_for(frbc, sd) + assert [segment["value"] for segment in flex_model["soc-minima"]] == ["0.0 kWh"] + + @pytest.mark.asyncio async def test_a_schedule_is_read_back_against_the_bounds_it_was_planned_against( system_description, @@ -456,3 +530,72 @@ def test_bounds_that_run_backwards_are_refused(): initial_fill_level=RANGE_BOTTOM, fill_level_bounds=(RANGE_TOP, RANGE_BOTTOM), ) + + +@pytest.mark.asyncio +async def test_a_negative_usage_forecast_is_recorded_as_no_usage_and_said_so( + system_description, caplog +): + """A forced gain FlexMeasures cannot relax made every schedule for the window infeasible.""" + from datetime import timedelta + + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + frbc = _frbc(system_description) + forecast = FRBCUsageForecast( + message_id=get_unique_id(), + start_time="2024-01-01T00:00:00+00:00", + elements=[ + FRBCUsageForecastElement( + duration=Duration.from_timedelta(timedelta(minutes=15)), + usage_rate_expected=rate, + ) + for rate in (0.01, -0.02, 0.0, -0.005) + ], + ) + await frbc.send_usage_forecast(forecast) + posted = frbc._fm_client.post_sensor_data.await_args.kwargs + assert min(posted["values"]) == 0.0 + assert posted["values"][0] == pytest.approx(0.01) + assert "negative usage on 2 of 4 steps" in caplog.text + + +@pytest.mark.asyncio +async def test_a_target_profile_beyond_the_declared_range_is_held_inside_it( + system_description, +): + from datetime import timedelta + + from s2python.common import Duration + from s2python.frbc import ( + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, + ) + + frbc = _frbc(system_description) + start = frbc.schedule_start(system_description.valid_from) + profile = FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration.from_timedelta(timedelta(hours=1)), + fill_level_range=NumberRange(start_of_range=10, end_of_range=90), + ) + ], + ) + frbc._fill_level_target_profile_history[str(profile.message_id)] = profile + flex_model, _ = await _asked_for(frbc, system_description) + assert [s["value"] for s in flex_model["soc-minima"]] == [f"{RANGE_BOTTOM} kWh"] + assert [s["value"] for s in flex_model["soc-maxima"]] == [f"{RANGE_TOP} kWh"] + + +@pytest.mark.asyncio +async def test_the_band_goes_out_in_the_unit_flexmeasures_takes(system_description): + frbc = _frbc(system_description, energy_unit="Wh") + flex_model, _ = await _asked_for(frbc, system_description) + assert flex_model["soc-unit"] == "kWh" + assert [s["value"] for s in flex_model["soc-minima"]] == [ + f"{RANGE_BOTTOM / 1000} kWh" + ] From c34155a19b52916a18d75a99d7f13b5596753c06 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:00:30 +0100 Subject: [PATCH 61/87] fix(FRBC): the flex model offers only what the actuator may use, and bands only where they say something Package P2.2 of the plan to finish #188, for review findings B4 and B14. B1, which the plan also put here, was fixed in #256. B4: the capacities and the operation-modes bands were built from every mode an actuator states, abnormal_condition_only included, which nothing read. So an emergency boost reserved for abnormal conditions became a band the scheduler planned in, and the instruction answering it, not marked abnormal, is one a conforming resource manager refuses. Such modes are now left out of the capacities and the bands, and out of the modes the translation instructs into. An actuator with no other mode is not planned for, with a warning, rather than solved for and then failing in the translation. B14: the bands went out wherever there was more than one of them, so an idle mode at zero beside a mode running from zero up, whose union is exactly what the two capacities allow, still turned the problem into a MILP, a binary per band per step. They now go out only where their union leaves a gap the capacities do not, as an actuator running at 400 W or not at all does. Not done, and said here: the capacities still take the widest range of every element, whatever the fill level, since FlexMeasures takes no fill-level dependent power range; B4's second half stays a relaxation the scheduler may plan into at fill levels where only a lower element applies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 32 ++++- .../s2/control_types/FRBC/utils.py | 19 ++- tests/s2/test_frbc_operation_modes.py | 119 ++++++++++++++++-- 4 files changed, 155 insertions(+), 17 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index cce3da96..5c9f5e40 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -40,7 +40,7 @@ Changed * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] * ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes, which a schedule's start is lined up with [`#240 `__] -* ``FRBCSimple`` asks for a schedule within the power ranges the actuator declares: a ``consumption-capacity`` and a ``production-capacity`` from the widest range it states, and each distinct range as one of the ``operation-modes``, except where a single range runs through zero and so says no more than the two capacities do. The scheduler then plans only power values the actuator can run at, instead of a fractional power that a resource manager has to round into a full-on block, overshooting the capacity its site is held to. An actuator that cannot produce at all is given a production capacity of zero, where before its storage could be planned as if it were a battery feeding the grid [`#248 `__] +* ``FRBCSimple`` asks for a schedule within the power ranges the actuator declares: a ``consumption-capacity`` and a ``production-capacity`` from the widest range it states, and each distinct range as one of the ``operation-modes``, except where the ranges together cover exactly what the two capacities allow, without a gap, and so say nothing new while each costs the solver a binary per step. Modes S2 reserves for abnormal conditions are left out of both, and are never instructed into, since a resource manager refuses an instruction into one that is not marked abnormal; an actuator with no other mode is not planned for, with a warning. The scheduler then plans only power values the actuator can run at, instead of a fractional power that a resource manager has to round into a full-on block, overshooting the capacity its site is held to. An actuator that cannot produce at all is given a production capacity of zero, where before its storage could be planned as if it were a battery feeding the grid [`#248 `__, `#272 `__] * The ``operation-modes`` field is asked for only of a FlexMeasures at v1.0.0 or newer, which is the release that added it. An older server is asked for a schedule without the power bands, and told so in the log, since a flex model naming a field the server does not know is refused outright and leaves the resource manager with no schedule at all [`#248 `__] * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ffa491a6..dc584b25 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -35,6 +35,7 @@ element_for_fill_level, fm_schedule_to_instructions, get_soc_min_max, + normal_operation_modes, om_label, operation_mode_conversion_efficiency, ) @@ -75,6 +76,17 @@ RETRYABLE_TRIGGER_REFUSALS = ("No recent state-of-charge value found",) +def _merged(ranges: list[tuple[float, float]]) -> list[tuple[float, float]]: + """The union of closed ranges, as the fewest ranges that cover it, in order.""" + merged: list[tuple[float, float]] = [] + for lower, upper in sorted(ranges): + if merged and lower <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], upper)) + else: + merged.append((lower, upper)) + return merged + + def _operation_mode_power_bands( power_ranges: list[tuple[float, float]], power_unit: str ) -> list[dict[str, list[str]]]: @@ -926,6 +938,11 @@ async def _plan( # One actuator, as everywhere else in this handler. actuator = system_description.actuators[0] + if not normal_operation_modes(actuator): + self._logger.warning( + "Not asking for a schedule: the actuator states no operation mode it may be instructed into under normal conditions." + ) + return # Every power range the actuator states, over all of its operation modes and their elements. # An S2 power range runs from its lower bound to its upper one, and S2 fixes one sign convention, # where positive means consumption, @@ -937,7 +954,9 @@ async def _plan( min(power_range.start_of_range, power_range.end_of_range), max(power_range.start_of_range, power_range.end_of_range), ) - for operation_mode in actuator.operation_modes + # Only modes the actuator may be instructed into under normal conditions, + # since a band the scheduler plans in is a mode it may then be instructed into. + for operation_mode in normal_operation_modes(actuator) for element in operation_mode.elements for power_range in element.power_ranges ] @@ -1031,16 +1050,17 @@ async def _plan( operation_modes = _operation_mode_power_bands(power_ranges, S2_POWER_UNIT) # Only where the bands say more than the two capacities above already do. # The capacities say the actuator can run at any power from minus its production capacity to its consumption capacity, - # so a single band running through zero is exactly that and says nothing new. - # Any other single band does: an actuator that runs at 200 W or not at all rules out everything in between, + # so bands that together cover exactly that, without a gap, say nothing new, + # such as an idle mode at zero beside a mode running from zero up, and each band costs the solver a binary per step. + # Bands with a gap between them do: an actuator that runs at 200 W or not at all rules out everything in between, # and so does one that only ever feeds back. # And only to a server that has the field. # FlexMeasures added operation-modes in v1.0.0, # and a flex model naming a field the server does not know is refused outright, # which would leave the resource manager with no schedule at all. - says_more_than_the_capacities = len(operation_modes) > 1 or not any( - lower <= 0 <= upper for lower, upper in power_ranges - ) + says_more_than_the_capacities = _merged(power_ranges) != [ + (-production_capacity, consumption_capacity) + ] if says_more_than_the_capacities: if await self._fm_client.server_version_at_least("1.0.0"): flex_model["operation-modes"] = operation_modes diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index cec7fa08..ab721e9e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -205,6 +205,19 @@ def net_rate(mode_element, leakage_element, moment) -> float: return min(max(fill_level, lowest), highest) +def normal_operation_modes(actuator) -> list[FRBCOperationMode]: + """The operation modes an actuator may be instructed into under normal conditions. + + S2 marks a mode abnormal_condition_only where it is reserved for an abnormal condition, such as an emergency boost, + and a resource manager refuses an instruction into one that is not marked abnormal itself, + which every instruction a schedule is turned into is not. + So such a mode is neither planned in nor instructed into. + """ + return [ + mode for mode in actuator.operation_modes if not mode.abnormal_condition_only + ] + + def fm_schedule_to_instructions( schedule: pd.DataFrame | dict, system_description: FRBCSystemDescription, @@ -270,7 +283,11 @@ def fm_schedule_to_instructions( f"{len(system_description.actuators)} were provided" ) actuator = system_description.actuators[0] - operation_modes: list[FRBCOperationMode] = actuator.operation_modes + operation_modes: list[FRBCOperationMode] = normal_operation_modes(actuator) + if not operation_modes: + raise ValueError( + "The actuator states no operation mode it may be instructed into under normal conditions." + ) modes_by_id = {str(mode.id): mode for mode in operation_modes} # Held between the hard bounds the schedule was planned against, diff --git a/tests/s2/test_frbc_operation_modes.py b/tests/s2/test_frbc_operation_modes.py index 747f7ad0..66e17bd0 100644 --- a/tests/s2/test_frbc_operation_modes.py +++ b/tests/s2/test_frbc_operation_modes.py @@ -170,12 +170,11 @@ async def test_a_band_that_spans_zero_is_split_at_zero(): @pytest.mark.asyncio async def test_a_band_that_spans_zero_reaches_both_directions_from_zero(): """FlexMeasures reads the two ranges of one band as a single band through zero.""" - system_description = _system_description((0.0, 0.0), (-500.0, 1000.0)) - frbc = _frbc(system_description) - - flex_model = await _asked_for(frbc, system_description) + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import ( + _operation_mode_power_bands, + ) - assert flex_model["operation-modes"] == [ + assert _operation_mode_power_bands([(0.0, 0.0), (-500.0, 1000.0)], "W") == [ {"consumption-range": ["0 W", "0 W"]}, { "consumption-range": ["0.0 W", "1000.0 W"], @@ -316,12 +315,114 @@ async def test_the_bands_are_in_watts_whatever_the_handler_posts_power_in(): @pytest.mark.asyncio async def test_a_bound_of_zero_is_stated_as_zero(): """Flipping the sign of a range that reaches up to zero leaves a negative zero behind.""" + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import ( + _operation_mode_power_bands, + ) + + assert _operation_mode_power_bands([(0.0, 0.0), (-2000.0, 0.0)], "W")[1] == { + "production-range": ["0.0 W", "2000.0 W"] + } system_description = _system_description((0.0, 0.0), (-2000.0, 0.0)) - frbc = _frbc(system_description) + flex_model = await _asked_for(_frbc(system_description), system_description) + assert flex_model["production-capacity"] == "2000.0 W" + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "ranges, bands_sent", + [ + ( + ((0.0, 0.0), (0.0, 1500.0)), + False, + ), # idle beside a mode from zero up: nothing the capacities lack + (((0.0, 0.0), (-500.0, 1000.0)), False), + ( + ((0.0, 0.0), (0.0, 800.0), (800.0, 1500.0)), + False, + ), # touching ranges leave no gap + (((0.0, 0.0), (400.0, 1500.0)), True), # nothing between 0 and 400 W + (((400.0, 1500.0),), True), # and nothing at all below 400 W + (((0.0, 0.0), (-2000.0, -500.0)), True), + ], +) +async def test_bands_go_out_only_where_they_rule_out_what_the_capacities_allow( + ranges, bands_sent +): + """Each band costs the solver a binary per step, so bands that only repeat the capacities turned an LP into a MILP.""" + system_description = _system_description(*ranges) + flex_model = await _asked_for(_frbc(system_description), system_description) + assert ("operation-modes" in flex_model) is bands_sent + + +@pytest.mark.asyncio +async def test_a_mode_for_abnormal_conditions_only_is_neither_planned_in_nor_instructed_into(): + """An instruction into it is not marked abnormal, so a conforming resource manager refuses it.""" + system_description = _system_description((0.0, 0.0), (400.0, 1000.0), (0.0, 5000.0)) + actuator = system_description.actuators[0] + boost = actuator.operation_modes[2].model_copy( + update={"abnormal_condition_only": True} + ) + system_description = system_description.model_copy( + update={ + "actuators": [ + actuator.model_copy( + update={"operation_modes": [*actuator.operation_modes[:2], boost]} + ) + ] + } + ) + frbc = _frbc(system_description) flex_model = await _asked_for(frbc, system_description) + assert flex_model["consumption-capacity"] == "1000.0 W" + assert all( + band.get("consumption-range", ["", ""])[1] != "5000.0 W" + for band in flex_model["operation-modes"] + ) + from flexmeasures_client.s2.control_types.FRBC.utils import normal_operation_modes - assert flex_model["operation-modes"][1] == { - "production-range": ["0.0 W", "2000.0 W"] + assert str(boost.id) not in { + str(m.id) for m in normal_operation_modes(system_description.actuators[0]) } - assert flex_model["production-capacity"] == "2000.0 W" + + +def test_an_actuator_with_no_normal_mode_cannot_be_instructed(): + + from flexmeasures_client.s2.control_types.FRBC.utils import ( + fm_schedule_to_instructions, + ) + + system_description = _system_description((0.0, 1000.0)) + actuator = system_description.actuators[0] + only = actuator.operation_modes[0].model_copy( + update={"abnormal_condition_only": True} + ) + system_description = system_description.model_copy( + update={"actuators": [actuator.model_copy(update={"operation_modes": [only]})]} + ) + with pytest.raises(ValueError, match="no operation mode"): + fm_schedule_to_instructions( + { + "start": "2024-01-01T00:00:00+00:00", + "duration": "PT15M", + "values": [500.0], + }, + system_description, + initial_fill_level=50.0, + ) + + +@pytest.mark.asyncio +async def test_no_schedule_is_asked_for_an_actuator_with_no_normal_mode(caplog): + """A solve whose answer could only be instructed into modes the resource manager refuses is a solve for nothing.""" + system_description = _system_description((0.0, 1000.0)) + actuator = system_description.actuators[0] + only = actuator.operation_modes[0].model_copy( + update={"abnormal_condition_only": True} + ) + system_description = system_description.model_copy( + update={"actuators": [actuator.model_copy(update={"operation_modes": [only]})]} + ) + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + assert "under normal conditions" in caplog.text From 9a1ddf659db936665af3bac4676408747416685f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:05:45 +0100 Subject: [PATCH 62/87] fix(FRBC): what the review of the declared band found - The schedule was planned without negative usage but read back with it, so the instructions followed fill levels the plan never had. The read-back now takes the usage forecast as it was recorded, with negative rates as none. - A target profile is floored to steps on its own clock and the window on the description's, so in a time zone half an hour off the other the two never shared an instant, and matching them by label dropped the whole profile without a word. The profile is now taken at each step by the instant it starts at, where the profile covers it. - A storage reported below zero was planned from zero but read back from below it; it is now read back from zero too. - relax-soc-constraints went to every server, and FlexMeasures before v0.26.0 refuses it, with the whole request. Against such a server the band is left out, and the log says so. - The changelog says what holds: FlexMeasures takes no negative minima, so the band is clipped at zero; a newer target profile replaces the one before it, as S2 means it to; and the band is left out for a server that cannot relax it. A comment that still said the band is read from sensors is corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 61 ++++++++++-- tests/s2/test_frbc_soft_band.py | 94 +++++++++++++++++++ 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 8f6448f9..ed9be409 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -44,7 +44,7 @@ Changed * The ``operation-modes`` field is asked for only of a FlexMeasures at v1.0.0 or newer, which is the release that added it. An older server is asked for a schedule without the power bands, and told so in the log, since a flex model naming a field the server does not know is refused outright and leaves the resource manager with no schedule at all [`#248 `__] * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] -* ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and the band clipped at zero, which FlexMeasures requires. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] +* ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to, and the declared range applies again beyond its end. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, and the log says so. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 33cd43d5..d86e7f4e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -75,6 +75,26 @@ RETRYABLE_TRIGGER_REFUSALS = ("No recent state-of-charge value found",) +def _without_negative_usage(usage_forecast): + """A usage forecast with each negative expected rate taken as no usage, as one is recorded and planned against.""" + if usage_forecast is None or all( + element.usage_rate_expected >= 0 for element in usage_forecast.elements + ): + return usage_forecast + return usage_forecast.model_copy( + update={ + "elements": [ + element.model_copy( + update={ + "usage_rate_expected": max(0.0, element.usage_rate_expected) + } + ) + for element in usage_forecast.elements + ] + } + ) + + def _operation_mode_power_bands( power_ranges: list[tuple[float, float]], power_unit: str ) -> list[dict[str, list[str]]]: @@ -541,13 +561,26 @@ def _fill_level_band( profile_minima, profile_maxima, _ = clip_fill_level_target_profile( profile_minima, profile_maxima, range_bottom, range_top ) - minima.update(profile_minima) - maxima.update(profile_maxima) + minima.update(self._on_the_window(profile_minima, index)) + maxima.update(self._on_the_window(profile_maxima, index)) return ( self._as_segments(minima.clip(lower=0) * to_soc_unit, soc_unit), self._as_segments(maxima.clip(lower=0) * to_soc_unit, soc_unit), ) + def _on_the_window(self, profile: pd.Series, index: pd.DatetimeIndex) -> pd.Series: + """A target profile's values at the window's steps, by the instant each step starts at, where the profile covers it. + + The profile is floored to steps on its own clock, and the window on the description's, + so in a time zone half an hour off the other the two sets of steps never share an instant, + and matching them by label dropped the whole profile without a word. + """ + if profile.empty: + return profile + covered_until = profile.index[-1] + pd.Timedelta(self._resolution) + at_steps = profile.reindex(index.union(profile.index)).ffill().reindex(index) + return at_steps[(index >= profile.index[0]) & (index < covered_until)] + def _as_segments(self, series: pd.Series, unit: str) -> list[dict]: """A series on this handler's resolution as FlexMeasures' time segments, one per run of equal values.""" segments: list[dict] = [] @@ -1026,6 +1059,8 @@ async def _plan( soc_at_start, ) soc_at_start = 0.0 + # And read back from zero, so the instructions follow the plan from where it started. + fill_level_at_start = 0.0 soc_max = max(soc_max, soc_at_start) # Where this schedule starts, which is both what it is asked for, @@ -1035,7 +1070,7 @@ async def _plan( # FlexMeasures takes a state of charge in kWh or MWh only. # So a fill level recorded in another energy unit is converted here, at the edge, rather than refused, # and the bounds read back below stay in the recorded unit. - # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. + # The usage and the leakage are read from sensors, which FlexMeasures converts itself, and the band is converted here. soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { "soc-unit": soc_unit, @@ -1049,8 +1084,16 @@ async def _plan( soc_minima, soc_maxima = self._fill_level_band( start, system_description, soc_unit, to_soc_unit ) - flex_model["soc-minima"] = soc_minima - flex_model["soc-maxima"] = soc_maxima + # FlexMeasures has the relax-soc-constraints that makes it a band from v0.26.0, + # and a flex context naming a field the server does not know is refused outright, leaving no schedule at all. + if await self._fm_client.server_version_at_least("0.26.0"): + flex_model["soc-minima"] = soc_minima + flex_model["soc-maxima"] = soc_maxima + else: + self._logger.info( + "Not asking for a schedule against the declared fill-level band: FlexMeasures can relax it from v0.26.0, and this server reports %s.", + self._fm_client.server_version, + ) if self._usage_forecast_sensor_id is not None: flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] if await self._record_storage_efficiency(start, fill_level_at_start): @@ -1112,7 +1155,8 @@ async def _plan( # which makes states a storage really reaches infeasible. # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. - flex_context["relax-soc-constraints"] = True + if "soc-minima" in flex_model: + flex_context["relax-soc-constraints"] = True if self._relax_site_capacity is not None: flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity @@ -1150,7 +1194,10 @@ async def _plan( # What the storage loses by itself and what is drawn from it, as the resource manager last said, # so that the fill level each step is answered at falls as well as rises. leakage_behaviour=self._latest(self._leakage_behaviour_history), - usage_forecast=self._latest(self._usage_forecast_history), + # Without negative usage, as it was recorded and planned against (see send_usage_forecast). + usage_forecast=_without_negative_usage( + self._latest(self._usage_forecast_history) + ), ) await self.send_instruction_batch( diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 24e17c88..9b9fedc7 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -599,3 +599,97 @@ async def test_the_band_goes_out_in_the_unit_flexmeasures_takes(system_descripti assert [s["value"] for s in flex_model["soc-minima"]] == [ f"{RANGE_BOTTOM / 1000} kWh" ] + + +def test_a_profile_on_a_half_hour_grid_reaches_the_steps_of_an_hourly_window( + system_description, +): + """Floored on its own clock, half an hour off the window's, no step shared an instant, and the profile was dropped.""" + from datetime import datetime, timedelta, timezone + from zoneinfo import ZoneInfo + + from s2python.common import Duration + from s2python.frbc import ( + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, + ) + + frbc = _frbc(system_description, resolution=timedelta(hours=1)) + start = datetime(2024, 1, 1, 23, tzinfo=timezone.utc) + profile = FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 2, 4, 0, tzinfo=ZoneInfo("Asia/Kolkata")), + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration.from_timedelta(timedelta(hours=6)), + fill_level_range=NumberRange(start_of_range=50, end_of_range=60), + ) + ], + ) + frbc._fill_level_target_profile_history[str(profile.message_id)] = profile + minima, _ = frbc._fill_level_band(start, system_description, "kWh", 1.0) + assert "50.0 kWh" in [segment["value"] for segment in minima] + + +@pytest.mark.asyncio +async def test_the_band_is_left_out_for_a_server_that_cannot_relax_it( + system_description, caplog +): + """FlexMeasures before v0.26.0 refuses relax-soc-constraints, and with it the whole request.""" + import logging + + caplog.set_level(logging.INFO) + frbc = _frbc(system_description) + frbc._fm_client.server_version_at_least = AsyncMock(return_value=False) + flex_model, flex_context = await _asked_for(frbc, system_description) + assert "soc-minima" not in flex_model + assert "relax-soc-constraints" not in flex_context + assert "v0.26.0" in caplog.text + + +@pytest.mark.asyncio +async def test_the_schedule_is_read_back_without_the_negative_usage_it_was_planned_without( + system_description, +): + """Read back with it, the instructions followed fill levels the plan never had.""" + from datetime import timedelta + from unittest.mock import patch + + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + frbc = _frbc(system_description) + forecast = FRBCUsageForecast( + message_id=get_unique_id(), + start_time="2024-01-01T00:00:00+00:00", + elements=[ + FRBCUsageForecastElement( + duration=Duration.from_timedelta(timedelta(hours=1)), + usage_rate_expected=-0.5, + ) + ], + ) + frbc._usage_forecast_history[str(forecast.message_id)] = forecast + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + read_back = translate.call_args.kwargs["usage_forecast"] + assert [e.usage_rate_expected for e in read_back.elements] == [0.0] + + +@pytest.mark.asyncio +async def test_a_storage_planned_from_zero_is_read_back_from_zero(system_description): + from unittest.mock import patch + + frbc = _frbc(system_description) + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=-5.0 + ) + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["initial_fill_level"] == 0.0 From fe65dcd4e7d058661bfb42165d547aa7716d8db9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:06:32 +0100 Subject: [PATCH 63/87] test(FRBC): an empty profile covers no step Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_frbc_soft_band.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 9b9fedc7..f160d3cc 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -693,3 +693,9 @@ async def test_a_storage_planned_from_zero_is_read_back_from_zero(system_descrip ) as translate: await frbc.trigger_schedule(system_description.message_id) assert translate.call_args.kwargs["initial_fill_level"] == 0.0 + + +def test_an_empty_profile_covers_no_step(system_description): + frbc = _frbc(system_description) + index = pd.date_range("2024-01-01T00:00:00+00:00", periods=4, freq="15min") + assert frbc._on_the_window(pd.Series(dtype=float), index).empty From f8c46b1f6ab8fbe2c95e09236c188cfa45710072 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:11:03 +0100 Subject: [PATCH 64/87] fix(FRBC): a mode for abnormal conditions does not set the charging efficiency either Review found that the charging efficiency, recorded as the lowest of what the modes convert at, still counted a mode reserved for abnormal conditions, so a less efficient emergency boost lowered the efficiency every plan was made against, though the scheduler no longer plans in it. It now counts the modes the schedule is planned in. The test handler that instructs into a dummy mode picks a normal one too. FRBC TUNES, which keeps its own mode lookups, is left as it is; its translation now raises where an actuator has no normal mode, where it used to instruct into an abnormal one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/__init__.py | 7 +++- .../s2/control_types/FRBC/frbc_simple.py | 3 +- tests/s2/test_frbc_operation_modes.py | 36 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 39ef719e..98c7adc9 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -392,11 +392,16 @@ async def trigger_schedule( actuator = system_description.actuators[0] + # Imported here, since the utilities import from this package. + from flexmeasures_client.s2.control_types.FRBC.utils import ( + normal_operation_modes, + ) + instruction = FRBCInstruction( message_id=get_unique_id(), id=get_unique_id(), actuator_id=actuator.id, - operation_mode=actuator.operation_modes[0].id, + operation_mode=normal_operation_modes(actuator)[0].id, operation_mode_factor=0.5, execution_time=system_description.valid_from, abnormal_condition=False, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index dc584b25..4ed39b0a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -702,7 +702,8 @@ async def _record_charging_efficiency( actuator = system_description.actuators[0] efficiencies: dict[str, float] = {} # The energy unit is one fill_rate_in_watts knows, which the constructor saw to. - for operation_mode in actuator.operation_modes: + # Only modes it may be instructed into under normal conditions, which are the ones the schedule is planned in. + for operation_mode in normal_operation_modes(actuator): efficiency = operation_mode_conversion_efficiency( operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) diff --git a/tests/s2/test_frbc_operation_modes.py b/tests/s2/test_frbc_operation_modes.py index 66e17bd0..88d24411 100644 --- a/tests/s2/test_frbc_operation_modes.py +++ b/tests/s2/test_frbc_operation_modes.py @@ -426,3 +426,39 @@ async def test_no_schedule_is_asked_for_an_actuator_with_no_normal_mode(caplog): await frbc.trigger_schedule(system_description.message_id) frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() assert "under normal conditions" in caplog.text + + +@pytest.mark.asyncio +async def test_a_mode_for_abnormal_conditions_only_does_not_set_the_charging_efficiency(): + """Less efficient than the normal modes, an emergency boost lowered the efficiency every plan was made against.""" + system_description = _system_description((0.0, 1000.0), (0.0, 5000.0)) + actuator = system_description.actuators[0] + boost = actuator.operation_modes[1].model_copy( + update={"abnormal_condition_only": True} + ) + system_description = system_description.model_copy( + update={ + "actuators": [ + actuator.model_copy( + update={"operation_modes": [actuator.operation_modes[0], boost]} + ) + ] + } + ) + frbc = _frbc(system_description, charging_efficiency_sensor_id=9) + seen = [] + from flexmeasures_client.s2.control_types.FRBC import frbc_simple + + real = frbc_simple.operation_mode_conversion_efficiency + + def spy(mode, *args): + seen.append(str(mode.id)) + return real(mode, *args) + + from unittest.mock import patch + + with patch.object( + frbc_simple, "operation_mode_conversion_efficiency", side_effect=spy + ): + await _asked_for(frbc, system_description) + assert str(boost.id) not in seen From 7c457461a3260f3a538b835c3ff5afa7613a5ef6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:35:46 +0100 Subject: [PATCH 65/87] feat(client): post_sensor_data(confirm_ingestion=True) raises where ingestion is unconfirmed A sensor-data post that waits for its ingestion job logs a timeout, or a failed lookup of the job, and carries on, since the data may land moments later. A caller that remembers having recorded the data then never records it again, which is how the FRBC handler's efficiency memo came to name a sensor that was empty for the window it planned (review finding B8). With confirm_ingestion, both are raised once logged, for that caller to handle. The default is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 25 +++++++++++-- tests/client/test_sensor.py | 61 +++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 9252c477..002cd671 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -664,6 +664,7 @@ async def post_sensor_data( file_path: str | None = None, belief_time_measured_instantly: bool = False, await_ingestion: bool = True, + confirm_ingestion: bool = False, ): """ Post sensor data for the given time range. @@ -689,6 +690,9 @@ async def post_sensor_data( since the data may well land moments later, and a caller that has waited that long is better served by carrying on than by failing. Pass ``False`` to return as soon as the server has accepted the data. A synchronous ``200 OK`` response is unaffected either way. + :param confirm_ingestion: whether ingestion that could not be confirmed while waiting for it raises, rather than being logged and ignored: + :class:`JobTimeoutError` where the job was still pending at the timeout, and what the lookup raised where looking the job up failed. + For a caller that remembers having recorded the data, and would otherwise never record it again. This function raises a ValueError when an unhandled status code is returned. """ @@ -740,6 +744,7 @@ async def post_sensor_data( unit=unit, prior=prior, await_ingestion=await_ingestion, + confirm_ingestion=confirm_ingestion, ) else: # Type assertion to help the type checker understand that file_path is not None @@ -752,6 +757,7 @@ async def post_sensor_data( belief_time_measured_instantly=belief_time_measured_instantly, unit=unit, await_ingestion=await_ingestion, + confirm_ingestion=confirm_ingestion, ) @staticmethod @@ -767,7 +773,9 @@ def _ingestion_job_id(response) -> str | None: job_id = response.get("job") or response.get("job_id") return job_id if isinstance(job_id, str) else None - async def _await_sensor_data_ingestion(self, job_id: str) -> None: + async def _await_sensor_data_ingestion( + self, job_id: str, confirm_ingestion: bool = False + ) -> None: """Wait for an ingestion job to finish, so that the data it carries can be read back. Waiting is left to :func:`wait_for_job`, so ingestion is backed off like every other job this client waits for, @@ -778,7 +786,8 @@ async def _await_sensor_data_ingestion(self, job_id: str) -> None: A timeout is logged as an error and otherwise ignored, for the reason given in :func:`post_sensor_data`, and so is a failure to look the job up at all, which says nothing about the data: the server accepted that, and a job record can be gone, or answered for by something in front of the server, by the time it is asked about. - Raising there would report a post that succeeded as a post that failed. + Raising there would report a post that succeeded as a post that failed, + unless the caller asked for ingestion to be confirmed, in which case both are raised once logged. A lookup that 404s is reported as a ``ConnectionError`` by :func:`request`, which wraps every client error that way, so that is among the ones caught here, alongside the ``ValueError`` an unexpected status code raises. """ @@ -793,6 +802,8 @@ async def _await_sensor_data_ingestion(self, job_id: str) -> None: self.logger.error( f"Ingestion of the data posted as job {job_id} is unconfirmed, so a read of it may come back without it. {e}" ) + if confirm_ingestion: + raise except ( ConnectionError, TimeoutError, @@ -803,6 +814,8 @@ async def _await_sensor_data_ingestion(self, job_id: str) -> None: self.logger.error( f"Ingestion of the data posted as job {job_id} is unconfirmed, because looking that job up failed, so a read of it may come back without it. {e}" ) + if confirm_ingestion: + raise async def _post_sensor_data_json( self, @@ -813,6 +826,7 @@ async def _post_sensor_data_json( unit: str, prior: str | datetime | None = None, await_ingestion: bool = True, + confirm_ingestion: bool = False, ): """ Post sensor data using JSON payload. @@ -849,7 +863,7 @@ async def _post_sensor_data_json( self.logger.info("Sensor data sent successfully via JSON.") if await_ingestion and status == 202 and job_id: - await self._await_sensor_data_ingestion(job_id) + await self._await_sensor_data_ingestion(job_id, confirm_ingestion) return response, status @@ -860,6 +874,7 @@ async def _post_sensor_data_file( belief_time_measured_instantly: bool = False, unit: str | None = None, await_ingestion: bool = True, + confirm_ingestion: bool = False, ): """ Post sensor data using file upload. @@ -981,7 +996,7 @@ async def _post_sensor_data_file( # Wait for ingestion outside the request timeout above, and outside the error handler that names this an upload failure: # a job takes as long as self.ingestion_polling_timeout allows, which is no concern of the request that queued it. if await_ingestion and status == 202 and upload_job_id: - await self._await_sensor_data_ingestion(upload_job_id) + await self._await_sensor_data_ingestion(upload_job_id, confirm_ingestion) return response_data, status @@ -995,6 +1010,7 @@ async def post_measurements( unit: str, prior: str | datetime | None = None, await_ingestion: bool = True, + confirm_ingestion: bool = False, ): """ Post sensor data for the given time range. @@ -1019,6 +1035,7 @@ async def post_measurements( unit=unit, prior=prior, await_ingestion=await_ingestion, + confirm_ingestion=confirm_ingestion, ) async def get_schedule( diff --git a/tests/client/test_sensor.py b/tests/client/test_sensor.py index d8181e96..e178ca37 100644 --- a/tests/client/test_sensor.py +++ b/tests/client/test_sensor.py @@ -16,6 +16,7 @@ from flexmeasures_client.exceptions import ( InsufficientServerVersionError, JobFailedError, + JobTimeoutError, ) @@ -1069,6 +1070,66 @@ async def test_post_sensor_data_await_ingestion_timeout(caplog): await client.close() +@pytest.mark.asyncio +async def test_post_sensor_data_confirm_ingestion_raises_on_timeout(caplog): + """A caller that remembers having recorded the data is told it may not have been, instead of carrying on.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.ingestion_polling_timeout = 0.05 + client.ingestion_polling_interval = 0.01 + sensor_id = 5 + m.post( + f"http://localhost:5000/api/v3_0/sensors/{sensor_id}/data", + status=202, + payload={"status": "ACCEPTED", "job": "job-4"}, + ) + m.get( + "http://localhost:5000/api/v3_0/jobs/job-4", + status=200, + payload={"status": "QUEUED", "message": "waiting", "result": None}, + repeat=True, + ) + + with pytest.raises(JobTimeoutError): + await client.post_sensor_data( + sensor_id=sensor_id, + start="2023-01-01T00:00+00:00", + duration="PT1H", + values=[1.0], + unit="MW", + confirm_ingestion=True, + ) + assert any("unconfirmed" in record.message for record in caplog.records) + await client.close() + + +@pytest.mark.asyncio +async def test_post_sensor_data_confirm_ingestion_raises_where_the_job_cannot_be_looked_up(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.request_retry_interval = 0.01 + sensor_id = 5 + m.post( + f"http://localhost:5000/api/v3_0/sensors/{sensor_id}/data", + status=202, + payload={"status": "ACCEPTED", "job": "job-5"}, + ) + m.get("http://localhost:5000/api/v3_0/jobs/job-5", status=404, repeat=True) + + with pytest.raises((ConnectionError, ValueError)): + await client.post_sensor_data( + sensor_id=sensor_id, + start="2023-01-01T00:00+00:00", + duration="PT1H", + values=[1.0], + unit="MW", + confirm_ingestion=True, + ) + await client.close() + + @pytest.mark.asyncio async def test_post_sensor_data_synchronous_response_is_not_polled(): """A synchronous 200 OK answer names no job, so nothing is polled.""" From f32b44db7a5e42736196f1a3109f0a24d55ca9c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:35:47 +0100 Subject: [PATCH 66/87] fix(FRBC): a failure is answered or said out loud Package P2.3 of the plan to finish #188, for review findings B6 to B9 and the three gaps the review named. - B6: the schedule retry caught JobFailedError and ValueError only, so a job no worker picked up (JobTimeoutError) and a gateway restart (ConnectionError), the two likeliest production failures, escaped on the first attempt, and the line saying the storage is left without an instruction never went out. Both are now caught, asked about again, and logged when given up on. - B7: the state of charge was posted first and unguarded, so a failed ingestion job unwound send_storage_status before planning. It is now guarded and logged; the schedule starts from the level the status reports, sent inline. - B8: an efficiency posted with an unconfirmed ingestion was remembered as recorded and never posted again for its window. Both efficiency posts now ask the client to confirm ingestion, and are remembered only once it is. - B9: before v0.33.0, FlexMeasures reports a failed job only as a bare HTTP 400, which matched nothing, so the retry was inert there. Against such a server a bare 400 is asked about again, within the attempts allowed; it cannot be told apart from a refused trigger, which then costs those attempts. - A description of several actuators is refused with INVALID_CONTENT when it arrives, rather than acknowledged and failing after each solve; timers and transitions, and each FRBC.TimerStatus (now kept), are warned about as not honoured; soc-usage is named only where the resource manager says it provides a usage forecast. get_reception_status takes a diagnostic label. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- CHANGELOG.rst | 1 + .../s2/control_types/FRBC/__init__.py | 29 ++++ .../s2/control_types/FRBC/frbc_simple.py | 106 ++++++++++++--- src/flexmeasures_client/s2/utils.py | 7 +- tests/s2/test_frbc_charging_efficiency.py | 23 +++- tests/s2/test_frbc_declared_profiles.py | 28 ++++ tests/s2/test_frbc_schedule_retry.py | 78 ++++++++++- tests/s2/test_frbc_simple_planning.py | 2 +- tests/s2/test_frbc_unsupported_is_said.py | 124 ++++++++++++++++++ 10 files changed, 376 insertions(+), 26 deletions(-) create mode 100644 tests/s2/test_frbc_unsupported_is_said.py diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b220c6ee..07bf8c19 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,9 +46,11 @@ Changed * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] * ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to, and the declared range applies again beyond its end. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, and the log says so. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] -* ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] +* ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A scheduling job no worker picked up in time, and a server that could not be reached, are asked about again too, as is a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] * The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] +* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where before it acknowledged the description and then failed after each solve, in a task nobody awaited. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each ``FRBC.TimerStatus``, which is now kept. ``FRBCSimple`` plans with the usage forecast sensor only where the resource manager says it provides a usage forecast [`#273 `__] +* ``FRBCSimple`` plans from a storage status even where recording its state of charge failed, which is logged, since the schedule starts from the level the status reports, sent inline; before, the failed post cancelled the planning it preceded. A charging or storage efficiency whose ingestion could not be confirmed is no longer remembered as recorded, so it is posted again for the next plan rather than never [`#273 `__] Removed ------- diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4d0d1f34..66b6e9a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,7 @@ Added * ``server_version_at_least``, which answers whether the server is of a given version or newer, looking its version up where that is not known yet, for a caller that sends a field or calls an endpoint only where the server has it. Release generations are compared, so a development build such as ``0.33.0.dev26`` counts as ``0.33.0`` [`#248 `__] * ``ingestion_polling_interval``, ``ingestion_polling_max_interval`` and ``ingestion_polling_timeout``, which govern how the ingestion of posted sensor data is waited for, on a tighter profile than the ``job_polling_*`` settings give other background jobs [`#242 `__] +* ``post_sensor_data`` takes ``confirm_ingestion``, which raises where the ingestion of the posted data could not be confirmed, rather than logging it and carrying on, for a caller that remembers having recorded the data and would otherwise never record it again [`#273 `__] * Optional ``rate_limit_notifier`` and ``job_status_notifier`` callbacks, to report rate limits and the progress of jobs being waited for, instead of logging them [`#225 `__] * ``get_automations``, ``get_automation``, ``add_automation``, ``update_automation``, ``delete_automation``, ``trigger_automation`` and ``trigger_and_await_automation``, to manage and run automations through the API on FlexMeasures v1.1.0 and up [`#230 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 98c7adc9..3c8af096 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -217,6 +217,18 @@ def handle_system_description( # descriptions, and not before: it would otherwise have dropped a planning request. # The TUNES handler has fingerprinted its descriptions this way all along, and can adopt # this shared helper separately. + # Every plan is made for one actuator, so a description of several is refused when it arrives, + # rather than acknowledged and then failing in the background on every round, after a solve. + if len(message.actuators) != 1: + self._logger.error( + "Refusing an FRBC.SystemDescription with %s actuators: this CEM plans for exactly one.", + len(message.actuators), + ) + return get_reception_status( + message, + status=ReceptionStatusValues.INVALID_CONTENT, + diagnostic_label=f"This CEM plans for exactly one actuator, and the description has {len(message.actuators)}.", + ) fingerprint = self.system_description_fingerprint(message) if fingerprint == self._last_system_description_fingerprint: self._logger.debug( @@ -230,6 +242,16 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message + # Said once per description, since nothing here would otherwise say it at all: + # an instruction can ask for a switch that a running timer or a missing transition forbids, and the device refuses it. + actuator = message.actuators[0] + if actuator.timers or actuator.transitions: + self._logger.warning( + "The actuator declares %s timers and %s transitions, which this CEM does not honour: an instruction may ask for a switch the device refuses.", + len(actuator.timers), + len(actuator.transitions), + ) + # A description says what the device can do, and a storage status what it holds now, # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), # but a description that arrives after the latest status, and differs from the description before it, @@ -332,6 +354,13 @@ def handle_fill_level_target_profile( @register(FRBCTimerStatus) def handle_frbc_timer_status(self, message: FRBCTimerStatus) -> pydantic.BaseModel: + # Kept, so that what the resource manager said is there to see, though nothing plans with it yet. + self._timer_status_history[str(message.message_id)] = message + self._logger.warning( + "Timer %s runs until %s, which this CEM does not honour: an instruction may ask for a switch it blocks.", + message.timer_id, + message.finished_at, + ) return get_reception_status(message, status=ReceptionStatusValues.OK) async def send_storage_status( diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index dc95f20a..2f73fbd4 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -25,9 +25,14 @@ import pandas as pd +from aiohttp import ClientError from flexmeasures_client.client import convert_units -from flexmeasures_client.exceptions import ContentTypeError, JobFailedError +from flexmeasures_client.exceptions import ( + ContentTypeError, + JobFailedError, + JobTimeoutError, +) from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( WATTS_PER_ENERGY_UNIT_PER_SECOND, @@ -75,6 +80,23 @@ #: and FlexMeasures refuses to plan from one it considers too old. RETRYABLE_TRIGGER_REFUSALS = ("No recent state-of-charge value found",) +#: What a request to FlexMeasures can fail with, short of a defect in this code: +#: refused (ValueError), unreachable (ConnectionError, TimeoutError, ClientError), answered oddly (ContentTypeError), +#: or queued and then failed or never picked up (JobFailedError, JobTimeoutError). +REQUEST_FAILURES = ( + JobFailedError, + JobTimeoutError, + ValueError, + ConnectionError, + TimeoutError, + ContentTypeError, + ClientError, +) + +#: What FlexMeasures before v0.33.0 answers a failed scheduling job with, having no job-status API to say so: +#: an HTTP 400, which the client raises as a ValueError with this message and nothing more. +BARE_REFUSAL = "Request failed with status code 400" + def _merged(ranges: list[tuple[float, float]]) -> list[tuple[float, float]]: """The union of closed ranges, as the fewest ranges that cover it, in order.""" @@ -424,16 +446,26 @@ async def send_storage_status( # without a prior, FlexMeasures took the moment the request reached it, # and a simulated reading was recorded as known years after it was taken. now = self.now() - await self._fm_client.post_measurements( - self._soc_sensor_id, - start=now, - # Scaled like every other fill level this handler records, or the level and the - # band around it would be in different scales wherever fill_level_scale is not 1. - values=[status.present_fill_level * self._fill_level_scale], - unit=self.energy_unit, - duration=await self._soc_duration(), - prior=now, - ) + # Guarded, because a failed post would otherwise unwind this method before the planning below, + # and a schedule starts from the state of charge this status reports, sent inline, not from the one recorded here. + try: + await self._fm_client.post_measurements( + self._soc_sensor_id, + start=now, + # Scaled like every other fill level this handler records, or the level and the + # band around it would be in different scales wherever fill_level_scale is not 1. + values=[status.present_fill_level * self._fill_level_scale], + unit=self.energy_unit, + duration=await self._soc_duration(), + prior=now, + ) + except REQUEST_FAILURES: + self._logger.exception( + "Could not record the state of charge %s on sensor %s for %s; planning from it all the same.", + status.present_fill_level, + self._soc_sensor_id, + now, + ) # The state of charge this status reports is what a schedule starts from, so this is # where a schedule is asked for. @@ -841,8 +873,11 @@ async def _record_charging_efficiency( # and for a heat pump, whose fill rate is the heat it delivers, it is well above one. unit="dimensionless", duration=self._schedule_duration, + # Remembered below as recorded, and so never posted again for this window: + # an ingestion still pending at the timeout is not taken as recorded. + confirm_ingestion=True, ) - except (JobFailedError, ValueError, ConnectionError, NotImplementedError): + except (*REQUEST_FAILURES, NotImplementedError): # A schedule planned as if the actuator converted power one for one beats no schedule at all, # as long as it is said out loud: # that plan overstates what a lossy storage gains from a given power, and understates what a heat pump puts in. @@ -857,7 +892,7 @@ async def _record_charging_efficiency( self._charging_efficiency_recorded = (start, efficiency) return True - def _is_worth_asking_again(self, failure: Exception) -> bool: + async def _is_worth_asking_again(self, failure: Exception) -> bool: """Whether a failed attempt at a schedule is worth making again. A job that reached the server and failed inside it is, @@ -873,9 +908,27 @@ def _is_worth_asking_again(self, failure: Exception) -> bool: message = str(failure) if any(marker in message for marker in NON_RETRYABLE_SCHEDULE_FAILURES): return False - if isinstance(failure, JobFailedError): + # A job nobody picked up in time, and a server that could not be reached, may both be there for the next attempt: + # a scheduling queue without a live worker, and a gateway restarting, are the likeliest of either. + if isinstance( + failure, + ( + JobFailedError, + JobTimeoutError, + ConnectionError, + TimeoutError, + ClientError, + ), + ): return True - return any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS) + if any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS): + return True + # FlexMeasures before v0.33.0 reports a failed job only as a bare HTTP 400, + # which cannot be told apart from a refused trigger, so it is asked again, within the attempts allowed. + return ( + message == BARE_REFUSAL + and not await self._fm_client.server_version_at_least("0.33.0") + ) async def _ask_for_a_schedule( self, @@ -896,8 +949,10 @@ async def _ask_for_a_schedule( :param start: the window the schedule is asked for, which every attempt asks for again. :param request_number: the planning request this answers; where a newer one arrives between attempts, no further attempt is made and None is returned, so that the newer one is planned sooner. - :raises JobFailedError: where the last attempt worth making failed inside the server. - :raises ValueError: where the trigger itself was refused. + :raises JobFailedError: where the last attempt worth making failed inside the server. + :raises JobTimeoutError: where the last attempt worth making was not picked up in time. + :raises ConnectionError: where the server could not be reached on the last attempt worth making. + :raises ValueError: where the trigger itself was refused. """ attempt = 0 while True: @@ -917,10 +972,10 @@ async def _ask_for_a_schedule( # and not in the power sensor's own unit, which is what a schedule comes back in by default. unit=S2_POWER_UNIT, ) - except (JobFailedError, ValueError) as failure: + except (*REQUEST_FAILURES, NotImplementedError) as failure: if ( attempt >= self._schedule_attempts - or not self._is_worth_asking_again(failure) + or not await self._is_worth_asking_again(failure) ): self._logger.exception( "The storage is left without an instruction for the window starting %s: asking for a schedule failed on attempt %s of %s.", @@ -1115,7 +1170,14 @@ async def _plan( self._fm_client.server_version, ) if self._usage_forecast_sensor_id is not None: - flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] + # Not where the resource manager said it sends no usage forecast, + # since nothing would then be recorded on the sensor for the scheduler to read. + if system_description.storage.provides_usage_forecast: + flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] + else: + self._logger.info( + "Planning without usage: the resource manager says it provides no usage forecast." + ) if await self._record_storage_efficiency(start, fill_level_at_start): flex_model["storage-efficiency"] = { "sensor": self._leakage_behaviour_sensor_id @@ -1383,8 +1445,10 @@ async def _record_storage_efficiency( values=[efficiency], unit="%", duration=self._schedule_duration, + # As for the charging efficiency: remembered below as recorded, so only once it is. + confirm_ingestion=True, ) - except (JobFailedError, ValueError, ConnectionError): + except REQUEST_FAILURES: self._logger.exception( "Not asking for a schedule against a storage efficiency: could not record %s %% on sensor %s for %s.", efficiency, diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index cfdcd55e..f0755643 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -71,13 +71,18 @@ class _HasMessageId(Protocol): def get_reception_status( subject_message: _HasMessageId, status: ReceptionStatusValues = ReceptionStatusValues.OK, + diagnostic_label: str | None = None, ): """ This function returns a ReceptionStatus message for the subject message `subject_message`. By default, the status ReceptionStatusValues.OK is sent. + + :param diagnostic_label: why, for a status other than OK, which the resource manager can log. """ return ReceptionStatus( - subject_message_id=str(subject_message.message_id), status=status + subject_message_id=str(subject_message.message_id), + status=status, + diagnostic_label=diagnostic_label, ) diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 4a74057f..05239a6c 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -27,7 +27,7 @@ ) from flexmeasures_client.client import FlexMeasuresClient, _server_version_at_least -from flexmeasures_client.exceptions import JobFailedError +from flexmeasures_client.exceptions import JobFailedError, JobTimeoutError from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple from flexmeasures_client.s2.control_types.FRBC.utils import ( fill_rate_in_watts, @@ -564,3 +564,24 @@ def test_a_batch_of_instructions_is_logged_once_and_by_name(caplog): ] assert len(batches) == 1 assert "'on'" in batches[0] and "'off'" in batches[0] + + +@pytest.mark.asyncio +async def test_an_efficiency_whose_ingestion_is_unconfirmed_is_recorded_again(): + """Remembered as recorded, it was never posted again for its window, and the schedule named a sensor empty for it.""" + system_description = _heater() + frbc = _frbc(system_description) + frbc._fm_client.post_sensor_data = AsyncMock( + side_effect=[JobTimeoutError("still pending"), ({}, 200)] + ) + + first = await _asked_for(frbc, system_description) + second = await _asked_for(frbc, system_description) + + assert "charging-efficiency" not in first + assert "charging-efficiency" in second + assert frbc._fm_client.post_sensor_data.await_count == 2 + assert all( + c.kwargs.get("confirm_ingestion") is True + for c in frbc._fm_client.post_sensor_data.await_args_list + ) diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index 201ed323..1d656d25 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -40,6 +40,7 @@ ) from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import JobTimeoutError from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple from flexmeasures_client.s2.control_types.FRBC.utils import ( clip_fill_level_target_profile, @@ -668,3 +669,30 @@ async def test_a_storage_efficiency_that_could_not_be_recorded_is_not_planned_ag assert await frbc._record_storage_efficiency(start, 50.0) is False assert frbc._storage_efficiency_recorded is None assert "could not record" in caplog.text + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_whose_ingestion_is_unconfirmed_is_not_remembered( + system_description, +): + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + frbc._fm_client.post_sensor_data.side_effect = JobTimeoutError("still pending") + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) is False + assert frbc._storage_efficiency_recorded is None + assert ( + frbc._fm_client.post_sensor_data.await_args.kwargs["confirm_ingestion"] is True + ) diff --git a/tests/s2/test_frbc_schedule_retry.py b/tests/s2/test_frbc_schedule_retry.py index b93a81c2..5c9cae64 100644 --- a/tests/s2/test_frbc_schedule_retry.py +++ b/tests/s2/test_frbc_schedule_retry.py @@ -27,7 +27,7 @@ ) from flexmeasures_client.client import FlexMeasuresClient, _server_version_at_least -from flexmeasures_client.exceptions import JobFailedError +from flexmeasures_client.exceptions import JobFailedError, JobTimeoutError from flexmeasures_client.s2.control_types.FRBC.frbc_simple import ( SCHEDULE_ATTEMPTS, FRBCSimple, @@ -388,3 +388,79 @@ def test_a_schedule_per_device_names_no_scheduler_to_check(caplog): frbc._warn_where_a_fallback_scheduler_answered([SCHEDULE, SCHEDULE], VALID_FROM) assert caplog.text == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [ + JobTimeoutError("Job 1 did not finish within 600 seconds."), + ConnectionError("Error occurred while communicating with the API: 502"), + ], + ids=["no worker picked the job up", "the gateway restarted"], +) +async def test_the_likeliest_failures_in_production_are_asked_again(failure, caplog): + """Neither was caught, so each escaped on the first attempt, and the line saying the storage was left without an instruction never went out.""" + system_description = _heater() + frbc = _frbc(system_description) + frbc._fm_client.trigger_and_get_schedule.side_effect = [failure, SCHEDULE] + + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == 2 + frbc.send_instruction_batch.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_giving_up_on_an_unreachable_server_says_the_storage_is_left_without_an_instruction( + caplog, +): + system_description = _heater() + frbc = _frbc(system_description) + frbc._fm_client.trigger_and_get_schedule.side_effect = ConnectionError("down") + + with pytest.raises(ConnectionError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == frbc._schedule_attempts + assert "left without an instruction" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("server_version, attempts", [("0.32.0", 3), ("0.33.0", 1)]) +async def test_a_bare_refusal_is_asked_again_only_where_the_server_cannot_say_a_job_failed( + server_version, attempts +): + """Before v0.33.0 a failed job comes back as a bare HTTP 400, which matched nothing, so the retry was inert there.""" + system_description = _heater() + frbc = _frbc(system_description, schedule_attempts=3) + frbc._fm_client.server_version_at_least = AsyncMock( + side_effect=lambda minimum: _server_version_at_least(server_version, minimum) + ) + frbc._fm_client.trigger_and_get_schedule.side_effect = ValueError( + "Request failed with status code 400" + ) + + with pytest.raises(ValueError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == attempts + + +@pytest.mark.asyncio +async def test_a_state_of_charge_that_could_not_be_recorded_is_planned_from_all_the_same( + caplog, +): + """The post went first and unguarded, so a failed ingestion job cancelled the planning it preceded.""" + system_description = _heater() + frbc = _frbc(system_description) + frbc._fm_client.post_measurements = AsyncMock( + side_effect=JobFailedError("Job 2 ended with status failed.") + ) + + await frbc.send_storage_status( + FRBCStorageStatus(message_id=get_unique_id(), present_fill_level=40.0) + ) + + assert _attempts(frbc) == 1 + assert "Could not record the state of charge" in caplog.text diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 428c76ee..74a39641 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -651,7 +651,7 @@ async def test_a_request_out_of_date_makes_no_further_attempt_while_the_newest_w frbc = frbc_simple() frbc._schedule_attempts = 5 frbc._schedule_retry_interval = td(milliseconds=10) - frbc._is_worth_asking_again = lambda failure: True + frbc._is_worth_asking_again = AsyncMock(return_value=True) first_attempt = asyncio.Event() planned = frbc._fm_client.trigger_and_get_schedule.return_value diff --git a/tests/s2/test_frbc_unsupported_is_said.py b/tests/s2/test_frbc_unsupported_is_said.py new file mode 100644 index 00000000..6815b6e2 --- /dev/null +++ b/tests/s2/test_frbc_unsupported_is_said.py @@ -0,0 +1,124 @@ +"""What this CEM does not support is said when it arrives, rather than discovered after a solve, or never.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone + +import pytest +from s2python.common import Duration, ReceptionStatusValues, Timer +from s2python.frbc import FRBCTimerStatus + +from flexmeasures_client.s2.utils import get_unique_id + +from .test_frbc_schedule_retry import _frbc, _heater + + +def _with(system_description, **actuator_update): + actuator = system_description.actuators[0].model_copy(update=actuator_update) + return system_description.model_copy(update={"actuators": [actuator]}) + + +@pytest.mark.asyncio +async def test_a_description_of_several_actuators_is_refused_when_it_arrives(caplog): + """Acknowledged, it failed in a task nobody awaited, after a solve, on every round.""" + system_description = _heater() + frbc = _frbc(system_description) + frbc._system_description_history.clear() + two = system_description.model_copy( + update={ + "message_id": get_unique_id(), + "actuators": [ + system_description.actuators[0], + system_description.actuators[0].model_copy( + update={"id": get_unique_id()} + ), + ], + } + ) + + status = await frbc.handle_message(two.to_dict()) + + assert status.status == ReceptionStatusValues.INVALID_CONTENT + assert "exactly one actuator" in status.diagnostic_label + assert not frbc._system_description_history + assert not frbc.background_tasks + + +@pytest.mark.asyncio +async def test_a_description_with_timers_or_transitions_is_taken_with_a_warning(caplog): + system_description = _heater() + frbc = _frbc(system_description) + frbc._storage_status_history.clear() + timer = Timer( + id=get_unique_id(), + diagnostic_label="minimum on time", + duration=Duration.from_milliseconds(60_000), + ) + described = _with(system_description, timers=[timer]) + described = described.model_copy(update={"message_id": get_unique_id()}) + + with caplog.at_level(logging.WARNING): + status = await frbc.handle_message(described.to_dict()) + + assert status.status == ReceptionStatusValues.OK + assert "1 timers and 0 transitions" in caplog.text + + +@pytest.mark.asyncio +async def test_a_description_without_timers_says_nothing_of_them(caplog): + system_description = _heater() + frbc = _frbc(system_description) + frbc._storage_status_history.clear() + with caplog.at_level(logging.WARNING): + status = await frbc.handle_message( + system_description.model_copy( + update={"message_id": get_unique_id()} + ).to_dict() + ) + # The description was taken, so silence is the handler's and not a description that never reached it. + assert status.status == ReceptionStatusValues.OK + assert len(frbc._system_description_history) == 2 + assert "does not honour" not in caplog.text + + +@pytest.mark.asyncio +async def test_a_timer_status_is_kept_and_said_to_be_unhonoured(caplog): + system_description = _heater() + frbc = _frbc(system_description) + timer_status = FRBCTimerStatus( + message_id=get_unique_id(), + timer_id=get_unique_id(), + actuator_id=system_description.actuators[0].id, + finished_at=datetime(2024, 1, 1, 0, 5, tzinfo=timezone.utc), + ) + + with caplog.at_level(logging.WARNING): + status = await frbc.handle_message(timer_status.to_dict()) + + assert status.status == ReceptionStatusValues.OK + assert str(timer_status.message_id) in frbc._timer_status_history + assert "does not honour" in caplog.text + + +async def _flex_model(frbc, system_description) -> dict: + await frbc.trigger_schedule(system_description.message_id) + return frbc._fm_client.trigger_and_get_schedule.await_args.kwargs["flex_model"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provided", [True, False]) +async def test_usage_is_planned_with_only_where_the_resource_manager_says_it_sends_it( + provided, +): + """Named regardless, the scheduler read a sensor nothing was ever recorded on.""" + system_description = _heater() + storage = system_description.storage.model_copy( + update={"provides_usage_forecast": provided} + ) + system_description = system_description.model_copy(update={"storage": storage}) + frbc = _frbc(system_description, usage_forecast_sensor_id=7) + + flex_model = await _flex_model(frbc, system_description) + + assert ("soc-usage" in flex_model) is provided From 1eb62ebf810f0017fc9f29b0ff2e18e2a0eb9ddd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:56:16 +0100 Subject: [PATCH 67/87] fix(FRBC): what the review of "a failure is answered or said out loud" found - A 4xx, which the client wraps in a ConnectionError like any HTTP error, answers the same next time; it is no longer asked about again. A 5xx or a failure to connect still is. - A job nobody picked up has already waited out its whole timeout, and a dead queue is still dead after two seconds, so it is asked about once more rather than on every attempt allowed. - The server-version lookup the bare-400 rule makes ran inside the handling of the failure it was for, so its own failure took that one's place and the "left without an instruction" line never went out. It is now caught. - A usage forecast sent despite provides_usage_forecast=False is planned with, since what a resource manager sends says more than a flag it set once. - A timer is warned about the first time a status names it, not on every one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- .../s2/control_types/FRBC/__init__.py | 7 ++- .../s2/control_types/FRBC/frbc_simple.py | 44 ++++++++----- tests/s2/test_frbc_schedule_retry.py | 63 +++++++++++++++++++ tests/s2/test_frbc_unsupported_is_said.py | 49 +++++++++++++++ 5 files changed, 148 insertions(+), 19 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 07bf8c19..ac8441eb 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,10 +46,10 @@ Changed * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] * ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to, and the declared range applies again beyond its end. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, and the log says so. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] -* ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A scheduling job no worker picked up in time, and a server that could not be reached, are asked about again too, as is a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] +* ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A scheduling job no worker picked up in time is asked about once more, a server that could not be reached or answered with a 5xx is asked again, and so is a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] * The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] -* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where before it acknowledged the description and then failed after each solve, in a task nobody awaited. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each ``FRBC.TimerStatus``, which is now kept. ``FRBCSimple`` plans with the usage forecast sensor only where the resource manager says it provides a usage forecast [`#273 `__] +* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where before it acknowledged the description and then failed after each solve, in a task nobody awaited. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every status is now kept. ``FRBCSimple`` plans with the usage forecast sensor only where the resource manager says it provides a usage forecast, or has sent one regardless [`#273 `__] * ``FRBCSimple`` plans from a storage status even where recording its state of charge failed, which is logged, since the schedule starts from the level the status reports, sent inline; before, the failed post cancelled the planning it preceded. A charging or storage efficiency whose ingestion could not be confirmed is no longer remembered as recorded, so it is posted again for the next plan rather than never [`#273 `__] Removed diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 3c8af096..f13c99a4 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -355,8 +355,13 @@ def handle_fill_level_target_profile( @register(FRBCTimerStatus) def handle_frbc_timer_status(self, message: FRBCTimerStatus) -> pydantic.BaseModel: # Kept, so that what the resource manager said is there to see, though nothing plans with it yet. + # Warned about once per timer, since a resource manager sends a status on every run of every timer. + warned = any( + known.timer_id == message.timer_id + for known in self._timer_status_history.values() + ) self._timer_status_history[str(message.message_id)] = message - self._logger.warning( + (self._logger.debug if warned else self._logger.warning)( "Timer %s runs until %s, which this CEM does not honour: an instruction may ask for a switch it blocks.", message.timer_id, message.finished_at, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 2f73fbd4..ae887831 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -25,7 +25,7 @@ import pandas as pd -from aiohttp import ClientError +from aiohttp import ClientError, ClientResponseError from flexmeasures_client.client import convert_units from flexmeasures_client.exceptions import ( @@ -892,7 +892,9 @@ async def _record_charging_efficiency( self._charging_efficiency_recorded = (start, efficiency) return True - async def _is_worth_asking_again(self, failure: Exception) -> bool: + async def _is_worth_asking_again( + self, failure: Exception, attempt: int = 1 + ) -> bool: """Whether a failed attempt at a schedule is worth making again. A job that reached the server and failed inside it is, @@ -910,25 +912,31 @@ async def _is_worth_asking_again(self, failure: Exception) -> bool: return False # A job nobody picked up in time, and a server that could not be reached, may both be there for the next attempt: # a scheduling queue without a live worker, and a gateway restarting, are the likeliest of either. + # A timeout has already waited out the job's whole timeout, and a dead queue will not come back two seconds later, + # so it is asked about once more, not on every attempt allowed. + if isinstance(failure, JobTimeoutError): + return attempt == 1 + # The client wraps every HTTP error in a ConnectionError, + # and one answering 4xx, such as an endpoint an old server lacks, answers the same next time. + cause = failure.__cause__ + if isinstance(cause, ClientResponseError) and cause.status < 500: + return False if isinstance( failure, - ( - JobFailedError, - JobTimeoutError, - ConnectionError, - TimeoutError, - ClientError, - ), + (JobFailedError, ConnectionError, TimeoutError, ClientError), ): return True if any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS): return True # FlexMeasures before v0.33.0 reports a failed job only as a bare HTTP 400, # which cannot be told apart from a refused trigger, so it is asked again, within the attempts allowed. - return ( - message == BARE_REFUSAL - and not await self._fm_client.server_version_at_least("0.33.0") - ) + if message != BARE_REFUSAL: + return False + # Looked up inside the handling of another failure, so a failure of its own must not take that one's place. + try: + return not await self._fm_client.server_version_at_least("0.33.0") + except REQUEST_FAILURES: + return False async def _ask_for_a_schedule( self, @@ -975,7 +983,7 @@ async def _ask_for_a_schedule( except (*REQUEST_FAILURES, NotImplementedError) as failure: if ( attempt >= self._schedule_attempts - or not await self._is_worth_asking_again(failure) + or not await self._is_worth_asking_again(failure, attempt) ): self._logger.exception( "The storage is left without an instruction for the window starting %s: asking for a schedule failed on attempt %s of %s.", @@ -1172,11 +1180,15 @@ async def _plan( if self._usage_forecast_sensor_id is not None: # Not where the resource manager said it sends no usage forecast, # since nothing would then be recorded on the sensor for the scheduler to read. - if system_description.storage.provides_usage_forecast: + # Or where it sent one anyway, which is what it says it will do, whatever its flag says. + if ( + system_description.storage.provides_usage_forecast + or self._usage_forecast_history + ): flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] else: self._logger.info( - "Planning without usage: the resource manager says it provides no usage forecast." + "Planning without usage: the resource manager says it provides no usage forecast, and has sent none." ) if await self._record_storage_efficiency(start, fill_level_at_start): flex_model["storage-efficiency"] = { diff --git a/tests/s2/test_frbc_schedule_retry.py b/tests/s2/test_frbc_schedule_retry.py index 5c9cae64..fbf2f8a2 100644 --- a/tests/s2/test_frbc_schedule_retry.py +++ b/tests/s2/test_frbc_schedule_retry.py @@ -464,3 +464,66 @@ async def test_a_state_of_charge_that_could_not_be_recorded_is_planned_from_all_ assert _attempts(frbc) == 1 assert "Could not record the state of charge" in caplog.text + + +def _wrapped_http_error(status: int) -> ConnectionError: + """What the client raises for an HTTP error: a ConnectionError, caused by aiohttp's own.""" + from aiohttp import ClientResponseError + + failure = ConnectionError("Error occurred while communicating with the API") + failure.__cause__ = ClientResponseError(None, (), status=status) + return failure + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status, attempts", [(404, 1), (503, 3)]) +async def test_an_http_error_is_asked_again_only_where_it_may_be_passing( + status, attempts +): + """A 4xx answers the same next time; a 5xx may be a restart.""" + system_description = _heater() + frbc = _frbc(system_description, schedule_attempts=3) + frbc._fm_client.trigger_and_get_schedule.side_effect = _wrapped_http_error(status) + + with pytest.raises(ConnectionError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == attempts + + +@pytest.mark.asyncio +async def test_a_job_nobody_picked_up_is_asked_about_once_more_only(): + """Each attempt waits out the whole job timeout, and a dead queue is still dead after it.""" + system_description = _heater() + frbc = _frbc(system_description, schedule_attempts=5) + frbc._fm_client.trigger_and_get_schedule.side_effect = JobTimeoutError("pending") + + with pytest.raises(JobTimeoutError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == 2 + + +@pytest.mark.asyncio +async def test_a_failed_version_lookup_does_not_take_the_place_of_the_failure_it_was_for( + caplog, +): + system_description = _heater() + frbc = _frbc(system_description, schedule_attempts=3) + + def version_at_least(minimum): + # Only the lookup the retry makes fails; the flex model's own lookups go through. + if minimum == "0.33.0": + raise ConnectionError("version lookup failed") + return _server_version_at_least("1.1.0", minimum) + + frbc._fm_client.server_version_at_least = AsyncMock(side_effect=version_at_least) + frbc._fm_client.trigger_and_get_schedule.side_effect = ValueError( + "Request failed with status code 400" + ) + + with pytest.raises(ValueError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == 1 + assert "left without an instruction" in caplog.text diff --git a/tests/s2/test_frbc_unsupported_is_said.py b/tests/s2/test_frbc_unsupported_is_said.py index 6815b6e2..79271586 100644 --- a/tests/s2/test_frbc_unsupported_is_said.py +++ b/tests/s2/test_frbc_unsupported_is_said.py @@ -122,3 +122,52 @@ async def test_usage_is_planned_with_only_where_the_resource_manager_says_it_sen flex_model = await _flex_model(frbc, system_description) assert ("soc-usage" in flex_model) is provided + + +@pytest.mark.asyncio +async def test_a_usage_forecast_sent_despite_the_flag_is_planned_with(): + """What a resource manager sends says more than the flag it set once.""" + from datetime import timedelta + + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + system_description = _heater() # provides_usage_forecast=False + frbc = _frbc(system_description, usage_forecast_sensor_id=7) + forecast = FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration.from_timedelta(timedelta(hours=1)), + usage_rate_expected=0.1, + ) + ], + ) + frbc._usage_forecast_history[str(forecast.message_id)] = forecast + + flex_model = await _flex_model(frbc, system_description) + + assert "soc-usage" in flex_model + + +@pytest.mark.asyncio +async def test_a_timer_is_warned_about_once(caplog): + """A resource manager sends a status on every run of every timer.""" + system_description = _heater() + frbc = _frbc(system_description) + timer_id = get_unique_id() + + def status(): + return FRBCTimerStatus( + message_id=get_unique_id(), + timer_id=timer_id, + actuator_id=system_description.actuators[0].id, + finished_at=datetime(2024, 1, 1, 0, 5, tzinfo=timezone.utc), + ).to_dict() + + with caplog.at_level(logging.WARNING): + await frbc.handle_message(status()) + await frbc.handle_message(status()) + + assert caplog.text.count("does not honour") == 1 + assert len(frbc._timer_status_history) == 2 From 2557896278469b10cf55221663d3ce8752b46d50 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 00:59:19 +0100 Subject: [PATCH 68/87] fix(S2): housekeeping, and tests that pin what they claim to Package P2.4 of the plan to finish #188. - C5: SizeLimitOrderedDict made room with popitem(), which takes the newest entry, so a full history kept a session's first entries and lost every later one but the last. It now evicts the oldest, and a key already present takes no room of its own. - C6: an actuator status arriving before any description, as one can after a reconnection, raised an IndexError in a task nobody awaited; it is now said and skipped, as its sibling does. - Tests that a mutation showed assert nothing: the opt-out of waiting for ingestion (the poll's ConnectionError is swallowed, so an unregistered URL never failed the test), post_measurements passing the opt-out on, and a synchronous 200 not being polled (its payload named no job) now check the requests made. Newly pinned: a 0.33.0 server's job_id, the file upload's ingestion wait, the default of five attempts, and the fill-level scale in the bounds a schedule is read back against. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 7 ++ src/flexmeasures_client/s2/utils.py | 15 +++- tests/client/test_sensor.py | 82 ++++++++++++++++++- tests/s2/test_frbc_actuator_measurement.py | 15 ++++ tests/s2/test_frbc_schedule_retry.py | 35 ++++++++ tests/s2/test_size_limit_ordered_dict.py | 22 +++++ 6 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 tests/s2/test_size_limit_ordered_dict.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index dc95f20a..187f788d 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -631,6 +631,13 @@ def _latest_fill_level(self) -> float | None: async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor + # After a reconnection an actuator status can come before any description, + # which this needs to tell what power the status stands for. + if not self._system_description_history: + self._logger.warning( + "Not recording the power of an FRBC.ActuatorStatus: no FRBC.SystemDescription has arrived yet to tell what its operation mode draws." + ) + return sd: FRBCSystemDescription = list(self._system_description_history.values())[-1] # Report the mode the actuator is ACTUALLY in, not whichever the system description happens diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index cfdcd55e..8ac6233d 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -28,13 +28,20 @@ def __init__(self, *args, max_size=100, **kwargs): super(SizeLimitOrderedDict, self).__init__(*args, **kwargs) self._max_size = max_size - # deleting values to make the dictionary of length _max_size at most + # deleting values to make the dictionary of length _max_size at most, oldest first while len(self) > self._max_size: - self.popitem() + self.popitem(last=False) def __setitem__(self, __key: KT, __value: VT) -> None: - if len(self) == self._max_size: - self.popitem() + # The oldest entry makes room, not the newest, which popitem() takes by default: + # a full history otherwise kept a session's first entries and lost every later one but the last. + # A key already present takes no room of its own. + if ( + self._max_size is not None + and __key not in self + and len(self) >= self._max_size + ): + self.popitem(last=False) return super().__setitem__(__key, __value) diff --git a/tests/client/test_sensor.py b/tests/client/test_sensor.py index d8181e96..a4e66261 100644 --- a/tests/client/test_sensor.py +++ b/tests/client/test_sensor.py @@ -1079,9 +1079,9 @@ async def test_post_sensor_data_synchronous_response_is_not_polled(): m.post( f"http://localhost:5000/api/v3_0/sensors/{sensor_id}/data", status=200, - payload={"status": "PROCESSED", "message": "ok"}, + # Naming a job all the same, so that only the status keeps it from being polled. + payload={"status": "PROCESSED", "message": "ok", "job": "job-9"}, ) - # No job endpoint is registered, so a poll would fail the test by raising here. response, status = await client.post_sensor_data( sensor_id=sensor_id, @@ -1092,6 +1092,7 @@ async def test_post_sensor_data_synchronous_response_is_not_polled(): ) assert status == 200 + assert not any(method == "GET" for method, _ in m.requests), "nothing is polled" await client.close() @@ -1112,7 +1113,8 @@ async def test_post_sensor_data_await_ingestion_false_is_not_polled(): "job": "job-4", }, ) - # No job endpoint is registered, so a poll would fail the test by raising here. + # An unregistered job endpoint would not fail the test by itself: + # the poll's ConnectionError is swallowed by the ingestion wait, so the requests are checked instead. response, status = await client.post_sensor_data( sensor_id=sensor_id, @@ -1124,6 +1126,7 @@ async def test_post_sensor_data_await_ingestion_false_is_not_polled(): ) assert status == 202 + assert not any(method == "GET" for method, _ in m.requests), "nothing is polled" await client.close() @@ -1246,6 +1249,79 @@ async def test_post_measurements_awaits_ingestion_and_can_opt_out(): unit="MW", await_ingestion=False, ) + assert not any( + method == "GET" and url.path == "/api/v3_0/jobs/job-8" + for method, url in m.requests + ), "the opt-out is passed on" + await client.close() + + +@pytest.mark.asyncio +async def test_post_sensor_data_awaits_a_job_named_the_way_0_33_named_it(): + """FlexMeasures 0.33.0 named the job ``job_id``, before 1.0.0 settled on ``job``.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post( + "http://localhost:5000/api/v3_0/sensors/5/data", + status=202, + payload={"status": "ACCEPTED", "job_id": "job-10"}, + ) + m.get( + "http://localhost:5000/api/v3_0/jobs/job-10", + status=200, + payload={"status": "FINISHED", "message": "ok", "result": None}, + ) + + await client.post_sensor_data( + sensor_id=5, + start="2023-01-01T00:00+00:00", + duration="PT1H", + values=[1.0], + unit="MW", + ) + + assert any( + method == "GET" and url.path == "/api/v3_0/jobs/job-10" + for method, url in m.requests + ) + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("await_ingestion", [True, False]) +async def test_a_file_upload_awaits_its_ingestion_unless_told_not_to( + await_ingestion, tmp_path +): + csv_path = tmp_path / "data.csv" + csv_path.write_text("datetime,value\n2023-01-01T00:00+00:00,1.0\n") + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.server_version = "1.1.0" + m.post( + "http://localhost:5000/api/v3_0/sensors/1/data/upload", + status=202, + payload={"status": "ACCEPTED", "job": "job-11"}, + ) + m.get( + "http://localhost:5000/api/v3_0/jobs/job-11", + status=200, + payload={"status": "FINISHED", "message": "ok", "result": None}, + ) + + await client.post_sensor_data( + sensor_id=1, + file_path=str(csv_path), + unit="kW", + await_ingestion=await_ingestion, + ) + + polled = any( + method == "GET" and url.path == "/api/v3_0/jobs/job-11" + for method, url in m.requests + ) + assert polled is await_ingestion await client.close() diff --git a/tests/s2/test_frbc_actuator_measurement.py b/tests/s2/test_frbc_actuator_measurement.py index 55319525..3913182b 100644 --- a/tests/s2/test_frbc_actuator_measurement.py +++ b/tests/s2/test_frbc_actuator_measurement.py @@ -348,3 +348,18 @@ async def test_a_unit_nothing_converts_between_is_reported(system_description, c # so that is the unit it is reported in: a pair that disagrees is worse than none. assert f"{ON_W} W" in caplog.text assert f"{ON_W} hp" not in caplog.text + + +@pytest.mark.asyncio +async def test_an_actuator_status_before_any_description_is_said_and_skipped( + system_description, caplog +): + """After a reconnection one can arrive first; indexing the empty history raised an IndexError in a task nobody awaited.""" + frbc = _frbc(system_description, consumption_as_positive=True) + status = _status(system_description, 0) + frbc._system_description_history.clear() + + await frbc.send_actuator_status(status) + + assert _posts_to_the_power_sensor(frbc) == [] + assert "no FRBC.SystemDescription has arrived yet" in caplog.text diff --git a/tests/s2/test_frbc_schedule_retry.py b/tests/s2/test_frbc_schedule_retry.py index b93a81c2..3c903f5b 100644 --- a/tests/s2/test_frbc_schedule_retry.py +++ b/tests/s2/test_frbc_schedule_retry.py @@ -388,3 +388,38 @@ def test_a_schedule_per_device_names_no_scheduler_to_check(caplog): frbc._warn_where_a_fallback_scheduler_answered([SCHEDULE, SCHEDULE], VALID_FROM) assert caplog.text == "" + + +@pytest.mark.asyncio +async def test_a_schedule_is_asked_for_five_times_by_default(): + """Pinned here, since every other test imports the default rather than stating it.""" + system_description = _heater() + frbc = _frbc(system_description) + frbc._fm_client.trigger_and_get_schedule.side_effect = A_FAILED_JOB + + with pytest.raises(JobFailedError): + await frbc.trigger_schedule(system_description.message_id) + + assert _attempts(frbc) == 5 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scale", [1.0, 10.0]) +async def test_the_bounds_a_schedule_is_read_back_against_are_in_the_resource_manager_s_units( + scale, +): + """At a scale of ten, read back unscaled, the translation clamped at ten times the true hard maximum.""" + from unittest.mock import patch + + system_description = _heater() + frbc = _frbc(system_description, fill_level_scale=scale) + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + + assert translate.call_args.kwargs["fill_level_bounds"] == ( + 0.0, + pytest.approx(RANGE_TOP * 1.5), + ) diff --git a/tests/s2/test_size_limit_ordered_dict.py b/tests/s2/test_size_limit_ordered_dict.py new file mode 100644 index 00000000..8d2e4689 --- /dev/null +++ b/tests/s2/test_size_limit_ordered_dict.py @@ -0,0 +1,22 @@ +from flexmeasures_client.s2.utils import SizeLimitOrderedDict + + +def test_a_full_history_keeps_the_newest_entries(): + """popitem() takes the newest by default, so a full history kept a session's first entries instead.""" + history = SizeLimitOrderedDict(max_size=3) + for n in range(5): + history[n] = n + assert list(history) == [2, 3, 4] + + +def test_setting_a_key_already_present_evicts_nothing(): + history = SizeLimitOrderedDict(max_size=2) + history["a"] = 1 + history["b"] = 2 + history["b"] = 3 + assert dict(history) == {"a": 1, "b": 3} + + +def test_made_from_more_than_fits_it_keeps_the_last(): + history = SizeLimitOrderedDict([(n, n) for n in range(5)], max_size=2) + assert list(history) == [3, 4] From e2d5e8d711dc0528cd943d9f82a602ce09ce1a82 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 01:09:30 +0100 Subject: [PATCH 69/87] fix(FRBC): guard the display-only band posts; say a negative top needs no head room of its own The fill-level band a schedule is planned against goes out inline since the declared range reaches the server, so the minima and maxima posts are for display; a failure in one is now logged, and the other still recorded, rather than raised in a task nobody awaits. hard_bound_factor stays multiplicative: a top below zero would be pulled further down, but the bound widens to take in the start, which is never below zero, so the mutation that tried the change showed it to make no difference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ .../s2/control_types/FRBC/frbc_simple.py | 27 +++++++++++++------ tests/s2/test_frbc_declared_profiles.py | 14 ++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index ac8441eb..c6c62bb9 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -61,6 +61,8 @@ Removed Fixed ----- +* A full message history keeps the newest messages: it made room by evicting the newest entry, so it kept a session's first messages and lost every later one but the last [`#274 `__] +* An FRBC actuator status that arrives before any system description, as one can after a reconnection, is logged and skipped, rather than raising in a background task. A fill-level target profile whose minima or maxima could not be recorded is logged, and the other bound is still recorded [`#274 `__] * A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 09fba884..801d77f0 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -1136,6 +1136,8 @@ async def _plan( _, range_top = get_soc_min_max(system_description, self._fill_level_scale) # Zero below, since a stock cannot be negative and FlexMeasures accepts no lower floor. soc_min = 0.0 + # Multiplied, a top below zero, as S2 allows, would be pulled further down, + # but the bound widens below to take in the state of charge at the start, which is never below zero. soc_max = range_top * self._hard_bound_factor # A state of charge outside the hard bounds is infeasible before the first step, # so the bounds widen to take it in. @@ -1357,18 +1359,27 @@ async def send_fill_level_target_profile( # that comes back begins there, not at the start_time the resource manager sent. Posting # it at the raw start_time would shift every value by the difference. start = self.align_to_resolution(fill_level_target_profile.start_time) + # Recorded to be seen, since the band a schedule is planned against goes out inline, + # so a failure is logged and the other bound still recorded, rather than raised in a task nobody awaits. for sensor_id, values in ( (self._soc_minima_sensor_id, soc_minima), (self._soc_maxima_sensor_id, soc_maxima), ): - await self._fm_client.post_sensor_data( - sensor_id=sensor_id, - start=start, - prior=now, - values=values.tolist(), - unit=self.energy_unit, - duration=duration, - ) + try: + await self._fm_client.post_sensor_data( + sensor_id=sensor_id, + start=start, + prior=now, + values=values.tolist(), + unit=self.energy_unit, + duration=duration, + ) + except REQUEST_FAILURES: + self._logger.exception( + "Could not record a bound of the fill-level target profile on sensor %s from %s.", + sensor_id, + start, + ) async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): """Record what a resource manager expects its storage to be drawn down by.""" diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index 1d656d25..1d3acda0 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -696,3 +696,17 @@ async def test_a_storage_efficiency_whose_ingestion_is_unconfirmed_is_not_rememb assert ( frbc._fm_client.post_sensor_data.await_args.kwargs["confirm_ingestion"] is True ) + + +@pytest.mark.asyncio +async def test_a_bound_that_could_not_be_recorded_leaves_the_other_recorded( + system_description, caplog +): + """The band is planned against inline, so these are for display, and one failing post is no reason to lose the other.""" + frbc = _frbc(system_description, with_sensors=True) + frbc._fm_client.post_sensor_data.side_effect = [ConnectionError("down"), ({}, 200)] + + await frbc.send_fill_level_target_profile(_target_profile(40.0, 60.0)) + + assert frbc._fm_client.post_sensor_data.await_count == 2 + assert "Could not record a bound" in caplog.text From ab2b0dc1d2474896a0262ed616ba87703c13f46c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 01:17:48 +0100 Subject: [PATCH 70/87] fix(S2): a message set again is the latest; what the review of housekeeping found - A key set again in a SizeLimitOrderedDict stayed where it was, so a resource manager reusing a message id left [-1] reading a stale message; it now moves to the end. - The band-post test checks that the second post, with the maxima, went out. - The changelog gives the actuator status and the band post a bullet each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 5 +++-- src/flexmeasures_client/s2/utils.py | 4 +++- tests/s2/test_frbc_declared_profiles.py | 2 ++ tests/s2/test_size_limit_ordered_dict.py | 6 +++--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c6c62bb9..e73479db 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -61,8 +61,9 @@ Removed Fixed ----- -* A full message history keeps the newest messages: it made room by evicting the newest entry, so it kept a session's first messages and lost every later one but the last [`#274 `__] -* An FRBC actuator status that arrives before any system description, as one can after a reconnection, is logged and skipped, rather than raising in a background task. A fill-level target profile whose minima or maxima could not be recorded is logged, and the other bound is still recorded [`#274 `__] +* A full message history keeps the newest messages: it made room by evicting the newest entry, so it kept a session's first messages and lost every later one but the last. A message set again under the same id becomes the latest [`#274 `__] +* An FRBC actuator status that arrives before any system description, as one can after a reconnection, is logged and skipped, rather than raising in a background task [`#274 `__] +* A fill-level target profile whose minima or maxima could not be recorded is logged, and the other bound is still recorded [`#274 `__] * A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index aa810eeb..0da31bcb 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -43,7 +43,9 @@ def __setitem__(self, __key: KT, __value: VT) -> None: ): self.popitem(last=False) - return super().__setitem__(__key, __value) + super().__setitem__(__key, __value) + # Set again, an entry is the latest, which is what reading the last entry means everywhere this is used. + self.move_to_end(__key) def get_unique_id() -> str: diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index 1d3acda0..035cadef 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -709,4 +709,6 @@ async def test_a_bound_that_could_not_be_recorded_leaves_the_other_recorded( await frbc.send_fill_level_target_profile(_target_profile(40.0, 60.0)) assert frbc._fm_client.post_sensor_data.await_count == 2 + second = frbc._fm_client.post_sensor_data.await_args_list[1].kwargs + assert second["sensor_id"] == frbc._soc_maxima_sensor_id assert "Could not record a bound" in caplog.text diff --git a/tests/s2/test_size_limit_ordered_dict.py b/tests/s2/test_size_limit_ordered_dict.py index 8d2e4689..be2de5d3 100644 --- a/tests/s2/test_size_limit_ordered_dict.py +++ b/tests/s2/test_size_limit_ordered_dict.py @@ -9,12 +9,12 @@ def test_a_full_history_keeps_the_newest_entries(): assert list(history) == [2, 3, 4] -def test_setting_a_key_already_present_evicts_nothing(): +def test_setting_a_key_already_present_evicts_nothing_and_makes_it_the_latest(): history = SizeLimitOrderedDict(max_size=2) history["a"] = 1 history["b"] = 2 - history["b"] = 3 - assert dict(history) == {"a": 1, "b": 3} + history["a"] = 3 + assert list(history.items()) == [("b", 2), ("a", 3)] def test_made_from_more_than_fits_it_keeps_the_last(): From f15b9e2a9646fde643cdc3e078a966de41e4684c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 01:24:18 +0100 Subject: [PATCH 71/87] fix(S2): what the sweep of the whole branch found - Five comments broke a line mid-phrase, against the rule that a line ends at punctuation; each pair is now one line. A scan of every comment the branch adds finds no others. - ensure_site fell back to the listing where a known asset could not be fetched, but not where the server answered with something other than an asset (ContentTypeError), which escaped and failed the setup. Left as they are, and why: FRBC TUNES keeps its broad except clauses and its 15-minute tick, which predate this branch and are its own story; usage_rate_at is used by the end-to-end example harness. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/__init__.py | 3 +-- .../s2/control_types/FRBC/frbc_simple.py | 9 +++------ src/flexmeasures_client/s2/control_types/FRBC/utils.py | 3 +-- src/flexmeasures_client/s2/site.py | 3 ++- tests/s2/test_site.py | 9 +++++++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index f13c99a4..c81901cd 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -258,8 +258,7 @@ def handle_system_description( # is answered with a schedule against that latest status. # Anything but the message id counts as a difference, its validity included, # so a resource manager that re-dates its description on every resend asks for a schedule every time. - # Otherwise a resource manager that sends its status first, or whose fill level holds steady - # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. + # Otherwise a resource manager that sends its status first, or whose fill level holds steady so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. if self._storage_status_history: request_number = self._count_request(STORAGE_STATUS_REQUESTS) task = asyncio.create_task( diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 801d77f0..63b7eaa1 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -369,8 +369,7 @@ def align_to_resolution(self, moment: datetime) -> datetime: return moment # Floored on the moment's own clock, in the offset it carries, which is how FlexMeasures floors a series posted with it, # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. - # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two - # that is not after the moment: taking the first pass every time put a boundary an hour or more too early. + # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two that is not after the moment: taking the first pass every time put a boundary an hour or more too early. # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. # Compared as pandas timestamps, which compare instants: two datetimes on one local clock compare by wall time, # whichever pass each is on. @@ -452,8 +451,7 @@ async def send_storage_status( await self._fm_client.post_measurements( self._soc_sensor_id, start=now, - # Scaled like every other fill level this handler records, or the level and the - # band around it would be in different scales wherever fill_level_scale is not 1. + # Scaled like every other fill level this handler records, or the level and the band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, duration=await self._soc_duration(), @@ -1446,8 +1444,7 @@ async def _record_storage_efficiency( or not self._system_description_history ): return False - # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate - # only at one fill level, and the loss scales with the stock away from it. + # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate only at one fill level, and the loss scales with the stock away from it. # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: # from a near-empty start, a share of the start level would let almost nothing remain per step, # and the loss would be overstated many times over as soon as the plan filled the storage. diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index ab721e9e..aef2c27a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -587,8 +587,7 @@ def power_to_fill_rate_with_metrics( if pr_end == pr_start: # Every factor draws this one power, so the power cannot say which factor is meant. - # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency - # declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. + # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. # Where the fill-rate range is a single value too, the factor makes no difference and stays at zero. fr_start = best_element.fill_rate.start_of_range fr_end = best_element.fill_rate.end_of_range diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py index f94185f0..cf3f783a 100644 --- a/src/flexmeasures_client/s2/site.py +++ b/src/flexmeasures_client/s2/site.py @@ -21,6 +21,7 @@ ) from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple _LOGGER = getLogger(__name__) @@ -109,7 +110,7 @@ async def ensure_site( candidate = await fm_client.get_asset( known_asset_id, parse_json_fields=True ) - except (ValueError, ConnectionError): + except (ValueError, ConnectionError, ContentTypeError): candidate = None if candidate is not None and candidate.get("name") == name: site_asset = candidate diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py index c67dbad9..69676bb3 100644 --- a/tests/s2/test_site.py +++ b/tests/s2/test_site.py @@ -10,6 +10,7 @@ from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.site import ( FRBC_SITE_SENSORS, @@ -331,10 +332,14 @@ async def forever(details): @pytest.mark.asyncio -async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): +@pytest.mark.parametrize( + "failure", + [ValueError("404"), ContentTypeError("Expected an asset, but got a list")], +) +async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(failure): """Deleted since, say: the listing finds it by name, or it is made again.""" fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) - fm.get_asset = AsyncMock(side_effect=ValueError("404")) + fm.get_asset = AsyncMock(side_effect=failure) asset, _ = await ensure_site( fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 ) From fa18fdd58250a6be24c9449fc86f17bf3762bade Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 01:25:22 +0100 Subject: [PATCH 72/87] docs(changelog): say once what the S2 integration ships since v0.9.0 Rewrite the Unreleased section of CHANGELOG-S2.rst so that each feature has one entry, described against v0.9.0 rather than against states that never shipped, and checked against the code at this commit. Folded, keeping every PR link: the newest-request entries (#237, #261, #269); the two subclass signature changes (#237, #240); the power bands and their v1.0.0 gate (#248, #272); the declared fill-level band, usage, leakage and hard bounds (#244, #245, #259, #271, #273); retrying a schedule (#254, #273); the state-of-charge unit (#244); the actuator's power, its sensor, sign, unit, step and belief time (#236, #252, #256, #258, #259); the example's sensors, instruction handling and prices (#241, #252, #259, #262); target-profile recording (#244, #274); revocations and the instruction-status history (#237); the planning triggers, re-sent descriptions and the guarded state-of-charge post (#240, #259, #273); the schedule start and DST boundary (#240, #259); the resolution and its validation (#240, #258); and leakage_behaviour_to_storage_efficiency (#256, #258, #259). The stray second Added list is merged into the first, and get_validation_error_summary moves to Removed. Fixes to behaviour that never shipped no longer have entries of their own. Corrected: the declared range now goes out inline whatever the sensors, so soc-minima and soc-maxima are no longer said to be named by sensor; soc-usage is named only where the resource manager provides a usage forecast; the retry entry no longer claims a missing state-of-charge refusal applies to FRBCSimple, which names no state-of-charge sensor; the fallback entry no longer claims a lost state of charge for later windows; a multi-actuator description is compared with v0.9.0, which planned for the first actuator; the schedule start and the example's prices describe what the code does now; and the hard bounds say plainly that v0.9.0 held a storage hard to its declared range. Added: fill_level_scale and power_unit; rm_discharge_sensor_id no longer written to; the breaking signature changes in FRBC.utils; the new keyword arguments of fm_schedule_to_instructions and leakage_behaviour_to_storage_efficiency; is_revoked finding a revoked object; schedule_attempts below one refused; and the new public FRBC methods. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 99 ++++++++++++++++++------------------------------ 1 file changed, 36 insertions(+), 63 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index e73479db..6b5bc82b 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -16,15 +16,20 @@ Other changes: see `Unreleased in the changelog `__] -* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] +* ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage, for it to be seen and planned with. A fill-level target profile is recorded on the minima and maxima sensors, only where both are given, on the handler's ``resolution`` steps from the step its start falls in, and held inside the declared fill-level range: a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows, and a target band lying wholly outside the range is recorded at the nearest end of it, with a warning. Where one of the two bounds cannot be recorded, that is logged, and the other is still recorded. A usage forecast is recorded in ``/s``, a negative expected usage as none, with a warning, and a leakage behaviour is recorded as a storage efficiency each time a schedule is asked for (see Changed). Without a sensor, that declaration is not recorded [`#244 `__, `#259 `__, `#271 `__, `#274 `__] +* ``FRBCSimple`` takes an ``energy_unit``, defaulting to ``kWh``, which a state of charge, the fill-level band around it, a fill rate and a usage forecast (the latter two per second) are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] +* ``FRBCSimple`` takes a ``fill_level_scale``, defaulting to 1, which every fill level and fill rate it sends to FlexMeasures is multiplied by: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule back into instructions keeps the resource manager's own units, since it matches fill levels against the operation modes stated in them. A scale of zero or below is refused. It also takes a ``power_unit``, defaulting to ``W``, which an actuator's power is converted into and labelled with when it is posted [`#236 `__, `#244 `__, `#245 `__, `#252 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] -* ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] -* ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all, and a mode reserved for abnormal conditions is not counted. It is recorded once per window, and recorded again for the next schedule where FlexMeasures did not confirm storing it; where it cannot be recorded, the schedule is asked for without it, and the log says so [`#253 `__, `#256 `__, `#272 `__, `#273 `__] +* ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two at a given fill level. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__, `#256 `__] +* ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for and how long between two attempts; fewer than one attempt is refused. v0.9.0 asked once, so any failure cost the storage its instruction for that window, while the resource manager kept polling for it. A scheduling job that failed inside the server is asked for again, unless its cause fails identically on every attempt, as a scheduler colliding with its own belief or an infeasible problem does. So is a server that could not be reached or answered with a 5xx, and a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A job no worker picked up in time is asked about once more. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] +* An FRBC handler tracks the instruction batch the resource manager is working from, and, with its ``send_revocations`` attribute set to ``True``, revokes the instructions of a batch it replaces that are still pending, that is, with no status reported yet, or reported as ``NEW`` or ``ACCEPTED``. Off by default, since a resource manager is not required to honour revocations [`#237 `__] +* ``FRBC.filter_instruction_transitions()``, ``FRBC.send_instruction_batch()`` and ``FRBC.system_description_fingerprint()``, which a handler uses to drop instructions that repeat their predecessor, to send a batch that replaces the one before it (dropped where a newer request arrived meanwhile), and to tell a changed system description from a re-sent one [`#237 `__] +* ``ControlTypeHandler.send_message()``, which handlers use instead of putting messages on the CEM's sending queue themselves [`#237 `__] +* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there, as a rate of fill level in ``/s`` rather than in a power unit, since S2 states a fill rate as fill level per second, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] * ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers and the moment a state of charge is known as of included [`#264 `__, `#268 `__] * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] @@ -35,85 +40,53 @@ Added Changed ------- -* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile, and a request out of date makes no further attempt after a failed one. Each status used to start a solve of its own beside the one in flight, so the newest is now planned after the attempt in flight rather than beside it [`#269 `__] -* An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] -* ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] -* ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] -* ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes, which a schedule's start is lined up with [`#240 `__] -* ``FRBCSimple`` asks for a schedule within the power ranges the actuator declares: a ``consumption-capacity`` and a ``production-capacity`` from the widest range it states, and each distinct range as one of the ``operation-modes``, except where the ranges together cover exactly what the two capacities allow, without a gap, and so say nothing new while each costs the solver a binary per step. Modes S2 reserves for abnormal conditions are left out of both, and are never instructed into, since a resource manager refuses an instruction into one that is not marked abnormal; an actuator with no other mode is not planned for, with a warning. The scheduler then plans only power values the actuator can run at, instead of a fractional power that a resource manager has to round into a full-on block, overshooting the capacity its site is held to. An actuator that cannot produce at all is given a production capacity of zero, where before its storage could be planned as if it were a battery feeding the grid [`#248 `__, `#272 `__] -* The ``operation-modes`` field is asked for only of a FlexMeasures at v1.0.0 or newer, which is the release that added it. An older server is asked for a schedule without the power bands, and told so in the log, since a flex model naming a field the server does not know is refused outright and leaves the resource manager with no schedule at all [`#248 `__] -* ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] -* ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] -* ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, sent with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, the latest fill-level target profile applying where it covers a step, held inside the range, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to, and the declared range applies again beyond its end. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, and the log says so. The hard ``soc-min`` and ``soc-max`` become wide bounds at zero and half again above the range, the top widened to take in a state of charge above it, and a storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band. A negative usage forecast, a gain FlexMeasures cannot relax, is recorded as no usage, with a warning that counts the steps [`#245 `__, `#271 `__] -* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] -* ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A scheduling job no worker picked up in time is asked about once more, a server that could not be reached or answered with a 5xx is asked again, and so is a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] -* ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] -* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where before it acknowledged the description and then failed after each solve, in a task nobody awaited. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every status is now kept. ``FRBCSimple`` plans with the usage forecast sensor only where the resource manager says it provides a usage forecast, or has sent one regardless [`#273 `__] -* ``FRBCSimple`` plans from a storage status even where recording its state of charge failed, which is logged, since the schedule starts from the level the status reports, sent inline; before, the failed post cancelled the planning it preceded. A charging or storage efficiency whose ingestion could not be confirmed is no longer remembered as recorded, so it is posted again for the next plan rather than never [`#273 `__] +* Breaking for subclasses: ``FRBC.trigger_schedule()`` and ``FRBC.send_storage_status()`` each take a ``request_number`` keyword argument, the number of the planning request they answer, so a subclass that overrides either has to accept it [`#237 `__, `#240 `__] +* Breaking: in ``flexmeasures_client.s2.control_types.FRBC.utils``, ``compute_next_fill_level()`` is now ``compute_next_fill_level(fill_level, fill_rate, step, leakage_behaviour=None, usage_rate=0.0)``, with the step a ``timedelta`` and every rate per second, where it took ``(fill_level, fill_rate, leakage_behaviour, deltaT, usage)`` with ``deltaT`` in hours; and ``power_to_fill_rate_with_metrics()`` returns six values, the operation-mode factor added, instead of five, as do the tuples ``explain_choice()`` takes [`#256 `__, `#258 `__] +* ``fm_schedule_to_instructions()`` takes three new keyword arguments. ``fill_level_bounds`` are the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned; without them the declared range is used, as before. ``leakage_behaviour`` and ``usage_forecast`` lower that running fill level as the storage leaks and is drawn from [`#245 `__, `#258 `__] +* An FRBC handler plans one schedule at a time, for the newest request only. In v0.9.0 every system description started a solve of its own, and every solve sent its own instruction batch, however many newer ones had arrived meanwhile. Now a request arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile; a request out of date makes no further attempt after a failed one, is not turned into instructions, and sends no batch [`#237 `__, `#261 `__, `#269 `__] +* An FRBC handler asks for a schedule on every storage status, since the state of charge it reports is what a schedule starts from, and on a system description that arrives after the latest status and differs from the one before it, its validity included. So a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves. v0.9.0 planned on system descriptions only, every one of them, so a resource manager re-sending an unchanged description was planned for again each time: now such a description is acknowledged and otherwise ignored. A state of charge that cannot be recorded is logged, and planned from all the same [`#240 `__, `#259 `__, `#273 `__] +* ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes: the step a schedule is planned on and its start lined up with, and the step a fill-level target profile, a usage forecast and a storage efficiency are recorded on, and an actuator's power and fill rate are recorded as covering. A ``resolution`` or ``schedule_duration`` that is not positive is refused, and so is a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only. In v0.9.0, whatever the power sensor's resolution, a schedule was read as quarter hours, and an actuator's status recorded as covering one [`#240 `__, `#258 `__] +* A schedule starts at a resolution boundary: the one at or before the system description's ``valid_from`` plus ``valid_from_shift``, or, where that is in the past, the next boundary after now. ``valid_from_shift`` still defaults to one day. v0.9.0 started a schedule at ``valid_from`` plus the shift, unaligned, which left FlexMeasures without prices for a window off its steps. A boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment [`#240 `__, `#259 `__] +* ``FRBCSimple`` asks for a schedule within the power ranges the actuator declares: a ``consumption-capacity`` and a ``production-capacity`` from the widest range it states, and, from a FlexMeasures at v1.0.0 or newer, which added the field, each distinct range as one of the ``operation-modes``, except where the ranges together cover exactly what the two capacities allow, without a gap, and so say nothing new while each costs the solver a binary per step. An older server is asked without the power bands, and the log says so, since a flex model naming a field the server does not know is refused outright. The scheduler then plans only power values the actuator can run at, instead of a fractional power that a resource manager has to round into a full-on block, overshooting the capacity its site is held to. An actuator that cannot produce at all is given a production capacity of zero, where before its storage could be planned as if it were a battery feeding the grid. Modes S2 reserves for abnormal conditions are left out of both, and are never instructed into, since a resource manager refuses an instruction into one that is not marked abnormal; an actuator with no other mode is not planned for, with a warning [`#248 `__, `#272 `__] +* ``FRBCSimple`` holds a storage to its declared fill-level range as a band whose breach is priced, rather than as hard bounds, and plans against what the resource manager declared about its storage. The band goes out with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, with ``relax-soc-constraints`` in the flex context: the declared range, with the latest fill-level target profile, held inside the range, applying where it covers a step, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to. It goes out with or without the band sensors, which only record it. The hard ``soc-min`` and ``soc-max`` are now zero and ``hard_bound_factor`` times the top of the range, widened to take in a state of charge above it, where v0.9.0 held the storage hard to the declared range: a state a storage really reaches, such as free-floating above the top of its range or being left below the bottom of it, was infeasible, and answered with no schedule at all, or by a fallback scheduler. A storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, with a note in the log, so the storage is planned anywhere between the wide hard bounds. The usage forecast is named as ``soc-usage`` where its sensor is given and the resource manager says it provides one, or has sent one regardless. A leakage behaviour is recorded as a ``storage-efficiency`` each time a schedule is asked for, over the window the schedule covers, from the element that applies at the fill level it starts from, as a share of that fill level or of the middle of the storage's range where it starts below that, and named only where FlexMeasures confirmed storing it. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#244 `__, `#245 `__, `#259 `__, `#271 `__, `#273 `__] +* ``FRBCSimple`` posts a state of charge in its ``energy_unit``, ``kWh`` by default, and states the flex model's ``soc-unit`` to match (``kWh`` where ``energy_unit`` is neither ``kWh`` nor ``MWh``), where v0.9.0 labelled every state of charge, and the ``soc-unit`` of every schedule, ``MWh``, whatever unit the resource manager reported in: against the example server's ``kWh`` sensor, a fill level of 10 was recorded as 10,000. A resource manager that reports in megawatt hours now needs ``energy_unit="MWh"`` [`#244 `__] +* ``FRBCSimple`` records an actuator's power on the power sensor, and no longer writes anything to ``rm_discharge_sensor_id``, which stays a required argument. v0.9.0 posted minus the fill rate of the first-listed operation mode, labelled ``MWh``, onto that discharge sensor. Now it posts the power of the operation mode the actuator reports it is in, from the power range of the element that applies at the latest fill level, interpolated by the reported factor, as covering one ``resolution`` step. It is posted in the sign the power sensor records consumption in, read from the sensor's ``consumption_is_positive`` attribute (as S2 states it where the sensor cannot be read, with the reason logged), and converted into ``power_unit``. A second status for the same step supersedes the first rather than colliding with it, and a power that cannot be recorded is logged with its value, unit, sensor and moment [`#236 `__, `#252 `__, `#256 `__, `#258 `__, `#259 `__] +* ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for, though it is a coarse charging policy rather than a plan [`#254 `__] +* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where v0.9.0 acknowledged it and planned for the first actuator alone. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every timer status is now kept [`#273 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions. It acknowledges each instruction, and reports the transition it asks for when the instruction's ``execution_time`` arrives, rather than printing it and carrying on; reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds; follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked; and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power point the same way, seeds its demo prices from the current day and again every 12 hours, so that the window it plans for keeps its prices, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] Removed ------- * ``flexmeasures_client.s2.cem.get_commodity_unit``, which stated power in kW where S2 states it in W [`#266 `__] * ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] +* ``flexmeasures_client.s2.utils.get_validation_error_summary``, which iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given [`#257 `__] Fixed ----- * A full message history keeps the newest messages: it made room by evicting the newest entry, so it kept a session's first messages and lost every later one but the last. A message set again under the same id becomes the latest [`#274 `__] +* ``Handler.is_revoked()`` finds an object the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] * An FRBC actuator status that arrives before any system description, as one can after a reconnection, is logged and skipped, rather than raising in a background task [`#274 `__] -* A fill-level target profile whose minima or maxima could not be recorded is logged, and the other bound is still recorded [`#274 `__] * A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] -* A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] +* A CEM without a sensor for a resource manager's measured power no longer records an L1 measurement on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] -* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] -* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] -* A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] +* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection, where a session used to keep working after its resource manager had disconnected [`#263 `__] +* A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] +* An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks [`#244 `__] +* ``FRBCSimple`` asks for a schedule in watts, which is what S2 states an operation mode's power ranges in, rather than in whatever unit the power sensor happens to be recorded in. The example server creates its power sensor in ``kW``, so every scheduled power was read as a thousandth of itself: it then sat nearest the operation mode that draws nothing, and the resource manager was instructed to idle through a schedule that asked it to run [`#247 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] -* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] -* ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] * Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] +* Translating a schedule into instructions no longer depends on the order a mode lists its elements in: where two elements share a boundary, the one that applies answers, not the first listed [`#256 `__] * A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, and only there, rather than wherever the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#237 `__, `#258 `__] * An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] -* A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] -* An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] +* ``leakage_behaviour_to_storage_efficiency()`` compounds a leakage rate per second, as S2 states it, where it compounded it per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002%. It takes the rate of the element that applies at a new ``fill_level`` keyword argument, or of the leakiest element where none is given, where it took the last one listed, which S2 does not require to be the top one; and a new ``share_of`` keyword argument sets the fill level the loss is a share of, which is otherwise the top of the highest element's range [`#256 `__, `#258 `__, `#259 `__] +* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] -* ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] -* A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] -* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind. Statuses about different steps may interleave, so the spacing is kept per step [`#252 `__, `#259 `__] -* A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] -* ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] -* The example server creates its power sensor with ``consumption_is_positive``, as S2 states power, so that an actuator's measured power and the schedule planned for it point the same way on the sensor that holds both [`#252 `__] -* The example resource manager answers an instruction: it acknowledges it, and reports the transition it asks for when the instruction's ``execution_time`` arrives, rather than printing it and carrying on. Answering on arrival, which is what the resource managers written against this example do, reports a mode the device is not in yet, so a customer energy manager records a measurement of a power nothing is drawing: two instructions sent together, hours apart in execution time, were answered within milliseconds of each other and contradicted one another [`#252 `__] -* Importing either S2 example script no longer runs it, so they can be read, imported and tested like any other module [`#251 `__] -* ``FRBCSimple`` posts an actuator's power in the sign the power sensor records consumption in, read from the sensor's ``consumption_is_positive`` attribute, rather than in S2's own convention of positive for consumption. FlexMeasures defaults a plain power sensor to the other convention and flips a schedule's sign on the way in, so a measurement of the actuator consuming 883.7 W sat opposite the plan to consume it, which reads as a device doing the reverse of what it was asked. A sensor that cannot be read is posted to as S2 states it, and the reason logged [`#252 `__] -* ``FRBCSimple`` asks for a schedule in watts, which is what S2 states an operation mode's power ranges in, rather than in whatever unit the power sensor happens to be recorded in. S2 states an operation mode's power ranges in watts, and the example server creates its power sensor in ``kW``, so every scheduled power was read as a thousandth of itself: it then sat nearest the operation mode that draws nothing, and the resource manager was instructed to idle through a schedule that asked it to run. The handler already stated the unit when posting an actuator's power; it just never said which one it wanted back [`#247 `__] -* A state of charge is posted in the unit the handler records fill levels in, instead of a hardcoded ``MWh``, which stopped FlexMeasures converting it: the example server creates its state-of-charge sensor in ``kWh``, so a fill level of 10 was being recorded as 10,000. A schedule is asked for in that same unit, so the bounds the scheduler plans against and the measurements it plans from agree [`#244 `__] -* A state of charge, and the whole flex-model a schedule is asked for with, are scaled by ``fill_level_scale`` like every other fill level the handler records. The scale is applied where the data reaches FlexMeasures and nowhere else: translating a schedule back into instructions keeps the resource manager's own units, since it matches fill levels against the operation modes stated in them [`#244 `__] -* A fill rate is recorded as a rate of fill level, in ``/s``, instead of in the power unit. S2 states both an actuator's fill rate and a usage forecast as fill level per second, which for a heat buffer is not electricity at all, and which even for a battery differs from kilowatts by the seconds in an hour [`#244 `__] -* A fill-level target profile is posted from the resolution boundary its translation lined it up with, rather than from the moment the resource manager sent it, which shifted every value by the difference [`#244 `__] -* An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks, and recorded none of what they declared [`#244 `__] -* A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] +* Importing either S2 example script no longer runs it, so they can be read, imported and tested like any other module [`#252 `__] [`issue #251 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] -* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and from the next resolution boundary rather than the start of the step under way, which a description days old made it slide into the past, and which, planned from the fill level reported mid-step, counted on the whole step where only part was left, so each replan within it asked for less power than the step needed [`#240 `__, `#259 `__] -* An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] -* An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] -* An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] -* The instruction-status history is keyed by a string, as its type declares, instead of by the ``UUID`` that a no-op ``cast`` let through, which made every lookup miss [`#237 `__] -* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the power range of the element that applies at the latest fill level, and posts it to the power sensor [`#236 `__, `#256 `__] -* A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] - -Added ------ - -* An FRBC handler tracks the instruction batch the resource manager is working from, and ``send_revocations`` makes it revoke the instructions of a batch it replaces that are still pending [`#237 `__] -* ``ControlTypeHandler.send_message()``, which handlers use instead of putting messages on the CEM's sending queue themselves [`#237 `__] -* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there instead of discarding it [`#236 `__] v0.9.0 | 2026-04-21 =================== From ea0b1d03561b56b72ce2eada4f85a3e1028f4fbb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 01:29:25 +0100 Subject: [PATCH 73/87] fix(FRBC): comments that later changes made false, and the unguarded fill-rate post - The fallback-scheduler warning said no state of charge was recorded for a later window to be planned from. This handler sends its state of charge inline, so a later window does not depend on one; the warning now says only that the schedule is a coarse charging policy, not a plan. - The power_unit comment said nothing converts a power value on the way out, with a TODO to do so; it is converted, so the comment says that. - The fill-rate post in send_actuator_status runs in a task nobody awaits, and is now guarded and logged like the power post beside it. Found by the changelog rewrite, which read the code at the head. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 38 ++++++++++-------- tests/s2/test_frbc_declared_profiles.py | 39 +++++++++++++++++++ tests/s2/test_frbc_schedule_retry.py | 2 +- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 63b7eaa1..2b3bcb9f 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -254,11 +254,8 @@ def __init__( # Half again as high, so that a storage overshooting its declared range stays feasible. # A deployment whose storage cannot overshoot at all can bring this down to 1. hard_bound_factor: float = 1.5, - # The unit a power value this handler posts is labelled with. - # Those values come straight from the operation modes' power ranges, which S2 states in watts, - # and nothing converts them on the way out, - # so a deployment that sets another unit here has them recorded at the wrong magnitude. - # TODO: either convert a power value into this unit before posting it, or drop this and post watts, as S2 states them. + # The unit a power value this handler posts is converted into and labelled with. + # Those values come from the operation modes' power ranges, which S2 states in watts. power_unit: str = "W", # The unit a fill level is recorded in: a state of charge and the minima and maxima # bounding it all go out in this one unit, so that they can be compared at all. @@ -799,14 +796,23 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): fill_rate.start_of_range + (fill_rate.end_of_range - fill_rate.start_of_range) * factor ) * self._fill_level_scale - await self._fm_client.post_sensor_data( - self._fill_rate_sensor_id, - start=start, - prior=self.now(), - values=[rate], - unit=self.fill_rate_unit(), - duration=self._resolution, - ) + # Guarded like the power above, since this runs in a task nobody awaits. + try: + await self._fm_client.post_sensor_data( + self._fill_rate_sensor_id, + start=start, + prior=self.now(), + values=[rate], + unit=self.fill_rate_unit(), + duration=self._resolution, + ) + except REQUEST_FAILURES: + self._logger.exception( + "Could not record the fill rate %s on sensor %s for %s.", + rate, + self._fill_rate_sensor_id, + start, + ) async def _record_charging_efficiency( self, @@ -1023,9 +1029,7 @@ def _warn_where_a_fallback_scheduler_answered( Against an older server, the schedule endpoint follows that fallback job rather than reporting the failure, or redirects to it where the server sets FLEXMEASURES_FALLBACK_REDIRECT, which this client follows, so either way a schedule nobody asked for arrives looking like the one that was. - It is a coarse charging policy rather than a plan, - and it records no state of charge, so a later window has nothing to resolve its soc-at-start against: - that is how a house in a co-simulation starved mid-run. + It is a coarse charging policy rather than a plan, made without the band, the efficiencies and the power bands this handler asked with. :param schedule: what came back, which names the scheduler that planned it. :param start: the window it was asked for. @@ -1037,7 +1041,7 @@ def _warn_where_a_fallback_scheduler_answered( if "fallback" not in scheduler.lower(): return self._logger.error( - "The schedule for the window starting %s was planned by %s, which is a fallback scheduler: the problem this handler asked about was infeasible. The schedule is a coarse charging policy, and no state of charge was recorded for a later window to be planned from.", + "The schedule for the window starting %s was planned by %s, which is a fallback scheduler: the problem this handler asked about was infeasible. The schedule is a coarse charging policy, not a plan.", start, scheduler, ) diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index 035cadef..a1489358 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -712,3 +712,42 @@ async def test_a_bound_that_could_not_be_recorded_leaves_the_other_recorded( second = frbc._fm_client.post_sensor_data.await_args_list[1].kwargs assert second["sensor_id"] == frbc._soc_maxima_sensor_id assert "Could not record a bound" in caplog.text + + +@pytest.mark.asyncio +async def test_a_fill_rate_that_could_not_be_recorded_is_logged( + system_description, caplog +): + """The post runs in a task nobody awaits, so an exception there was asyncio's to report, if anyone's.""" + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + fill_rate_sensor_id=20, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._fm_client.get_sensor = AsyncMock( + return_value=power_sensor_recording(consumption_as_positive=True) + ) + # The power goes through; the fill rate does not. + frbc._fm_client.post_sensor_data = AsyncMock( + side_effect=[({}, 200), ConnectionError("down")] + ) + frbc._logger = logging.getLogger(__name__) + frbc._system_description_history = { + system_description.message_id: system_description + } + actuator = system_description.actuators[0] + + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=0.5, + ) + ) + + assert frbc._fm_client.post_sensor_data.await_count == 2 + assert "Could not record the fill rate" in caplog.text diff --git a/tests/s2/test_frbc_schedule_retry.py b/tests/s2/test_frbc_schedule_retry.py index 99df6b3f..39faa561 100644 --- a/tests/s2/test_frbc_schedule_retry.py +++ b/tests/s2/test_frbc_schedule_retry.py @@ -349,7 +349,7 @@ async def test_a_schedule_from_a_fallback_scheduler_is_said_out_loud(caplog): await frbc.trigger_schedule(system_description.message_id) assert "StorageFallbackScheduler" in caplog.text - assert "no state of charge was recorded" in caplog.text + assert "coarse charging policy, not a plan" in caplog.text # The schedule is still followed: a coarse policy beats leaving the actuator to itself. frbc.send_instruction_batch.assert_awaited_once() From 2f89b72d1bc036f9e53bb7e665d0980050712d49 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Nov 2026 02:18:46 +0100 Subject: [PATCH 74/87] fix(FRBC): warn about transitions only where they rule a switch out The demo gate on the final head showed the warning that timers and transitions are not honoured firing on the repository's own example, an on/off heat pump that declares both of its switches and no timers. Such transitions rule nothing out, so there is nothing this CEM fails to honour. The warning now goes out only where the actuator declares timers, or where some switch between two modes usable under normal conditions has no transition usable under normal conditions. A blocking timer needs a declared timer, which s2python checks, so timers cover blocked transitions too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/__init__.py | 29 +++++++++- tests/s2/test_frbc_unsupported_is_said.py | 53 +++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 6b5bc82b..e45c802d 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -52,7 +52,7 @@ Changed * ``FRBCSimple`` posts a state of charge in its ``energy_unit``, ``kWh`` by default, and states the flex model's ``soc-unit`` to match (``kWh`` where ``energy_unit`` is neither ``kWh`` nor ``MWh``), where v0.9.0 labelled every state of charge, and the ``soc-unit`` of every schedule, ``MWh``, whatever unit the resource manager reported in: against the example server's ``kWh`` sensor, a fill level of 10 was recorded as 10,000. A resource manager that reports in megawatt hours now needs ``energy_unit="MWh"`` [`#244 `__] * ``FRBCSimple`` records an actuator's power on the power sensor, and no longer writes anything to ``rm_discharge_sensor_id``, which stays a required argument. v0.9.0 posted minus the fill rate of the first-listed operation mode, labelled ``MWh``, onto that discharge sensor. Now it posts the power of the operation mode the actuator reports it is in, from the power range of the element that applies at the latest fill level, interpolated by the reported factor, as covering one ``resolution`` step. It is posted in the sign the power sensor records consumption in, read from the sensor's ``consumption_is_positive`` attribute (as S2 states it where the sensor cannot be read, with the reason logged), and converted into ``power_unit``. A second status for the same step supersedes the first rather than colliding with it, and a power that cannot be recorded is logged with its value, unit, sensor and moment [`#236 `__, `#252 `__, `#256 `__, `#258 `__, `#259 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for, though it is a coarse charging policy rather than a plan [`#254 `__] -* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where v0.9.0 acknowledged it and planned for the first actuator alone. A description whose actuator declares timers or transitions is taken with a warning that they are not honoured, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every timer status is now kept [`#273 `__] +* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where v0.9.0 acknowledged it and planned for the first actuator alone. A description whose actuator declares timers, or transitions that rule out a switch between its operation modes, is taken with a warning that they are not honoured; transitions allowing every switch, as an on/off device declaring both of its switches does, rule out nothing and are taken without one, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every timer status is now kept [`#273 `__] * The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions. It acknowledges each instruction, and reports the transition it asks for when the instruction's ``execution_time`` arrives, rather than printing it and carrying on; reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds; follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked; and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power point the same way, seeds its demo prices from the current day and again every 12 hours, so that the window it plans for keeps its prices, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] Removed diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index c81901cd..169de5aa 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -245,9 +245,9 @@ def handle_system_description( # Said once per description, since nothing here would otherwise say it at all: # an instruction can ask for a switch that a running timer or a missing transition forbids, and the device refuses it. actuator = message.actuators[0] - if actuator.timers or actuator.transitions: + if _transitions_restrict(actuator): self._logger.warning( - "The actuator declares %s timers and %s transitions, which this CEM does not honour: an instruction may ask for a switch the device refuses.", + "The actuator declares %s timers and %s transitions that rule out some switches between its operation modes, which this CEM does not honour: an instruction may ask for a switch the device refuses.", len(actuator.timers), len(actuator.transitions), ) @@ -441,3 +441,28 @@ async def trigger_schedule( ) await self.send_message(instruction) + + +def _transitions_restrict(actuator) -> bool: + """Whether an actuator's timers and transitions rule out a switch between two of its operation modes. + + Transitions that allow every switch between modes usable under normal conditions, with no timer to wait for, + rule out nothing, as an on/off device that declares both of its switches does, + so only where something is ruled out is there anything this CEM fails to honour. + """ + # A transition can only be blocked by a timer the actuator declares, which s2python checks, so timers cover that too. + if actuator.timers: + return True + if not actuator.transitions: + return False + modes = [ + str(mode.id) + for mode in actuator.operation_modes + if not mode.abnormal_condition_only + ] + allowed = { + (str(t.from_), str(t.to)) + for t in actuator.transitions + if not t.abnormal_condition_only + } + return any((a, b) not in allowed for a in modes for b in modes if a != b) diff --git a/tests/s2/test_frbc_unsupported_is_said.py b/tests/s2/test_frbc_unsupported_is_said.py index 79271586..ad71e746 100644 --- a/tests/s2/test_frbc_unsupported_is_said.py +++ b/tests/s2/test_frbc_unsupported_is_said.py @@ -171,3 +171,56 @@ def status(): assert caplog.text.count("does not honour") == 1 assert len(frbc._timer_status_history) == 2 + + +def _two_modes_with(transitions_between, blocking_timers=()): + """The heater, with an idle mode beside it, and transitions between the two as given.""" + from s2python.common import Transition + + system_description = _heater() + actuator = system_description.actuators[0] + on = actuator.operation_modes[0] + off = on.model_copy(update={"id": get_unique_id(), "diagnostic_label": "off"}) + transitions = [ + Transition( + id=get_unique_id(), + from_=a.id, + to=b.id, + start_timers=[], + blocking_timers=list(blocking_timers), + abnormal_condition_only=False, + ) + for a, b in transitions_between(on, off) + ] + return _with(system_description, operation_modes=[on, off], transitions=transitions) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "transitions_between, blocking_timers, warned", + [ + (lambda on, off: [(on, off), (off, on)], (), False), + (lambda on, off: [(off, on)], (), True), + ], + ids=[ + "every switch allowed", + "switching off ruled out", + ], +) +async def test_transitions_are_warned_about_only_where_they_rule_a_switch_out( + transitions_between, blocking_timers, warned, caplog +): + """An on/off device that declares both of its switches rules nothing out, as the example resource manager does.""" + system_description = _two_modes_with(transitions_between, blocking_timers) + frbc = _frbc(system_description) + frbc._storage_status_history.clear() + + with caplog.at_level(logging.WARNING): + status = await frbc.handle_message( + system_description.model_copy( + update={"message_id": get_unique_id()} + ).to_dict() + ) + + assert status.status == ReceptionStatusValues.OK + assert ("does not honour" in caplog.text) is warned From e35ffd33514dfe59586aaec9a04f055e96604c1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:23:12 +0100 Subject: [PATCH 75/87] docs(changelog): say each S2 change plainly Rewrite the Unreleased entries of CHANGELOG-S2.rst, and the three this work added to CHANGELOG.rst, for legibility: lead with what changes for the reader, in short sentences with plain word order, and split the longest entries into a lead sentence and a list. No claim is added or dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 183 ++++++++++++++++++++++++++++++++--------------- CHANGELOG.rst | 6 +- 2 files changed, 128 insertions(+), 61 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index e45c802d..363ca164 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -16,77 +16,144 @@ Other changes: see `Unreleased in the changelog /s``, a negative expected usage as none, with a warning, and a leakage behaviour is recorded as a storage efficiency each time a schedule is asked for (see Changed). Without a sensor, that declaration is not recorded [`#244 `__, `#259 `__, `#271 `__, `#274 `__] -* ``FRBCSimple`` takes an ``energy_unit``, defaulting to ``kWh``, which a state of charge, the fill-level band around it, a fill rate and a usage forecast (the latter two per second) are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] -* ``FRBCSimple`` takes a ``fill_level_scale``, defaulting to 1, which every fill level and fill rate it sends to FlexMeasures is multiplied by: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule back into instructions keeps the resource manager's own units, since it matches fill levels against the operation modes stated in them. A scale of zero or below is refused. It also takes a ``power_unit``, defaulting to ``W``, which an actuator's power is converted into and labelled with when it is posted [`#236 `__, `#244 `__, `#245 `__, `#252 `__] -* ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all, and a mode reserved for abnormal conditions is not counted. It is recorded once per window, and recorded again for the next schedule where FlexMeasures did not confirm storing it; where it cannot be recorded, the schedule is asked for without it, and the log says so [`#253 `__, `#256 `__, `#272 `__, `#273 `__] -* ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two at a given fill level. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__, `#256 `__] -* ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for and how long between two attempts; fewer than one attempt is refused. v0.9.0 asked once, so any failure cost the storage its instruction for that window, while the resource manager kept polling for it. A scheduling job that failed inside the server is asked for again, unless its cause fails identically on every attempt, as a scheduler colliding with its own belief or an infeasible problem does. So is a server that could not be reached or answered with a 5xx, and a bare HTTP 400 from a FlexMeasures older than v0.33.0, which reports a failed job no other way. A job no worker picked up in time is asked about once more. Giving up is logged as an error that says the storage is left without an instruction [`#254 `__, `#273 `__] -* ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] -* ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] -* An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] -* An FRBC handler tracks the instruction batch the resource manager is working from, and, with its ``send_revocations`` attribute set to ``True``, revokes the instructions of a batch it replaces that are still pending, that is, with no status reported yet, or reported as ``NEW`` or ``ACCEPTED``. Off by default, since a resource manager is not required to honour revocations [`#237 `__] -* ``FRBC.filter_instruction_transitions()``, ``FRBC.send_instruction_batch()`` and ``FRBC.system_description_fingerprint()``, which a handler uses to drop instructions that repeat their predecessor, to send a batch that replaces the one before it (dropped where a newer request arrived meanwhile), and to tell a changed system description from a re-sent one [`#237 `__] -* ``ControlTypeHandler.send_message()``, which handlers use instead of putting messages on the CEM's sending queue themselves [`#237 `__] -* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there, as a rate of fill level in ``/s`` rather than in a power unit, since S2 states a fill rate as fill level per second, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] -* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers and the moment a state of charge is known as of included [`#264 `__, `#268 `__] -* ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] -* ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] -* ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] -* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__, `#267 `__] -* ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] +* ``FRBCSimple`` can record what a resource manager declares about its storage, so it can be seen and planned with. Give it any of the new ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``; a declaration without a sensor is not recorded. + + - A fill-level target profile is recorded on the minima and maxima sensors, if both are given. It is recorded on the handler's ``resolution`` steps, starting from the step its start falls in. It is kept inside the declared fill-level range: a target past either end, such as a night setback below the range, is cut back to the range, and a target band wholly outside the range is recorded at its nearest end, with a warning. If one of the two bounds cannot be recorded, this is logged and the other is still recorded. + - A usage forecast is recorded in ``/s``. A negative expected usage is recorded as zero, with a warning. + - A leakage behaviour is recorded as a storage efficiency, each time a schedule is requested (see Changed). + + [`#244 `__, `#259 `__, `#271 `__, `#274 `__] +* ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge, the fill-level band, the fill rate and the usage forecast are recorded in this unit (the last two per second). Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] +* ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Instructions stay in the resource manager's own units, because they are matched against its operation modes. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which is the unit an actuator's power is posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] +* ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a lower bound would narrow the band to something the resource manager never asked for [`#245 `__] +* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: it records there what the operation modes say about converting drawn power into stored energy, and requests a schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets with less power than the actuator needs. + + - FlexMeasures takes one efficiency per device. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. If modes differ, the lowest is used, and the choice is logged. + - A mode that draws power without filling the storage, such as standby or circulation, does not count. Counting it as zero would tell the scheduler that the storage cannot be filled at all. Modes reserved for abnormal conditions do not count either. + - The efficiency is recorded once per window. It is recorded again for the next schedule if FlexMeasures did not confirm storing it. If it cannot be recorded, the schedule is requested without it, and this is logged. + + [`#253 `__, `#256 `__, `#272 `__, `#273 `__] +* ``fill_rate_in_watts()`` converts a fill rate into watts, the unit of S2 power ranges. ``operation_mode_conversion_efficiency()`` gives the ratio between the two at a given fill level. A fill rate is per second, in the unit fill levels are recorded in, so one kilowatt hour per second is 3.6 million watts. An unknown energy unit is refused, rather than converted with a factor that is off by thousands [`#253 `__, `#256 `__] +* ``FRBCSimple`` retries a failed schedule request. Two new arguments set the limits: ``schedule_attempts`` (default 5, at least 1) and ``schedule_retry_interval`` (default 2 seconds). In v0.9.0, a single failure left the storage without an instruction for that window, while the resource manager kept waiting for one. + + - Retried: a scheduling job that failed on the server, an unreachable server, a 5xx response, and a bare HTTP 400 from a FlexMeasures older than v0.33.0 (its only way to report a failed job). + - Not retried: failures that would repeat on every attempt, such as an infeasible problem or a scheduler colliding with its own belief. + - A job that no worker picked up in time is checked once more. + - Giving up is logged as an error, saying that the storage is left without an instruction. + + [`#254 `__, `#273 `__] +* ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for sites where feeding back is priced differently from drawing, such as under a feed-in tariff. Without it, ``price_sensor_id`` prices both directions, as before [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``. With ``True``, a schedule may exceed the site's connection capacity at a price; use this where the capacity is a target set by others. With ``False``, the capacity is a hard limit. If unset, FlexMeasures decides, and since v1.0 it relaxes the capacity by default. So for a home whose connection is a physical limit, pass ``False`` [`#261 `__] +* An FRBC handler logs each instruction batch once, at debug level, listing the operation modes by name [`#261 `__] +* An FRBC handler can revoke instructions it no longer wants carried out. Set its ``send_revocations`` attribute to ``True``. When a new batch replaces the one the resource manager is working from, the handler revokes the old batch's pending instructions: those without a reported status, or reported as ``NEW`` or ``ACCEPTED``. This is off by default, because a resource manager does not have to honour revocations [`#237 `__] +* New ``FRBC`` methods for handlers. ``filter_instruction_transitions()`` drops instructions that repeat the previous one. ``send_instruction_batch()`` sends a batch that replaces the previous one, unless a newer request has arrived in the meantime. ``system_description_fingerprint()`` tells a changed system description from a re-sent one [`#237 `__] +* ``ControlTypeHandler.send_message()``. Handlers now use it to send messages, instead of putting them on the CEM's sending queue themselves [`#237 `__] +* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there. The fill rate is recorded in ``/s``, not as a power: S2 states a fill rate as fill level per second, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``. The clock sets the time for the CEM and for every handler it registers. That includes their timers, and the time a recorded state of charge is known from [`#264 `__, `#268 `__] +* ``CEM(accept_simulation_metadata=True)`` accepts messages from a co-simulation that are wrapped with the simulated time (``S2Wrapper``), and sets the CEM's clock from them. It is off by default, and then a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] +* ``flexmeasures_client.s2.site``, with ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``. It gives each resource manager its own FlexMeasures site, named after it, with an FRBC handler connected to that site's sensors [`#265 `__] +* ``CEM(site_configurator=...)`` sets up a resource manager's site once its details arrive. Until then, the CEM answers its messages with a temporary error. The example server enables this with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] +* ``examples/S2-co-simulation``: a FlexMeasures stack with one CEM per simulated home, for co-simulations whose resource managers connect over websockets [`#265 `__, `#267 `__] +* ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)`` record a resource manager's measured power. Use one recorder per server, so that a resource manager that reconnects is not recorded twice [`#266 `__] Changed ------- -* Breaking for subclasses: ``FRBC.trigger_schedule()`` and ``FRBC.send_storage_status()`` each take a ``request_number`` keyword argument, the number of the planning request they answer, so a subclass that overrides either has to accept it [`#237 `__, `#240 `__] -* Breaking: in ``flexmeasures_client.s2.control_types.FRBC.utils``, ``compute_next_fill_level()`` is now ``compute_next_fill_level(fill_level, fill_rate, step, leakage_behaviour=None, usage_rate=0.0)``, with the step a ``timedelta`` and every rate per second, where it took ``(fill_level, fill_rate, leakage_behaviour, deltaT, usage)`` with ``deltaT`` in hours; and ``power_to_fill_rate_with_metrics()`` returns six values, the operation-mode factor added, instead of five, as do the tuples ``explain_choice()`` takes [`#256 `__, `#258 `__] -* ``fm_schedule_to_instructions()`` takes three new keyword arguments. ``fill_level_bounds`` are the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned; without them the declared range is used, as before. ``leakage_behaviour`` and ``usage_forecast`` lower that running fill level as the storage leaks and is drawn from [`#245 `__, `#258 `__] -* An FRBC handler plans one schedule at a time, for the newest request only. In v0.9.0 every system description started a solve of its own, and every solve sent its own instruction batch, however many newer ones had arrived meanwhile. Now a request arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile; a request out of date makes no further attempt after a failed one, is not turned into instructions, and sends no batch [`#237 `__, `#261 `__, `#269 `__] -* An FRBC handler asks for a schedule on every storage status, since the state of charge it reports is what a schedule starts from, and on a system description that arrives after the latest status and differs from the one before it, its validity included. So a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves. v0.9.0 planned on system descriptions only, every one of them, so a resource manager re-sending an unchanged description was planned for again each time: now such a description is acknowledged and otherwise ignored. A state of charge that cannot be recorded is logged, and planned from all the same [`#240 `__, `#259 `__, `#273 `__] -* ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes: the step a schedule is planned on and its start lined up with, and the step a fill-level target profile, a usage forecast and a storage efficiency are recorded on, and an actuator's power and fill rate are recorded as covering. A ``resolution`` or ``schedule_duration`` that is not positive is refused, and so is a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only. In v0.9.0, whatever the power sensor's resolution, a schedule was read as quarter hours, and an actuator's status recorded as covering one [`#240 `__, `#258 `__] -* A schedule starts at a resolution boundary: the one at or before the system description's ``valid_from`` plus ``valid_from_shift``, or, where that is in the past, the next boundary after now. ``valid_from_shift`` still defaults to one day. v0.9.0 started a schedule at ``valid_from`` plus the shift, unaligned, which left FlexMeasures without prices for a window off its steps. A boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment [`#240 `__, `#259 `__] -* ``FRBCSimple`` asks for a schedule within the power ranges the actuator declares: a ``consumption-capacity`` and a ``production-capacity`` from the widest range it states, and, from a FlexMeasures at v1.0.0 or newer, which added the field, each distinct range as one of the ``operation-modes``, except where the ranges together cover exactly what the two capacities allow, without a gap, and so say nothing new while each costs the solver a binary per step. An older server is asked without the power bands, and the log says so, since a flex model naming a field the server does not know is refused outright. The scheduler then plans only power values the actuator can run at, instead of a fractional power that a resource manager has to round into a full-on block, overshooting the capacity its site is held to. An actuator that cannot produce at all is given a production capacity of zero, where before its storage could be planned as if it were a battery feeding the grid. Modes S2 reserves for abnormal conditions are left out of both, and are never instructed into, since a resource manager refuses an instruction into one that is not marked abnormal; an actuator with no other mode is not planned for, with a warning [`#248 `__, `#272 `__] -* ``FRBCSimple`` holds a storage to its declared fill-level range as a band whose breach is priced, rather than as hard bounds, and plans against what the resource manager declared about its storage. The band goes out with each request as ``soc-minima`` and ``soc-maxima`` over the whole window, with ``relax-soc-constraints`` in the flex context: the declared range, with the latest fill-level target profile, held inside the range, applying where it covers a step, and clipped at zero, since FlexMeasures takes no negative minima. A newer target profile replaces the one before it, as S2 means it to. It goes out with or without the band sensors, which only record it. The hard ``soc-min`` and ``soc-max`` are now zero and ``hard_bound_factor`` times the top of the range, widened to take in a state of charge above it, where v0.9.0 held the storage hard to the declared range: a state a storage really reaches, such as free-floating above the top of its range or being left below the bottom of it, was infeasible, and answered with no schedule at all, or by a fallback scheduler. A storage reported below zero is planned from zero, with a warning, since FlexMeasures refuses a negative ``soc-min``. Against a FlexMeasures older than v0.26.0, which cannot relax the band, the band is left out, with a note in the log, so the storage is planned anywhere between the wide hard bounds. The usage forecast is named as ``soc-usage`` where its sensor is given and the resource manager says it provides one, or has sent one regardless. A leakage behaviour is recorded as a ``storage-efficiency`` each time a schedule is asked for, over the window the schedule covers, from the element that applies at the fill level it starts from, as a share of that fill level or of the middle of the storage's range where it starts below that, and named only where FlexMeasures confirmed storing it. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#244 `__, `#245 `__, `#259 `__, `#271 `__, `#273 `__] -* ``FRBCSimple`` posts a state of charge in its ``energy_unit``, ``kWh`` by default, and states the flex model's ``soc-unit`` to match (``kWh`` where ``energy_unit`` is neither ``kWh`` nor ``MWh``), where v0.9.0 labelled every state of charge, and the ``soc-unit`` of every schedule, ``MWh``, whatever unit the resource manager reported in: against the example server's ``kWh`` sensor, a fill level of 10 was recorded as 10,000. A resource manager that reports in megawatt hours now needs ``energy_unit="MWh"`` [`#244 `__] -* ``FRBCSimple`` records an actuator's power on the power sensor, and no longer writes anything to ``rm_discharge_sensor_id``, which stays a required argument. v0.9.0 posted minus the fill rate of the first-listed operation mode, labelled ``MWh``, onto that discharge sensor. Now it posts the power of the operation mode the actuator reports it is in, from the power range of the element that applies at the latest fill level, interpolated by the reported factor, as covering one ``resolution`` step. It is posted in the sign the power sensor records consumption in, read from the sensor's ``consumption_is_positive`` attribute (as S2 states it where the sensor cannot be read, with the reason logged), and converted into ``power_unit``. A second status for the same step supersedes the first rather than colliding with it, and a power that cannot be recorded is logged with its value, unit, sensor and moment [`#236 `__, `#252 `__, `#256 `__, `#258 `__, `#259 `__] -* ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for, though it is a coarse charging policy rather than a plan [`#254 `__] -* An FRBC handler refuses a system description of more than one actuator with ``INVALID_CONTENT`` and a diagnostic label, since every plan is made for one, where v0.9.0 acknowledged it and planned for the first actuator alone. A description whose actuator declares timers, or transitions that rule out a switch between its operation modes, is taken with a warning that they are not honoured; transitions allowing every switch, as an on/off device declaring both of its switches does, rule out nothing and are taken without one, and so is each timer the first time an ``FRBC.TimerStatus`` names it; every timer status is now kept [`#273 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions. It acknowledges each instruction, and reports the transition it asks for when the instruction's ``execution_time`` arrives, rather than printing it and carrying on; reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds; follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked; and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power point the same way, seeds its demo prices from the current day and again every 12 hours, so that the window it plans for keeps its prices, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] +* Breaking for subclasses: ``FRBC.trigger_schedule()`` and ``FRBC.send_storage_status()`` each take a ``request_number`` keyword argument: the number of the planning request they answer. A subclass that overrides either method has to accept it [`#237 `__, `#240 `__] +* Breaking: two functions in ``flexmeasures_client.s2.control_types.FRBC.utils`` changed signature. + + - ``compute_next_fill_level()`` is now ``compute_next_fill_level(fill_level, fill_rate, step, leakage_behaviour=None, usage_rate=0.0)``. The step is a ``timedelta``, and every rate is per second. Before, it was ``(fill_level, fill_rate, leakage_behaviour, deltaT, usage)``, with ``deltaT`` in hours. + - ``power_to_fill_rate_with_metrics()`` returns six values instead of five; the operation-mode factor is new. The tuples that ``explain_choice()`` takes changed the same way. + + [`#256 `__, `#258 `__] +* ``fm_schedule_to_instructions()`` takes three new keyword arguments. ``fill_level_bounds`` are the bounds of the fill level it tracks while translating. Pass them if the schedule was requested with bounds other than the declared fill-level range, so that each step gets the operation mode for the state the scheduler actually planned. Without them, the declared range is used, as before. ``leakage_behaviour`` and ``usage_forecast`` lower the tracked fill level as the storage leaks and is used [`#245 `__, `#258 `__] +* An FRBC handler plans one schedule at a time, and only for the newest request. In v0.9.0, every system description started its own solve, and every solve sent its own instruction batch, even when newer ones had arrived. Now a request that arrives during planning waits. It is dropped if an even newer request arrives. A request that is out of date is not retried after a failure, not turned into instructions, and sends no batch [`#237 `__, `#261 `__, `#269 `__] +* An FRBC handler requests a schedule on every storage status, because a schedule starts from the reported state of charge. It also requests one on a system description that arrives after the latest status and differs from the previous one, including in its validity. So a resource manager gets a plan for whichever of the two it sends first, whether or not its fill level changes. In v0.9.0, only system descriptions triggered planning, every one of them, so re-sending an unchanged description triggered a new plan each time. Now such a description is acknowledged and otherwise ignored. If the state of charge cannot be recorded, this is logged, and the schedule is still requested [`#240 `__, `#259 `__, `#273 `__] +* ``FRBCSimple`` takes a ``resolution`` (default 15 minutes). It is the step a schedule is planned on and aligned to. Fill-level target profiles, usage forecasts and storage efficiencies are recorded on this step, and each recorded actuator power and fill rate covers one step. A ``resolution`` or ``schedule_duration`` that is not positive is refused. So is a ``schedule_duration`` that is not a whole number of steps, because FlexMeasures only answers with whole steps. In v0.9.0, a schedule was always read as quarter hours, whatever the power sensor's resolution, and an actuator status was recorded as covering a quarter hour [`#240 `__, `#258 `__] +* A schedule starts on a resolution boundary: the last one at or before the system description's ``valid_from`` plus ``valid_from_shift``, or, if that has passed, the next one after now. ``valid_from_shift`` still defaults to one day. In v0.9.0, a schedule started exactly at ``valid_from`` plus the shift. That moment could lie between steps, where FlexMeasures had no prices. Boundaries are computed in the UTC offset of the moment itself, as FlexMeasures does for a series posted with that offset. In the repeated hour at the end of daylight saving time, the later of the two boundaries is used, as long as it is not after the moment [`#240 `__, `#259 `__] +* ``FRBCSimple`` keeps a schedule within the power ranges the actuator declares, so the scheduler only plans powers the actuator can run at. Before, it could plan a fractional power, which the resource manager had to round up to a full-on block, exceeding the site's capacity. + + - The widest declared range sets ``consumption-capacity`` and ``production-capacity``. An actuator that cannot produce gets a production capacity of zero; before, its storage could be planned as if it were a battery feeding the grid. + - Each distinct range is also sent as one of the ``operation-modes``, if the server runs FlexMeasures v1.0.0 or newer (the release that added this field). This is skipped if the ranges together cover exactly what the capacities allow, without gaps: then they add nothing, and each one would cost the solver a binary variable per step. An older server is asked without the operation modes, and this is logged, because it would refuse a flex model with a field it does not know. + - Modes that S2 reserves for abnormal conditions are left out, and are never used in instructions: a resource manager refuses an instruction into such a mode if the instruction is not marked as abnormal. An actuator that only has such modes is not planned for, with a warning. + + [`#248 `__, `#272 `__] +* ``FRBCSimple`` treats the declared fill-level range as a soft band whose breach is priced, instead of as hard bounds. It also plans with what the resource manager declared about its storage. + + - The band is sent with every schedule request as ``soc-minima`` and ``soc-maxima`` over the whole window, with ``relax-soc-constraints`` in the flex context. It is the declared range, narrowed by the latest fill-level target profile wherever that profile covers a step, and clipped at zero, because FlexMeasures accepts no negative minima. A newer target profile replaces the previous one, as S2 intends. The band is sent whether or not the band sensors are given; they only record it. + - The hard bounds ``soc-min`` and ``soc-max`` are now zero and ``hard_bound_factor`` times the top of the range, raised to include a state of charge above it. In v0.9.0, the declared range itself was the hard bound. So a storage that really was outside it, for example floating above the top or left below the bottom, made the problem infeasible: there was no schedule at all, or one from a fallback scheduler. A storage reported below zero is planned from zero, with a warning, because FlexMeasures refuses a negative ``soc-min``. + - FlexMeasures older than v0.26.0 cannot relax the band. Then the band is left out, and this is logged, so the storage is planned anywhere between the wide hard bounds. + - The usage forecast is sent as ``soc-usage`` if its sensor is given and the resource manager provides a forecast (or has sent one anyway). + - The leakage behaviour is recorded as a ``storage-efficiency`` with each schedule request, over the window of the schedule. It uses the leakage at the starting fill level, as a share of that fill level (or of the middle of the range, if the storage starts below that). It is only sent if FlexMeasures confirmed storing it. FlexMeasures takes a storage efficiency as a share of the stock, whereas S2 states a rate, so the loss is exact only at that one fill level. + + [`#244 `__, `#245 `__, `#259 `__, `#271 `__, `#273 `__] +* ``FRBCSimple`` posts the state of charge in its ``energy_unit`` (``kWh`` by default), and sets the flex model's ``soc-unit`` to match (to ``kWh`` if ``energy_unit`` is neither ``kWh`` nor ``MWh``). In v0.9.0, every state of charge and every schedule's ``soc-unit`` was labelled ``MWh``, whatever unit the resource manager reported in. On the example server's ``kWh`` sensor, a fill level of 10 was recorded as 10,000. A resource manager that reports in megawatt hours now needs ``energy_unit="MWh"`` [`#244 `__] +* ``FRBCSimple`` records the actuator's power on the power sensor. It no longer writes to ``rm_discharge_sensor_id``, which is still a required argument. In v0.9.0, it posted the negative fill rate of the first listed operation mode, labelled ``MWh``, to that discharge sensor. + + - The power is now that of the operation mode the actuator reports being in: taken from the power range that applies at the latest fill level, interpolated by the reported factor, and posted as covering one ``resolution`` step. + - The sign follows the power sensor's ``consumption_is_positive`` attribute. If the sensor cannot be read, S2's sign is used, and the reason is logged. The power is converted into ``power_unit``. + - A second status for the same step replaces the first, instead of colliding with it. If the power cannot be recorded, the log gives its value, unit, sensor and time. + + [`#236 `__, `#252 `__, `#256 `__, `#258 `__, `#259 `__] +* ``FRBCSimple`` logs an error if a schedule was made by a fallback scheduler instead of the requested one. Until v1.0.0, FlexMeasures answered an infeasible problem that way. An older server either serves the fallback job in place of the failed one, or redirects to it, which this client follows. Either way, the substitute looks like the requested schedule, but it is only a coarse charging policy, not a plan [`#254 `__] +* An FRBC handler refuses a system description with more than one actuator, answering ``INVALID_CONTENT`` with a diagnostic label, because every plan is made for one actuator. In v0.9.0, it accepted such a description and planned for the first actuator only. + + - An actuator that declares timers, or transitions that rule out switching between some of its operation modes, is accepted with a warning that these are not honoured. Transitions that allow every switch rule nothing out, so they cause no warning; an on/off device that declares both its switches is an example. + - The first ``FRBC.TimerStatus`` for each timer also causes the warning. Every timer status is now stored. + + [`#273 `__] +* The example resource manager is now a heat pump that keeps a hot-water buffer warm. + + - It modulates from 400 to 1500 W, with a COP of 4 while the buffer is cold and 3 once it is warm. It declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer using S2's own definitions. + - It acknowledges each instruction, and reports the transition when the instruction's ``execution_time`` arrives, instead of printing it and moving on. It reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. + - It follows only the latest plan, and honours the revocations the example server now sends. It tells the CEM the status of each instruction, so that only instructions not yet carried out are revoked. + - It re-sends its leakage, usage forecast and comfort profile every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. + + The example server records each declaration on its own sensor. It creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power have the same sign. It seeds its demo prices from the current day, and again every 12 hours, so the planned window always has prices. It sets up the site's graphs to show the buffer's fill level within its comfort band, and what fills and drains it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] Removed ------- -* ``flexmeasures_client.s2.cem.get_commodity_unit``, which stated power in kW where S2 states it in W [`#266 `__] -* ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] -* ``flexmeasures_client.s2.utils.get_validation_error_summary``, which iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given [`#257 `__] +* ``flexmeasures_client.s2.cem.get_commodity_unit``. It stated power in kW, while S2 states it in W [`#266 `__] +* ``op_mode_compute_factor``. It derived a factor from a fill rate, and returned zero wherever an element has a single fill rate. The factor now comes with the chosen operation mode, from its power range [`#258 `__] +* ``flexmeasures_client.s2.utils.get_validation_error_summary``. It iterated over a pydantic ``ValidationError``, which is not iterable, so it raised on every error [`#257 `__] Fixed ----- -* A full message history keeps the newest messages: it made room by evicting the newest entry, so it kept a session's first messages and lost every later one but the last. A message set again under the same id becomes the latest [`#274 `__] -* ``Handler.is_revoked()`` finds an object the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] -* An FRBC actuator status that arrives before any system description, as one can after a reconnection, is logged and skipped, rather than raising in a background task [`#274 `__] -* A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] -* A CEM without a sensor for a resource manager's measured power no longer records an L1 measurement on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] -* A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] -* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection, where a session used to keep working after its resource manager had disconnected [`#263 `__] -* A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] -* An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks [`#244 `__] -* ``FRBCSimple`` asks for a schedule in watts, which is what S2 states an operation mode's power ranges in, rather than in whatever unit the power sensor happens to be recorded in. The example server creates its power sensor in ``kW``, so every scheduled power was read as a thousandth of itself: it then sat nearest the operation mode that draws nothing, and the resource manager was instructed to idle through a schedule that asked it to run [`#247 `__] -* On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] -* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] -* Translating a schedule into instructions no longer depends on the order a mode lists its elements in: where two elements share a boundary, the one that applies answers, not the first listed [`#256 `__] -* A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, and only there, rather than wherever the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#237 `__, `#258 `__] -* An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] -* ``leakage_behaviour_to_storage_efficiency()`` compounds a leakage rate per second, as S2 states it, where it compounded it per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002%. It takes the rate of the element that applies at a new ``fill_level`` keyword argument, or of the leakiest element where none is given, where it took the last one listed, which S2 does not require to be the top one; and a new ``share_of`` keyword argument sets the fill level the loss is a share of, which is otherwise the top of the highest element's range [`#256 `__, `#258 `__, `#259 `__] -* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] -* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] -* ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] +* A full message history now keeps the newest messages. It used to make room by evicting the newest entry, so it kept a session's first messages and lost all later ones except the last. A message stored again under the same id now counts as the latest [`#274 `__] +* ``Handler.is_revoked()`` now finds objects the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] +* An FRBC actuator status that arrives before any system description is logged and skipped, instead of raising an error in a background task. This can happen after a reconnection [`#274 `__] +* A CEM records measured power by the time it was measured. It records one average per step of the sensor's resolution (15 minutes on the example's sites, previously five), in watts, as S2 states it. A step is recorded with the first measurement that arrives after the step has ended by the CEM's clock. Before, measurements were grouped by the CEM's current time, which dropped every measurement sent after the fact, as co-simulations do. They were also labelled as kilowatts instead of watts, and kept in a buffer shared by every CEM in the process. Recording now happens in the background, with a few retries if it fails [`#266 `__] +* A CEM without a sensor for a resource manager's measured power no longer records L1 measurements on sensor 357. That was whatever sensor had that id on the server, on any site, and the CEM reported the measurement as recorded [`#265 `__] +* A CEM that is selecting a control type now takes a message of that control type as confirmation of the selection, because a resource manager only sends such messages once it uses that control type. Before, the CEM stayed without a control type, and answered each such message with a temporary error, which the resource manager retried; co-simulations whose resource managers reconnect showed this. A message of any other control type activates nothing [`#263 `__] +* Closing a control-type handler cancels its remaining work, including recording. The example server's session now ends when its connection closes; it used to keep working after the resource manager disconnected [`#263 `__] +* A control-type handler can log before it is registered, and a handler without its own ``close()`` no longer breaks the CEM's teardown [`#236 `__] +* An ``FRBCSimple`` handler now accepts ``FRBC.FillLevelTargetProfile``, ``FRBC.UsageForecast`` and ``FRBC.LeakageBehaviour`` messages, instead of raising ``NotImplementedError`` for each. The CEM runs these as tasks that nothing awaits, so a resource manager sending all three caused three "Task exception was never retrieved" tracebacks on every connection [`#244 `__] +* ``FRBCSimple`` requests schedules in watts, the unit of S2 operation-mode power ranges, instead of in the power sensor's unit. The example server's power sensor is in ``kW``, so every scheduled power was read as a thousandth of its value. It then matched the operation mode that draws nothing, and the resource manager was told to idle while the schedule asked it to run [`#247 `__] +* On a state-of-charge sensor that records instants, a state of charge is now recorded as a reading at an instant. Before, it was recorded as covering the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses instants, it is still recorded over one minute [`#259 `__] +* Translating a schedule into instructions now treats every rate as per second, as S2 states it. + + - It used to advance the tracked fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little. As a result, every step got the operation mode for the fill level the schedule started from. + - Leakage now lowers the tracked fill level, using the element of the latest ``FRBC.LeakageBehaviour`` that applies. Before, leakage was read from two fields that S2 does not have, so it was never counted. Usage from the latest ``FRBC.UsageForecast`` now lowers it too. + - Each step is followed exactly, as a resource manager follows an instruction. If the fill level crosses into another element, or usage changes within a step, that is taken into account at the moment it happens. A level that reaches a boundary, beyond which it would be pushed back, stays at that boundary. + + [`#258 `__] +* Translating a schedule into instructions no longer depends on the order in which a mode lists its elements. Where two elements share a boundary, the one that applies is used, not the first one listed [`#256 `__] +* A schedule is read at the resolution FlexMeasures answers in: its duration divided by its number of values. It used to be read as quarter hours, so a schedule for an hourly or a five-minute power sensor raised an error, and no instruction was sent [`#258 `__] +* A new instruction is sent whenever the operation mode or its factor changes, and only then, instead of whenever the power changes. This includes a constant power at which the fill level moves into another element of the mode. A change in the factor of a millionth or less does not count: that is rounding noise from converting a schedule from MW to W [`#237 `__, `#258 `__] +* An instruction now asks for the factor that draws the scheduled power, taken from the operation mode's power range. It used to derive the factor from the fill rate, which asked for the bottom of the power range wherever the fill rate is a single value and the power a range [`#258 `__] +* ``leakage_behaviour_to_storage_efficiency()`` compounds a leakage rate per second, as S2 states it. It compounded it per hour, so a buffer that loses about 7% of its heat over twelve hours was described as losing 0.002%. + + - A new ``fill_level`` keyword argument selects the element that applies at that fill level. Without it, the leakiest element is used. Before, it used the last element listed, which S2 does not require to be the top one. + - A new ``share_of`` keyword argument sets the fill level that the loss is a share of. Without it, that is the top of the highest element's range. + + [`#256 `__, `#258 `__, `#259 `__] +* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value in MW means to FlexMeasures. It used to post a quarter-hour amount, which FlexMeasures planned as a quarter of the real usage. It reads the forecast back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FillRateBasedControlTUNES`` now plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` in a label that may be missing, so a single unlabelled mode stopped planning, and stopped the efficiencies of all modes listed after it from being recorded [`#257 `__] +* ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the resulting error was mistaken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * Importing either S2 example script no longer runs it, so they can be read, imported and tested like any other module [`#252 `__] [`issue #251 `__] -* The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] +* The example CEM server answers an S2 handshake immediately. It used to configure its site in FlexMeasures first, which took longer than a resource manager waits for the answer [`#241 `__] v0.9.0 | 2026-04-21 =================== diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66b6e9a8..f93b4948 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,20 +21,20 @@ Added * ``server_version_at_least``, which answers whether the server is of a given version or newer, looking its version up where that is not known yet, for a caller that sends a field or calls an endpoint only where the server has it. Release generations are compared, so a development build such as ``0.33.0.dev26`` counts as ``0.33.0`` [`#248 `__] * ``ingestion_polling_interval``, ``ingestion_polling_max_interval`` and ``ingestion_polling_timeout``, which govern how the ingestion of posted sensor data is waited for, on a tighter profile than the ``job_polling_*`` settings give other background jobs [`#242 `__] -* ``post_sensor_data`` takes ``confirm_ingestion``, which raises where the ingestion of the posted data could not be confirmed, rather than logging it and carrying on, for a caller that remembers having recorded the data and would otherwise never record it again [`#273 `__] +* ``post_sensor_data`` takes ``confirm_ingestion``. With ``True``, it raises an error if it cannot confirm that FlexMeasures ingested the posted data, instead of logging this and carrying on. Use it if your code remembers what it has recorded, and would otherwise never record the data again [`#273 `__] * Optional ``rate_limit_notifier`` and ``job_status_notifier`` callbacks, to report rate limits and the progress of jobs being waited for, instead of logging them [`#225 `__] * ``get_automations``, ``get_automation``, ``add_automation``, ``update_automation``, ``delete_automation``, ``trigger_automation`` and ``trigger_and_await_automation``, to manage and run automations through the API on FlexMeasures v1.1.0 and up [`#230 `__] Changed ------- -* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required, and half a location is refused [`#261 `__] +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, just like FlexMeasures. This allows assets without a location, such as one that represents a community. ``generic_asset_type_id`` is still required, and a latitude without a longitude (or the other way around) is refused [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] Removed ------- -* ``response_handling.check_content_type``, which refused an answer in a JSON variant that aiohttp parses, such as ``application/problem+json``, with an error that raised an ``AttributeError`` of its own when printed. Such an answer is now read like any other. A content type that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 `__] +* ``response_handling.check_content_type``. It refused responses in JSON variants that aiohttp can parse, such as ``application/problem+json``, and the error it raised caused an ``AttributeError`` of its own when printed. Such responses are now read like any other. A response that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 `__] Fixed ----- From 95cda2993e1d458402f7edac09fd5d98fcc6e917 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:27:01 +0100 Subject: [PATCH 76/87] docs(changelog): keep what the plain rewrite lost or shifted A check of every entry against the text it replaced found the rewrite had given the charging-efficiency risk a direction (wrong for a heat pump, whose COP exceeds one), made "a lower bound" of the hard upper bound, and dropped or shifted several smaller facts: planning from an unrecorded state of charge, the storage efficiency named only once confirmed, the DST boundary rule, abnormal modes being left out of both capacities and bands, power being converted into power_unit, polling, and the handler always tracking the batch in force. Restore each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 363ca164..12887b18 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -24,9 +24,9 @@ Added [`#244 `__, `#259 `__, `#271 `__, `#274 `__] * ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge, the fill-level band, the fill rate and the usage forecast are recorded in this unit (the last two per second). Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] -* ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Instructions stay in the resource manager's own units, because they are matched against its operation modes. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which is the unit an actuator's power is posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] -* ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a lower bound would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: it records there what the operation modes say about converting drawn power into stored energy, and requests a schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets with less power than the actuator needs. +* ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule into instructions keeps the resource manager's own units, because it matches fill levels against the operation modes, which are stated in those units. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which an actuator's power is converted into and posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] +* ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a hard upper bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] +* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: it records there what the operation modes say about converting drawn power into stored energy, and requests a schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. - FlexMeasures takes one efficiency per device. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. If modes differ, the lowest is used, and the choice is logged. - A mode that draws power without filling the storage, such as standby or circulation, does not count. Counting it as zero would tell the scheduler that the storage cannot be filled at all. Modes reserved for abnormal conditions do not count either. @@ -34,7 +34,7 @@ Added [`#253 `__, `#256 `__, `#272 `__, `#273 `__] * ``fill_rate_in_watts()`` converts a fill rate into watts, the unit of S2 power ranges. ``operation_mode_conversion_efficiency()`` gives the ratio between the two at a given fill level. A fill rate is per second, in the unit fill levels are recorded in, so one kilowatt hour per second is 3.6 million watts. An unknown energy unit is refused, rather than converted with a factor that is off by thousands [`#253 `__, `#256 `__] -* ``FRBCSimple`` retries a failed schedule request. Two new arguments set the limits: ``schedule_attempts`` (default 5, at least 1) and ``schedule_retry_interval`` (default 2 seconds). In v0.9.0, a single failure left the storage without an instruction for that window, while the resource manager kept waiting for one. +* ``FRBCSimple`` retries a failed schedule request. Two new arguments set the limits: ``schedule_attempts`` (default 5, at least 1) and ``schedule_retry_interval`` (default 2 seconds). In v0.9.0, a single failure left the storage without an instruction for that window, while the resource manager kept polling for one. - Retried: a scheduling job that failed on the server, an unreachable server, a 5xx response, and a bare HTTP 400 from a FlexMeasures older than v0.33.0 (its only way to report a failed job). - Not retried: failures that would repeat on every attempt, such as an infeasible problem or a scheduler colliding with its own belief. @@ -45,7 +45,7 @@ Added * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for sites where feeding back is priced differently from drawing, such as under a feed-in tariff. Without it, ``price_sensor_id`` prices both directions, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``. With ``True``, a schedule may exceed the site's connection capacity at a price; use this where the capacity is a target set by others. With ``False``, the capacity is a hard limit. If unset, FlexMeasures decides, and since v1.0 it relaxes the capacity by default. So for a home whose connection is a physical limit, pass ``False`` [`#261 `__] * An FRBC handler logs each instruction batch once, at debug level, listing the operation modes by name [`#261 `__] -* An FRBC handler can revoke instructions it no longer wants carried out. Set its ``send_revocations`` attribute to ``True``. When a new batch replaces the one the resource manager is working from, the handler revokes the old batch's pending instructions: those without a reported status, or reported as ``NEW`` or ``ACCEPTED``. This is off by default, because a resource manager does not have to honour revocations [`#237 `__] +* An FRBC handler tracks the instruction batch the resource manager is working from, and can revoke instructions it no longer wants carried out. Set its ``send_revocations`` attribute to ``True``. When a new batch replaces that one, the handler revokes the old batch's pending instructions: those without a reported status, or reported as ``NEW`` or ``ACCEPTED``. This is off by default, because a resource manager does not have to honour revocations [`#237 `__] * New ``FRBC`` methods for handlers. ``filter_instruction_transitions()`` drops instructions that repeat the previous one. ``send_instruction_batch()`` sends a batch that replaces the previous one, unless a newer request has arrived in the meantime. ``system_description_fingerprint()`` tells a changed system description from a re-sent one [`#237 `__] * ``ControlTypeHandler.send_message()``. Handlers now use it to send messages, instead of putting them on the CEM's sending queue themselves [`#237 `__] * ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there. The fill rate is recorded in ``/s``, not as a power: S2 states a fill rate as fill level per second, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] @@ -68,14 +68,14 @@ Changed [`#256 `__, `#258 `__] * ``fm_schedule_to_instructions()`` takes three new keyword arguments. ``fill_level_bounds`` are the bounds of the fill level it tracks while translating. Pass them if the schedule was requested with bounds other than the declared fill-level range, so that each step gets the operation mode for the state the scheduler actually planned. Without them, the declared range is used, as before. ``leakage_behaviour`` and ``usage_forecast`` lower the tracked fill level as the storage leaks and is used [`#245 `__, `#258 `__] * An FRBC handler plans one schedule at a time, and only for the newest request. In v0.9.0, every system description started its own solve, and every solve sent its own instruction batch, even when newer ones had arrived. Now a request that arrives during planning waits. It is dropped if an even newer request arrives. A request that is out of date is not retried after a failure, not turned into instructions, and sends no batch [`#237 `__, `#261 `__, `#269 `__] -* An FRBC handler requests a schedule on every storage status, because a schedule starts from the reported state of charge. It also requests one on a system description that arrives after the latest status and differs from the previous one, including in its validity. So a resource manager gets a plan for whichever of the two it sends first, whether or not its fill level changes. In v0.9.0, only system descriptions triggered planning, every one of them, so re-sending an unchanged description triggered a new plan each time. Now such a description is acknowledged and otherwise ignored. If the state of charge cannot be recorded, this is logged, and the schedule is still requested [`#240 `__, `#259 `__, `#273 `__] +* An FRBC handler requests a schedule on every storage status, because a schedule starts from the reported state of charge. It also requests one on a system description that arrives after the latest status and differs from the previous one, including in its validity. So a resource manager gets a plan for whichever of the two it sends first, whether or not its fill level changes. In v0.9.0, only system descriptions triggered planning, every one of them, so re-sending an unchanged description triggered a new plan each time. Now such a description is acknowledged and otherwise ignored. If the state of charge cannot be recorded, this is logged, and the schedule is still planned from it [`#240 `__, `#259 `__, `#273 `__] * ``FRBCSimple`` takes a ``resolution`` (default 15 minutes). It is the step a schedule is planned on and aligned to. Fill-level target profiles, usage forecasts and storage efficiencies are recorded on this step, and each recorded actuator power and fill rate covers one step. A ``resolution`` or ``schedule_duration`` that is not positive is refused. So is a ``schedule_duration`` that is not a whole number of steps, because FlexMeasures only answers with whole steps. In v0.9.0, a schedule was always read as quarter hours, whatever the power sensor's resolution, and an actuator status was recorded as covering a quarter hour [`#240 `__, `#258 `__] -* A schedule starts on a resolution boundary: the last one at or before the system description's ``valid_from`` plus ``valid_from_shift``, or, if that has passed, the next one after now. ``valid_from_shift`` still defaults to one day. In v0.9.0, a schedule started exactly at ``valid_from`` plus the shift. That moment could lie between steps, where FlexMeasures had no prices. Boundaries are computed in the UTC offset of the moment itself, as FlexMeasures does for a series posted with that offset. In the repeated hour at the end of daylight saving time, the later of the two boundaries is used, as long as it is not after the moment [`#240 `__, `#259 `__] +* A schedule starts on a resolution boundary: the last one at or before the system description's ``valid_from`` plus ``valid_from_shift``, or, if that has passed, the next one after now. ``valid_from_shift`` still defaults to one day. In v0.9.0, a schedule started exactly at ``valid_from`` plus the shift. That moment could lie between steps, which left FlexMeasures without prices for the window. Boundaries are computed in the UTC offset of the moment itself, as FlexMeasures does for a series posted with that offset. In the repeated hour at the end of daylight saving time, the later boundary that is not after the moment is used [`#240 `__, `#259 `__] * ``FRBCSimple`` keeps a schedule within the power ranges the actuator declares, so the scheduler only plans powers the actuator can run at. Before, it could plan a fractional power, which the resource manager had to round up to a full-on block, exceeding the site's capacity. - The widest declared range sets ``consumption-capacity`` and ``production-capacity``. An actuator that cannot produce gets a production capacity of zero; before, its storage could be planned as if it were a battery feeding the grid. - Each distinct range is also sent as one of the ``operation-modes``, if the server runs FlexMeasures v1.0.0 or newer (the release that added this field). This is skipped if the ranges together cover exactly what the capacities allow, without gaps: then they add nothing, and each one would cost the solver a binary variable per step. An older server is asked without the operation modes, and this is logged, because it would refuse a flex model with a field it does not know. - - Modes that S2 reserves for abnormal conditions are left out, and are never used in instructions: a resource manager refuses an instruction into such a mode if the instruction is not marked as abnormal. An actuator that only has such modes is not planned for, with a warning. + - Modes that S2 reserves for abnormal conditions are left out of both the capacities and the operation modes, and are never used in instructions: a resource manager refuses an instruction into such a mode if the instruction is not marked as abnormal. An actuator that only has such modes is not planned for, with a warning. [`#248 `__, `#272 `__] * ``FRBCSimple`` treats the declared fill-level range as a soft band whose breach is priced, instead of as hard bounds. It also plans with what the resource manager declared about its storage. @@ -84,7 +84,7 @@ Changed - The hard bounds ``soc-min`` and ``soc-max`` are now zero and ``hard_bound_factor`` times the top of the range, raised to include a state of charge above it. In v0.9.0, the declared range itself was the hard bound. So a storage that really was outside it, for example floating above the top or left below the bottom, made the problem infeasible: there was no schedule at all, or one from a fallback scheduler. A storage reported below zero is planned from zero, with a warning, because FlexMeasures refuses a negative ``soc-min``. - FlexMeasures older than v0.26.0 cannot relax the band. Then the band is left out, and this is logged, so the storage is planned anywhere between the wide hard bounds. - The usage forecast is sent as ``soc-usage`` if its sensor is given and the resource manager provides a forecast (or has sent one anyway). - - The leakage behaviour is recorded as a ``storage-efficiency`` with each schedule request, over the window of the schedule. It uses the leakage at the starting fill level, as a share of that fill level (or of the middle of the range, if the storage starts below that). It is only sent if FlexMeasures confirmed storing it. FlexMeasures takes a storage efficiency as a share of the stock, whereas S2 states a rate, so the loss is exact only at that one fill level. + - The leakage behaviour is recorded as a ``storage-efficiency`` with each schedule request, over the window of the schedule. It uses the leakage at the starting fill level, as a share of that fill level (or of the middle of the range, if the storage starts below that). It is only named in the flex model if FlexMeasures confirmed storing it. FlexMeasures takes a storage efficiency as a share of the stock, whereas S2 states a rate, so the loss is exact only at that one fill level. [`#244 `__, `#245 `__, `#259 `__, `#271 `__, `#273 `__] * ``FRBCSimple`` posts the state of charge in its ``energy_unit`` (``kWh`` by default), and sets the flex model's ``soc-unit`` to match (to ``kWh`` if ``energy_unit`` is neither ``kWh`` nor ``MWh``). In v0.9.0, every state of charge and every schedule's ``soc-unit`` was labelled ``MWh``, whatever unit the resource manager reported in. On the example server's ``kWh`` sensor, a fill level of 10 was recorded as 10,000. A resource manager that reports in megawatt hours now needs ``energy_unit="MWh"`` [`#244 `__] @@ -105,11 +105,11 @@ Changed * The example resource manager is now a heat pump that keeps a hot-water buffer warm. - It modulates from 400 to 1500 W, with a COP of 4 while the buffer is cold and 3 once it is warm. It declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer using S2's own definitions. - - It acknowledges each instruction, and reports the transition when the instruction's ``execution_time`` arrives, instead of printing it and moving on. It reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. + - It acknowledges each instruction, and reports the transition it asks for when the instruction's ``execution_time`` arrives, instead of printing it and moving on. It reports its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. - It follows only the latest plan, and honours the revocations the example server now sends. It tells the CEM the status of each instruction, so that only instructions not yet carried out are revoked. - It re-sends its leakage, usage forecast and comfort profile every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. - The example server records each declaration on its own sensor. It creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power have the same sign. It seeds its demo prices from the current day, and again every 12 hours, so the planned window always has prices. It sets up the site's graphs to show the buffer's fill level within its comfort band, and what fills and drains it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] + The example server records each declaration on its own sensor. It creates its power sensor with ``consumption_is_positive``, as S2 states power, so that measured and scheduled power have the same sign. It seeds its demo prices from the current day, and again every 12 hours, so that the window it plans for keeps its prices. It sets up the site's graphs to show the buffer's fill level within its comfort band, and what fills and drains it [`#241 `__, `#252 `__, `#259 `__, `#262 `__] Removed ------- @@ -124,7 +124,7 @@ Fixed * A full message history now keeps the newest messages. It used to make room by evicting the newest entry, so it kept a session's first messages and lost all later ones except the last. A message stored again under the same id now counts as the latest [`#274 `__] * ``Handler.is_revoked()`` now finds objects the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] * An FRBC actuator status that arrives before any system description is logged and skipped, instead of raising an error in a background task. This can happen after a reconnection [`#274 `__] -* A CEM records measured power by the time it was measured. It records one average per step of the sensor's resolution (15 minutes on the example's sites, previously five), in watts, as S2 states it. A step is recorded with the first measurement that arrives after the step has ended by the CEM's clock. Before, measurements were grouped by the CEM's current time, which dropped every measurement sent after the fact, as co-simulations do. They were also labelled as kilowatts instead of watts, and kept in a buffer shared by every CEM in the process. Recording now happens in the background, with a few retries if it fails [`#266 `__] +* A CEM records measured power by the time it was measured. It records one average per step of the sensor's resolution (15 minutes on the example's sites, previously five), in watts, as S2 states it. A step is recorded with the first measurement that arrives after the step has ended by the CEM's clock. Before, measurements were grouped by the CEM's current time, which dropped every measurement sent after the fact, as co-simulations do. They were also labelled as kilowatts instead of watts, and kept in a buffer shared by every CEM in the process. Recording happens in the background, with a few retries if it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records L1 measurements on sensor 357. That was whatever sensor had that id on the server, on any site, and the CEM reported the measurement as recorded [`#265 `__] * A CEM that is selecting a control type now takes a message of that control type as confirmation of the selection, because a resource manager only sends such messages once it uses that control type. Before, the CEM stayed without a control type, and answered each such message with a temporary error, which the resource manager retried; co-simulations whose resource managers reconnect showed this. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels its remaining work, including recording. The example server's session now ends when its connection closes; it used to keep working after the resource manager disconnected [`#263 `__] From 00a6136bc568e0d6598984f7ffea257fbd936ba1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:46:20 +0100 Subject: [PATCH 77/87] test(S2): cover what the handler base, the CEM and the example server leave untested Raise combined line coverage from 95.17% to 95.93%, above main's 95.68% before the S2 package counted: an invalid message answered as INVALID_DATA, messages given as JSON, failure callbacks, activating the control type in force or one the resource manager lacks, re-registering a control type, an empty schedule, and the example server's websocket producer and consumer (send, reset, bad JSON, close, connection error). Each test fails when the code it covers is broken (6 of 6 mutants killed). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_example_server_websocket.py | 139 +++++++++++++++++++ tests/s2/test_handler_base.py | 159 ++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 tests/s2/test_example_server_websocket.py create mode 100644 tests/s2/test_handler_base.py diff --git a/tests/s2/test_example_server_websocket.py b/tests/s2/test_example_server_websocket.py new file mode 100644 index 00000000..06aa76c0 --- /dev/null +++ b/tests/s2/test_example_server_websocket.py @@ -0,0 +1,139 @@ +"""How the example server passes messages between a websocket and its CEM.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import aiohttp +import pytest + +from flexmeasures_client.s2.script import websockets_server as server + + +class FakeCEM: + """Just enough of a CEM: a queue of messages to send, and a record of what it handled.""" + + def __init__(self, to_send=(), handle_raises=None): + self._to_send = list(to_send) + self.handled = [] + self.closed = False + self._handle_raises = handle_raises + self._logger = server.LOGGER + + def is_closed(self): + return self.closed or not self._to_send + + async def get_message(self): + return self._to_send.pop(0) + + async def handle_message(self, message): + if self._handle_raises: + raise self._handle_raises + self.handled.append(message) + + async def close(self): + self.closed = True + + +class FakeWebSocket: + def __init__(self, incoming=(), send_fails_after=None): + self._incoming = list(incoming) + self.sent = [] + self.closed = False + self._send_fails_after = send_fails_after + + async def send_json(self, message): + if ( + self._send_fails_after is not None + and len(self.sent) >= self._send_fails_after + ): + raise aiohttp.ClientConnectionResetError("gone") + self.sent.append(message) + + async def close(self): + self.closed = True + + def exception(self): + return RuntimeError("broken pipe") + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._incoming: + raise StopAsyncIteration + return self._incoming.pop(0) + + +def text(data: str): + return SimpleNamespace(type=aiohttp.WSMsgType.TEXT, data=data) + + +@pytest.mark.asyncio +async def test_the_producer_sends_what_the_cem_queues(): + cem = FakeCEM(to_send=[{"n": 1}, {"n": 2}]) + ws = FakeWebSocket() + + await server.websocket_producer(ws, cem) + + assert ws.sent == [{"n": 1}, {"n": 2}] + + +@pytest.mark.asyncio +async def test_the_producer_stops_when_the_connection_is_reset(): + cem = FakeCEM(to_send=[{"n": 1}, {"n": 2}, {"n": 3}]) + ws = FakeWebSocket(send_fails_after=1) + + await server.websocket_producer(ws, cem) + + assert ws.sent == [{"n": 1}] + assert cem._to_send == [ + {"n": 3} + ], "nothing more is taken from the CEM once the connection is gone" + + +@pytest.mark.asyncio +async def test_the_consumer_hands_each_message_to_the_cem(): + cem = FakeCEM() + ws = FakeWebSocket(incoming=[text('{"message_type": "Handshake"}')]) + + await server.websocket_consumer(ws, cem) + + assert cem.handled == [{"message_type": "Handshake"}] + + +@pytest.mark.asyncio +async def test_the_consumer_logs_a_message_that_is_not_json_and_carries_on(caplog): + cem = FakeCEM() + ws = FakeWebSocket( + incoming=[text("not json"), text('{"message_type": "Handshake"}')] + ) + + await server.websocket_consumer(ws, cem) + + assert cem.handled == [{"message_type": "Handshake"}] + assert "handle_message failed for 'not json'" in caplog.text + + +@pytest.mark.asyncio +async def test_close_closes_the_cem_and_the_connection(): + cem = FakeCEM() + ws = FakeWebSocket(incoming=[text("close")]) + + await server.websocket_consumer(ws, cem) + + assert cem.closed and ws.closed + assert cem.handled == [] + + +@pytest.mark.asyncio +async def test_a_connection_error_closes_the_cem(caplog): + cem = FakeCEM() + ws = FakeWebSocket( + incoming=[SimpleNamespace(type=aiohttp.WSMsgType.ERROR, data=None)] + ) + + await server.websocket_consumer(ws, cem) + + assert cem.closed + assert "ws connection closed with exception broken pipe" in caplog.text diff --git a/tests/s2/test_handler_base.py b/tests/s2/test_handler_base.py new file mode 100644 index 00000000..3e9af23f --- /dev/null +++ b/tests/s2/test_handler_base.py @@ -0,0 +1,159 @@ +"""What every S2 handler does with the messages it is given, whatever its control type.""" + +from __future__ import annotations + +import json + +import pandas as pd +import pytest +from s2python.common import ControlType, Handshake, ReceptionStatusValues + +from flexmeasures_client.s2 import Handler, register +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types.FRBC import FRBCTest +from flexmeasures_client.s2.utils import get_reception_status + + +class HandshakeHandler(Handler): + """A handler for one message type, which counts what reaches it.""" + + def __init__(self): + super().__init__() + self.handled = [] + + @register(Handshake) + def handle_handshake(self, message: Handshake): + self.handled.append(message) + return get_reception_status(message) + + +@pytest.mark.asyncio +async def test_an_invalid_message_is_answered_as_invalid_data(rm_handshake): + handler = HandshakeHandler() + message = rm_handshake.to_dict() + message["supported_protocol_versions"] = "not a list" + + answer = await handler.handle_message(message) + + assert answer.status == ReceptionStatusValues.INVALID_DATA + assert str(answer.subject_message_id) == str(message["message_id"]) + assert answer.diagnostic_label, "the answer says what was invalid" + assert handler.handled == [], "the handler itself never sees an invalid message" + assert answer in handler.outgoing_messages.values(), "the answer is kept as sent" + + +@pytest.mark.asyncio +async def test_a_valid_message_reaches_its_handler(rm_handshake): + """The positive control for the test above: the same message, unaltered, is handled.""" + handler = HandshakeHandler() + + answer = await handler.handle_message(rm_handshake.to_dict()) + + assert answer.status == ReceptionStatusValues.OK + assert len(handler.handled) == 1 + + +@pytest.mark.asyncio +async def test_a_message_can_be_given_as_json(rm_handshake): + handler = HandshakeHandler() + + answer = await handler.handle_message(rm_handshake.to_json()) + + assert answer.status == ReceptionStatusValues.OK + assert len(handler.handled) == 1 + + +def test_a_failure_callback_is_stored_with_its_arguments(): + handler = HandshakeHandler() + called = [] + + handler.register_failure_callbacks( + "some-id", lambda **kwargs: called.append(kwargs), why="test" + ) + handler.failure_callbacks["some-id"]() + + assert called == [{"why": "test"}] + assert handler.success_callbacks == {} + + +def test_a_message_given_as_json_is_recognised_as_supported(rm_handshake): + handler = HandshakeHandler() + + assert handler.supports_message(rm_handshake.to_json()) + assert not handler.supports_message( + json.dumps({"message_type": "FRBC.StorageStatus"}) + ) + + +async def _cem_with_details(resource_manager_details, rm_handshake) -> CEM: + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + while not cem._sending_queue.empty(): + await cem.get_message() + return cem + + +@pytest.mark.asyncio +async def test_activating_the_control_type_in_force_sends_nothing( + resource_manager_details, rm_handshake +): + cem = await _cem_with_details(resource_manager_details, rm_handshake) + assert cem.control_type == ControlType.NO_SELECTION + + answer = await cem.activate_control_type(ControlType.NO_SELECTION) + + assert answer is None + assert cem._sending_queue.empty() + + +@pytest.mark.asyncio +async def test_a_control_type_the_resource_manager_lacks_is_not_activated( + resource_manager_details, rm_handshake +): + cem = await _cem_with_details(resource_manager_details, rm_handshake) + + answer = await cem.activate_control_type(ControlType.POWER_ENVELOPE_BASED_CONTROL) + + assert answer is None + assert cem._sending_queue.empty() + assert cem.control_type == ControlType.NO_SELECTION + + +def test_registering_a_control_type_again_replaces_its_handler(caplog): + cem = CEM(fm_client=None) + first, second = FRBCTest(), FRBCTest() + cem.register_control_type(first) + + cem.register_control_type(second) + + assert cem._control_types_handlers[ControlType.FILL_RATE_BASED_CONTROL] is second + assert "already registered" in caplog.text + + +def test_an_empty_schedule_gives_no_instructions(frbc_system_description): + from flexmeasures_client.s2.control_types.FRBC.utils import ( + fm_schedule_to_instructions, + ) + + no_values = { + "start": "2026-01-01T00:00:00+00:00", + "duration": "PT0H", + "values": [], + "unit": "W", + } + no_frame = pd.DataFrame({"schedule": []}, index=pd.DatetimeIndex([], tz="UTC")) + + assert ( + fm_schedule_to_instructions( + no_values, frbc_system_description, initial_fill_level=0.0 + ) + == [] + ) + assert ( + fm_schedule_to_instructions( + no_frame, frbc_system_description, initial_fill_level=0.0 + ) + == [] + ) From 481602427545e45042c28b852cb8fda929eebee5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:46:22 +0100 Subject: [PATCH 78/87] test(FRBC): time the long-schedule translation in CPU time It measured wall-clock time against 0.5 s, and failed in 2 of 12 runs on a busy machine; the S2 job now also runs under coverage. In CPU time it takes 0.11 s, and 0.22 s under coverage, still well apart from the 1.7 s it guards against. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_frbc_time_base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py index eb940673..89c22530 100644 --- a/tests/s2/test_frbc_time_base.py +++ b/tests/s2/test_frbc_time_base.py @@ -694,11 +694,14 @@ def test_a_handler_needs_a_positive_resolution_and_duration(resolution, duration def test_a_long_schedule_against_a_detailed_forecast_is_translated_quickly(): - """48 hours, at quarter hours, against the largest forecast S2 allows: well under the 1.7 s that stepping each minute took.""" + """48 hours, at quarter hours, against the largest forecast S2 allows: well under the 1.7 s that stepping each minute took. + + Timed in CPU time, so that other processes on a busy machine do not count against it. + """ import time detailed = _usage(*[(timedelta(minutes=10), 1e-4 * (i % 3)) for i in range(288)]) - started = time.perf_counter() + started = time.process_time() fm_schedule_to_instructions( _schedule([500.0, 0.0] * 96), _system_description(LOW, HIGH), @@ -706,7 +709,7 @@ def test_a_long_schedule_against_a_detailed_forecast_is_translated_quickly(): usage_forecast=detailed, leakage_behaviour=_leakage((0, 50, 1e-4), (50, 100, 2e-4)), ) - assert time.perf_counter() - started < 0.5 + assert time.process_time() - started < 0.5 def test_a_level_held_at_its_lower_bound_rises_from_there_once_filling_wins(): From e1c9e6454ebf348bcb57cd38220ca27b3ff870c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:48:40 +0100 Subject: [PATCH 79/87] docs(S2): say docstrings and comments plainly Rewrite the docstrings and comments that the S2 work added or changed, so that each one leads with what the code does or why, in short sentences with plain word order. Every fact, condition, unit and caveat is kept. Only text changed: each touched file parses to the same AST once docstrings are set aside. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 69 +- src/flexmeasures_client/s2/__init__.py | 4 +- src/flexmeasures_client/s2/cem.py | 99 +-- src/flexmeasures_client/s2/clock.py | 28 +- .../s2/control_types/FRBC/__init__.py | 60 +- .../s2/control_types/FRBC/frbc_simple.py | 597 +++++++++--------- .../s2/control_types/FRBC/frbc_tunes.py | 29 +- .../s2/control_types/FRBC/utils.py | 256 ++++---- .../s2/control_types/__init__.py | 20 +- .../s2/control_types/translations.py | 24 +- src/flexmeasures_client/s2/measurements.py | 88 +-- src/flexmeasures_client/s2/script/__init__.py | 5 +- .../s2/script/websockets_client.py | 98 +-- .../s2/script/websockets_server.py | 68 +- src/flexmeasures_client/s2/site.py | 83 +-- src/flexmeasures_client/s2/utils.py | 15 +- 16 files changed, 818 insertions(+), 725 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 002cd671..819228e7 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -459,11 +459,11 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - # request_once has already parsed this answer with aiohttp's json(), - # which refuses a content type that is not JSON, and the loop above reports that as a ConnectionError. - # A check that used to follow here refused the JSON variants aiohttp accepts too, such as application/problem+json, - # with an error that could not even be printed, so it is gone. - # A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError. + # request_once has already parsed this response with aiohttp's json(). + # json() refuses a content type that is not JSON, and the loop above reports that as a ConnectionError. + # A check used to follow here, but it also refused JSON variants that aiohttp accepts, such as application/problem+json. + # Its error could not even be printed, so the check was removed. + # A body that is not JSON under a JSON content type makes json() itself raise a JSONDecodeError. return await response.json(), response.status async def request_once( @@ -677,22 +677,24 @@ async def post_sensor_data( The method automatically chooses the appropriate API endpoint based on the provided parameters. - Since FlexMeasures 0.33.0, a server with an ingestion worker available answers ``202 Accepted`` with the id of a background job, - rather than ``200 OK``, and ingests the data after the request has returned. - By default this method then waits for that job to finish, - so that a caller which reads the data back, or asks for a schedule over it, sees what it just posted. + Since FlexMeasures 0.33.0, a server with an available ingestion worker responds ``202 Accepted`` instead of ``200 OK``, + with the id of a background job. + The server then ingests the data after the request has returned. + By default, this method waits for that job to finish. + This way, a caller that reads the data back, or asks for a schedule over it, sees the data it just posted. :param await_ingestion: whether to wait for asynchronous ingestion to finish, - when the server answered ``202 Accepted`` with a job id. - Waiting is polled on the ``ingestion_polling_*`` profile, which is tighter than the one other jobs are polled on. + when the server responded ``202 Accepted`` with a job id. + The job is polled on the ``ingestion_polling_*`` profile, which is tighter than the profile for other jobs. A job that fails raises :class:`JobFailedError`. - A job still pending when ``self.ingestion_polling_timeout`` is reached is logged as an error and otherwise ignored, - since the data may well land moments later, and a caller that has waited that long is better served by carrying on than by failing. + A job that is still pending when ``self.ingestion_polling_timeout`` is reached is logged as an error and otherwise ignored. + The data may well land moments later, and a caller that has waited that long is better off carrying on than failing. Pass ``False`` to return as soon as the server has accepted the data. - A synchronous ``200 OK`` response is unaffected either way. - :param confirm_ingestion: whether ingestion that could not be confirmed while waiting for it raises, rather than being logged and ignored: - :class:`JobTimeoutError` where the job was still pending at the timeout, and what the lookup raised where looking the job up failed. - For a caller that remembers having recorded the data, and would otherwise never record it again. + Either way, a synchronous ``200 OK`` response is unaffected. + :param confirm_ingestion: whether to raise when ingestion could not be confirmed while waiting for it, instead of logging and ignoring that. + If the job was still pending at the timeout, this raises :class:`JobTimeoutError`. + If looking the job up failed, this raises the error from the lookup. + Use this if the caller remembers having recorded the data, and would otherwise never record it again. This function raises a ValueError when an unhandled status code is returned. """ @@ -778,18 +780,21 @@ async def _await_sensor_data_ingestion( ) -> None: """Wait for an ingestion job to finish, so that the data it carries can be read back. - Waiting is left to :func:`wait_for_job`, so ingestion is backed off like every other job this client waits for, - and reported through the same ``job_status_notifier``, on the ``ingestion_polling_*`` intervals rather than the ``job_polling_*`` ones. - - A failed job propagates as :class:`JobFailedError`, for the caller to handle like any other failed post. - Nothing else does. - A timeout is logged as an error and otherwise ignored, for the reason given in :func:`post_sensor_data`, - and so is a failure to look the job up at all, which says nothing about the data: - the server accepted that, and a job record can be gone, or answered for by something in front of the server, by the time it is asked about. - Raising there would report a post that succeeded as a post that failed, - unless the caller asked for ingestion to be confirmed, in which case both are raised once logged. - A lookup that 404s is reported as a ``ConnectionError`` by :func:`request`, which wraps every client error that way, - so that is among the ones caught here, alongside the ``ValueError`` an unexpected status code raises. + Waiting is delegated to :func:`wait_for_job`. + So ingestion is backed off like every other job this client waits for, + and reported through the same ``job_status_notifier``. + The only difference is that it uses the ``ingestion_polling_*`` intervals, instead of the ``job_polling_*`` ones. + + A failed job raises :class:`JobFailedError`, which the caller handles like any other failed post. + By default, no other error is raised. + A timeout is logged as an error and otherwise ignored, for the reason given in :func:`post_sensor_data`. + The same goes for a failure to look the job up at all, which says nothing about the data. + After all, the server did accept the data. + By the time the job is looked up, its record may be gone, or something in front of the server may respond instead. + Raising in these cases would report a successful post as a failed post. + If the caller asked for ingestion to be confirmed, both cases are logged and then raised. + A lookup that returns 404 is reported as a ``ConnectionError`` by :func:`request`, which wraps every client error that way. + So ``ConnectionError`` is among the errors caught here, alongside the ``ValueError`` that an unexpected status code raises. """ try: await self.wait_for_job( @@ -1555,10 +1560,10 @@ async def add_asset( } This function raises a ValueError when an unhandled status code is returned, - when no generic_asset_type_id is given, and when only one of latitude and longitude is. + when no generic_asset_type_id is given, and when only one of latitude and longitude is given. """ if generic_asset_type_id is None: - # Optional in the signature only so that latitude and longitude, before it, can be left out, + # The signature makes this optional only so that latitude and longitude, which come before it, can be left out, # while every argument keeps its position. raise ValueError("add_asset needs a generic_asset_type_id.") if (latitude is None) != (longitude is None): @@ -1570,7 +1575,7 @@ async def add_asset( account_id=account_id, generic_asset_type_id=generic_asset_type_id, ) - # A location is optional to FlexMeasures, which many assets have no use for, such as one standing for a community. + # FlexMeasures does not require a location, and many assets have no use for one, such as an asset that represents a community. if latitude is not None: asset["latitude"] = latitude if longitude is not None: diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 04a7cbcb..9f2ed60e 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -127,7 +127,7 @@ def __init__(self, max_size: int = 100, timezone: str = "UTC") -> None: self.outgoing_messages_status = SizeLimitOrderedDict(max_size=max_size) self._timezone = ZoneInfo(timezone) - #: The time this handler goes by; a CEM gives each handler it registers its own clock. + #: The clock this handler reads the time from. A CEM sets this to its own clock for each handler it registers. self._clock: Clock = WallClock() self.discover() @@ -253,5 +253,5 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: ) def now(self) -> datetime: - """The current moment by this handler's clock, in its time zone.""" + """Return the current moment by this handler's clock, in the handler's timezone.""" return self._clock.now().astimezone(self._timezone) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 6d1f70e4..eb3707a5 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -48,7 +48,7 @@ class ConfiguredHandler(Protocol): - """What a site configurator gives back: the handler for a resource manager, and its measured-power sensors.""" + """What a site configurator returns: the handler for a resource manager, and its measured-power sensors.""" @property def handler(self) -> ControlTypeHandler: ... @@ -60,10 +60,10 @@ def power_sensor_ids(self) -> dict[str, int]: ... def asset_id(self) -> int: ... -#: Gives a resource manager a site and a handler of its own, from its details (see flexmeasures_client.s2.site). +#: A callable that takes a resource manager's details and gives it its own site and handler (see flexmeasures_client.s2.site). SiteConfigurator = Callable[[ResourceManagerDetails], Awaitable[ConfiguredHandler]] -#: The control type each family of control-type messages belongs to, by the prefix of its message type. +#: Maps the prefix of a message type to the control type that its messages belong to. CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, "PEBC.": ControlType.POWER_ENVELOPE_BASED_CONTROL, @@ -111,22 +111,24 @@ def __init__( """ Customer Energy Manager (CEM) - :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; - called with its details, in the background (see FRBCSiteConfigurator). - Until the handler is registered, its messages are answered with TEMPORARY_ERROR. - Without one, the handlers registered up front serve every resource manager. - :param measurement_recorder: where the resource manager's measured power is recorded; one of its own by default. - A server that makes a CEM per connection hands each the same one, - so a resource manager that reconnects is not recorded twice. - :param clock: the time this CEM and every handler it registers go by; - the wall clock by default. - :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, + :param site_configurator: gives a resource manager its own site and handler when it connects. + It is called with the resource manager's details, in the background (see FRBCSiteConfigurator). + Until the handler is registered, the CEM responds to the resource manager's messages with TEMPORARY_ERROR. + Without a site configurator, the handlers registered up front serve every resource manager. + By default, there is none. + :param measurement_recorder: records the resource manager's measured power. + By default, the CEM creates its own recorder. + A server that creates a CEM per connection should hand each CEM the same recorder. + Then a resource manager that reconnects does not get its measurements recorded twice. + :param clock: the clock that this CEM and every handler it registers read the time from. + By default, this is the wall clock. + :param accept_simulation_metadata: whether to accept messages wrapped with a simulated time, as a co-simulation sends them, and set the clock to that time (see flexmeasures_client.s2.wrapper.S2Wrapper). - Off by default, since the wrapper is not part of S2, + This is off by default, because the wrapper is not part of S2, and a live CEM should not let a resource manager set its time. - A resource manager that wraps its messages, from its handshake on, - is then refused with INVALID_MESSAGE and cannot connect. - On, the clock has to be a SimulatedClock, and one is made where none is given. + When it is off, a resource manager that wraps its messages from its handshake on is refused with INVALID_MESSAGE, and cannot connect. + When it is on, the clock must be a SimulatedClock. + If no clock is given, a SimulatedClock is created. """ super(CEM, self).__init__() if accept_simulation_metadata: @@ -149,7 +151,7 @@ def __init__( fm_client, logger=logger ) self._configuring: asyncio.Task | None = None - #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. + #: The control type this CEM asked the resource manager to select, and the id of that request. Reset once confirmed. self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type @@ -163,11 +165,11 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - # By this handler's clock, so that a co-simulation's timers run on simulated time. + # Use this handler's clock, so that a co-simulation's timers run on simulated time. now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + # A timer without a timezone used to be compared with the host's local time, so keep reading it as local time. due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period @@ -185,8 +187,8 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) - # On a period boundary the ceiling is the moment itself, - # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + # On a period boundary, the ceiling equals the current moment. + # A clock that stands still (as a simulated clock does between messages) would then find the timer due on every call. if next_due <= now: next_due += self._minimum_measurement_period self._timers[name] = next_due @@ -231,7 +233,7 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): # add fm_client to control_type handler control_type_handler._fm_client = self._fm_client - # The handler goes by the CEM's time, which a co-simulation sets. + # The handler uses the CEM's clock, which a co-simulation sets. control_type_handler._clock = self._clock # add sending queue @@ -261,7 +263,7 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): message = json.loads(message) assert isinstance( message, dict - ) # what a message is from here on, whatever form it came in + ) # from here on, the message is a dict, whatever form it came in if "metadata" in message and "message" in message: unwrapped = self._unwrap(message) @@ -273,8 +275,8 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): self._activate_implicitly(message) # try to handle the message with the control_type handle - # A control type without a handler yet, as while a resource manager's site is being configured, - # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. + # A control type may have no handler yet, for example while the resource manager's site is being configured. + # Its messages are then left unhandled, so the CEM responds with a TEMPORARY_ERROR, and the resource manager retries. handler = self._control_types_handlers.get(self._control_type) if handler is None and self._control_type is not None: self._ensure_configuring() @@ -308,10 +310,11 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): await self._sending_queue.put(response) def _unwrap(self, wrapped: dict) -> dict | None: - """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. + """Return the S2 message inside a co-simulation's wrapper, and set the clock to the wrapper's simulated time. - Where this CEM does not accept the wrapper, the time cannot be read, or there is no message inside, - the message is refused, the clock is left as it was, and None is returned so that nothing is handled. + The message is refused in three cases: + this CEM does not accept the wrapper, the time cannot be read, or there is no message inside. + Then the clock is left unchanged, and None is returned, so that nothing is handled. """ inner = wrapped["message"] reason = None @@ -345,15 +348,16 @@ def _unwrap(self, wrapped: dict) -> dict | None: return None def _activate_implicitly(self, message: dict) -> None: - """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. - - A resource manager sends a control type's messages only once that control type is active on its side. - So where one arrives for the control type this CEM asked for, while the CEM still waits for the confirmation, - the resource manager is working in it already. #188 saw the CEM stay in NO_SELECTION while a resource manager, - reconnecting during setup in a co-simulation, kept sending FRBC messages, each answered with a TEMPORARY_ERROR - and retried; whether the confirmation was lost or never sent was not established. - Only the control type being selected is activated, never another one a message happens to belong to, - and its pending confirmation is then no longer waited for. + """Treat a message of the control type being selected as the resource manager's acceptance of that control type. + + A resource manager only sends a control type's messages once that control type is active on its side. + Suppose such a message arrives for the control type this CEM asked for, while the CEM still waits for the confirmation. + Then the resource manager is already working in that control type. + In #188, a resource manager reconnected during setup in a co-simulation, and kept sending FRBC messages. + The CEM stayed in NO_SELECTION, responded to each message with a TEMPORARY_ERROR, and the resource manager retried. + It was not established whether the confirmation was lost or never sent. + This method only activates the control type being selected, never another control type that a message happens to belong to. + After that, the CEM no longer waits for the pending confirmation. """ if self._control_type != ControlType.NO_SELECTION or self._selecting is None: return @@ -481,12 +485,12 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) def _ensure_configuring(self) -> None: - """Set the resource manager's site up in the background, unless that is done or under way. + """Set up the resource manager's site in the background, unless that is already done or under way. - In the background, since a site takes a few requests to FlexMeasures to set up, - and the resource manager waits for a reception status meanwhile. - A setup that failed is tried again on the next message that needs it, - while the resource manager's messages are answered with TEMPORARY_ERROR, which it retries. + Setting up a site takes a few requests to FlexMeasures, and meanwhile the resource manager waits for a reception status. + That is why this runs in the background. + If the setup fails, it is tried again on the next message that needs it. + Meanwhile, the CEM responds to the resource manager's messages with TEMPORARY_ERROR, and the resource manager retries them. """ if ( self._site_configurator is None @@ -523,7 +527,7 @@ async def _configure_site(self, details: ResourceManagerDetails) -> None: async def handle_power_measurement(self, message: PowerMeasurement): if self._site_configurator is not None and not self._control_types_handlers: - # Its site is still being set up, so there is nowhere to record it yet: the resource manager is asked to resend it. + # The site is still being set up, so there is no sensor to record on yet. Ask the resource manager to resend the message. self._ensure_configuring() return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) @@ -531,8 +535,8 @@ async def handle_power_measurement(self, message: PowerMeasurement): for power_measurement in message.values: commodity_quantity = power_measurement.commodity_quantity.value - # Without a sensor for it, a measurement is not recorded. - # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. + # A measurement without a sensor is not recorded. + # An ELECTRIC.POWER.L1 measurement used to go to sensor 357, which could belong to any site on the server. sensor_id = (self.power_sensor_id or {}).get(commodity_quantity) if sensor_id is None: self._logger.debug( @@ -544,7 +548,8 @@ async def handle_power_measurement(self, message: PowerMeasurement): continue taken.append((sensor_id, power_measurement.value)) - # Every sensor is made ready before any value is taken, so that a message asked for again is not taken in part twice. + # Get every sensor ready before taking any value. + # Otherwise, part of a message could be taken twice, if the resource manager is asked to send it again. try: for sensor_id, _ in taken: await self._measurement_recorder.resolution(sensor_id) diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py index 55646905..19aa6045 100644 --- a/src/flexmeasures_client/s2/clock.py +++ b/src/flexmeasures_client/s2/clock.py @@ -1,9 +1,9 @@ -"""The time an S2 CEM and its handlers go by. +"""Clocks that tell an S2 CEM and its handlers what time it is. -A CEM that runs live goes by the wall clock. -One that steers a co-simulation goes by the time the simulation says it is, -which a resource manager sends along with its messages (see CEM's accept_simulation_metadata), -so that what the CEM records and plans is placed at simulated moments rather than at the moment it ran. +A CEM that runs live uses the wall clock. +A CEM that steers a co-simulation uses the simulated time instead. +The resource manager sends that time along with its messages (see the CEM's accept_simulation_metadata). +This way, the CEM records and plans at simulated moments, not at the moments the code actually ran. """ from __future__ import annotations @@ -14,24 +14,26 @@ class Clock(Protocol): def now(self) -> datetime: - """The current moment, timezone-aware.""" + """Return the current moment, timezone-aware.""" ... class WallClock: - """The time it is.""" + """A clock that returns the current wall-clock time, in UTC.""" def now(self) -> datetime: return datetime.now(timezone.utc) class SimulatedClock: - """The time a simulation says it is, as last set. - - It stands still between settings, as a simulation's time does between the messages that carry it. - Until it is first set it goes by the wall clock, for a resource manager that does not wrap its first messages; - one that wraps every message from its handshake on, as the co-simulation's does, sets it with the first. - Background work, such as a schedule being planned, reads it as it stands, which may be a later message's time. + """A clock that returns the simulated time it was last set to. + + The clock stands still between settings, just as simulated time stands still between the messages that carry it. + Until it is first set, it returns the wall-clock time. + This suits a resource manager that does not wrap its first messages. + A resource manager that wraps every message from its handshake on (as the co-simulation's does) sets the clock with its first message. + Background work, such as planning a schedule, reads the clock at the moment it runs. + By then, a later message may have set it to a later time. """ def __init__(self, start: datetime | None = None) -> None: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 169de5aa..140bd290 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -209,16 +209,17 @@ def system_description_fingerprint(message: FRBCSystemDescription) -> int: def handle_system_description( self, message: FRBCSystemDescription ) -> pydantic.BaseModel: - # A resend is acknowledged, and otherwise ignored: storing it again would push a - # description out of a bounded history for no reason, and a handler that did act on a - # description would act on it twice. - # Acknowledging it still matters, or the resource manager retries. - # Ignoring a resend is safe now that planning hangs off storage statuses rather than - # descriptions, and not before: it would otherwise have dropped a planning request. - # The TUNES handler has fingerprinted its descriptions this way all along, and can adopt - # this shared helper separately. - # Every plan is made for one actuator, so a description of several is refused when it arrives, - # rather than acknowledged and then failing in the background on every round, after a solve. + # A resend is acknowledged, and otherwise ignored. + # Storing it again would push an older description out of the bounded history for no reason. + # Also, a handler that acts on a description would act on it twice. + # The acknowledgement still matters, because without it the resource manager retries. + # Ignoring a resend is safe because planning is triggered by storage statuses, not by descriptions. + # Before that was the case, ignoring a resend would have dropped a planning request. + # The TUNES handler has always fingerprinted its descriptions this way. + # It can adopt this shared helper separately. + # Every plan is made for exactly one actuator. + # So a description with several actuators is refused on arrival. + # Otherwise it would be acknowledged, and then fail in the background on every round, after a solve. if len(message.actuators) != 1: self._logger.error( "Refusing an FRBC.SystemDescription with %s actuators: this CEM plans for exactly one.", @@ -242,8 +243,8 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message - # Said once per description, since nothing here would otherwise say it at all: - # an instruction can ask for a switch that a running timer or a missing transition forbids, and the device refuses it. + # Warn once per description, because nothing else here would mention it. + # An instruction can ask for a switch that a running timer or a missing transition forbids, and then the device refuses it. actuator = message.actuators[0] if _transitions_restrict(actuator): self._logger.warning( @@ -252,13 +253,16 @@ def handle_system_description( len(actuator.transitions), ) - # A description says what the device can do, and a storage status what it holds now, - # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), - # but a description that arrives after the latest status, and differs from the description before it, - # is answered with a schedule against that latest status. - # Anything but the message id counts as a difference, its validity included, - # so a resource manager that re-dates its description on every resend asks for a schedule every time. - # Otherwise a resource manager that sends its status first, or whose fill level holds steady so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. + # A description says what the device can do, and a storage status says what it holds now. + # A schedule needs both. + # Planning is normally triggered by a storage status (see handle_storage_status). + # But suppose a description arrives after the latest status, and differs from the previous description. + # Then the CEM plans a schedule from that latest status. + # Any difference other than the message id counts, including the validity period. + # So a resource manager that re-dates its description on every resend triggers a schedule every time. + # Without this, two kinds of resource managers would be served badly. + # One that sends its status first would never be planned for. + # One whose fill level holds steady sends no new status, so it would keep being planned for with capabilities it no longer has. if self._storage_status_history: request_number = self._count_request(STORAGE_STATUS_REQUESTS) task = asyncio.create_task( @@ -353,8 +357,8 @@ def handle_fill_level_target_profile( @register(FRBCTimerStatus) def handle_frbc_timer_status(self, message: FRBCTimerStatus) -> pydantic.BaseModel: - # Kept, so that what the resource manager said is there to see, though nothing plans with it yet. - # Warned about once per timer, since a resource manager sends a status on every run of every timer. + # Store the timer status, so that it can be inspected, even though no planning uses it yet. + # Warn only once per timer, because a resource manager sends a status every time any timer runs. warned = any( known.timer_id == message.timer_id for known in self._timer_status_history.values() @@ -425,7 +429,7 @@ async def trigger_schedule( actuator = system_description.actuators[0] - # Imported here, since the utilities import from this package. + # Import here, because the utilities module imports from this package. from flexmeasures_client.s2.control_types.FRBC.utils import ( normal_operation_modes, ) @@ -444,13 +448,15 @@ async def trigger_schedule( def _transitions_restrict(actuator) -> bool: - """Whether an actuator's timers and transitions rule out a switch between two of its operation modes. + """Return whether an actuator's timers and transitions rule out a switch between two of its operation modes. - Transitions that allow every switch between modes usable under normal conditions, with no timer to wait for, - rule out nothing, as an on/off device that declares both of its switches does, - so only where something is ruled out is there anything this CEM fails to honour. + Some transitions rule out nothing. + That is the case when they allow every switch between the modes usable under normal conditions, and there is no timer to wait for. + An example is an on/off device that declares both of its switches. + This CEM does not honour timers and transitions, but that only matters if they rule something out. """ - # A transition can only be blocked by a timer the actuator declares, which s2python checks, so timers cover that too. + # A transition can only be blocked by a timer that the actuator declares (s2python checks this). + # So checking for timers covers blocked transitions too. if actuator.timers: return True if not actuator.transitions: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 2b3bcb9f..bfaad55a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -50,39 +50,41 @@ translate_usage_forecast_to_fm, ) -#: The unit S2 states a power value in, and so the unit of an operation mode's power ranges. -#: A schedule is read back in this unit, whatever unit the power sensor holding it is recorded in, -#: and what goes out to FlexMeasures in it is what came in from a resource manager in it. +#: The unit of power values in S2, and so the unit of an operation mode's power ranges. +#: A schedule is read back in this unit, whatever unit its power sensor records in. +#: Values sent to FlexMeasures in this unit are passed on as they came in from the resource manager. S2_POWER_UNIT = "W" -#: How many times a schedule is asked for before the storage is left without an instruction for its window. -#: Every attempt costs the server a solve, so this is a handful rather than a vigil. +#: How many times to request a schedule before leaving the storage without an instruction for its window. +#: Every attempt costs the server a solve, so this number is kept small. SCHEDULE_ATTEMPTS = 5 #: How long to wait between two attempts at a schedule. -#: Flat rather than backed off: what it waits out is a post landing or a worker taking the job up, -#: and a resource manager is waiting on an instruction for the whole of it. +#: The wait is fixed, without backoff. +#: It only needs to cover a post landing or a worker picking up the job, and meanwhile a resource manager is waiting for an instruction. SCHEDULE_RETRY_INTERVAL = timedelta(seconds=2) -#: Causes of a failed scheduling job that fail the same way however often the job is asked for again. -#: Asking again only multiplies failed jobs over the server's workers, so these are never asked again. +#: Causes of a failed scheduling job that fail the same way, however often the job is requested again. +#: Requesting again only multiplies failed jobs across the server's workers, so these are never retried. NON_RETRYABLE_SCHEDULE_FAILURES = ( - # A scheduler that records a belief twice collides with its own first write on every attempt. + # A scheduler that records a belief twice collides with its own first write, on every attempt. "UniqueViolation", "duplicate key value violates unique constraint", # An infeasible problem is planned from the same flex model and context every time, so it stays infeasible. - # FlexMeasures answered one with a fallback scheduler until v1.0.0 retired it, and fails the job itself since. + # Until v1.0.0, FlexMeasures handled one with a fallback scheduler. + # Since v1.0.0 retired that fallback, the job itself fails. "InfeasibleProblemException", ) -#: The refusal of a trigger, rather than a failure of the job behind it, that is worth asking again about. -#: A schedule can be asked for before the state of charge it is to be planned from has landed, -#: and FlexMeasures refuses to plan from one it considers too old. +#: Refusals of a trigger (not failures of the job behind it) that are worth retrying. +#: A schedule can be requested before the state of charge to plan from has landed. +#: FlexMeasures refuses to plan from a state of charge that it considers too old. RETRYABLE_TRIGGER_REFUSALS = ("No recent state-of-charge value found",) -#: What a request to FlexMeasures can fail with, short of a defect in this code: -#: refused (ValueError), unreachable (ConnectionError, TimeoutError, ClientError), answered oddly (ContentTypeError), -#: or queued and then failed or never picked up (JobFailedError, JobTimeoutError). +#: The exceptions a request to FlexMeasures can raise when there is no defect in this code. +#: The request may be refused (ValueError), the server unreachable (ConnectionError, TimeoutError, ClientError), +#: or the response unexpected (ContentTypeError). +#: A queued job may also fail, or never be picked up (JobFailedError, JobTimeoutError). REQUEST_FAILURES = ( JobFailedError, JobTimeoutError, @@ -93,13 +95,13 @@ ClientError, ) -#: What FlexMeasures before v0.33.0 answers a failed scheduling job with, having no job-status API to say so: -#: an HTTP 400, which the client raises as a ValueError with this message and nothing more. +#: How FlexMeasures before v0.33.0 reports a failed scheduling job, since it has no job-status API. +#: It responds with an HTTP 400, which the client raises as a ValueError with only this message. BARE_REFUSAL = "Request failed with status code 400" def _merged(ranges: list[tuple[float, float]]) -> list[tuple[float, float]]: - """The union of closed ranges, as the fewest ranges that cover it, in order.""" + """Return the union of closed ranges, as the fewest ranges that cover it, in order.""" merged: list[tuple[float, float]] = [] for lower, upper in sorted(ranges): if merged and lower <= merged[-1][1]: @@ -110,7 +112,10 @@ def _merged(ranges: list[tuple[float, float]]) -> list[tuple[float, float]]: def _without_negative_usage(usage_forecast): - """A usage forecast with each negative expected rate taken as no usage, as one is recorded and planned against.""" + """Return a usage forecast in which each negative expected rate is replaced by zero. + + This is how a usage forecast is recorded and planned against. + """ if usage_forecast is None or all( element.usage_rate_expected >= 0 for element in usage_forecast.elements ): @@ -132,26 +137,27 @@ def _without_negative_usage(usage_forecast): def _operation_mode_power_bands( power_ranges: list[tuple[float, float]], power_unit: str ) -> list[dict[str, list[str]]]: - """Turn the power ranges an actuator states into the bands FlexMeasures plans within. - - A device with operation modes may only run inside one of its bands, - so an on/off heater is asked for as {0 W} and {883.7 W} rather than as everything in between, - which the resource manager would have to round into a full-on block, - overshooting whatever capacity the site is held to. - - S2 states one signed range, where positive means consumption, - while FlexMeasures asks for the two directions separately, each non-negative. - So a range is split at zero: its non-negative part is the consumption range, - its negative part is the production range with the sign flipped, - and a range spanning zero becomes both, each starting at zero. - - :param power_ranges: lower and upper bound of each range the actuator states, signed as S2 states them. - :param power_unit: the unit those bounds are in. - :returns: one band per distinct range, in the order the actuator stated them. + """Turn the power ranges an actuator declares into the bands FlexMeasures plans within. + + A device with operation modes may only run inside one of its bands. + So an on/off heater is requested as {0 W} and {883.7 W}, not as everything in between. + Otherwise, the resource manager would have to round a value in between into a full-on block. + That block would overshoot any capacity limit the site is held to. + + S2 gives one signed range, where positive means consumption. + FlexMeasures expects the two directions separately, each non-negative. + So a range is split at zero. + Its non-negative part becomes the consumption range. + Its negative part, with the sign flipped, becomes the production range. + A range that spans zero becomes both, each starting at zero. + + :param power_ranges: lower and upper bound of each range the actuator declares, signed as in S2. + :param power_unit: the unit of those bounds. + :returns: one band per distinct range, in the order the actuator declared them. """ - # Zero is the first argument of each max() below, so that a bound of exactly zero is stated as zero, - # rather than as the negative zero that flipping a sign leaves behind. + # Zero is the first argument of each max() below, so that a bound of exactly zero is written as zero, + # not as the negative zero that flipping a sign leaves behind. bands: list[dict[str, list[str]]] = [] for lower, upper in power_ranges: band: dict[str, list[str]] = {} @@ -166,7 +172,7 @@ def _operation_mode_power_bands( f"{-lower} {power_unit}", ] if not band: - # A range of exactly {0}: the mode in which the actuator draws nothing either way. + # A range of exactly {0}: the mode in which the actuator neither consumes nor produces. band["consumption-range"] = [f"0 {power_unit}", f"0 {power_unit}"] if band not in bands: bands.append(band) @@ -181,14 +187,14 @@ class FRBCSimple(FRBC): _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None - #: Where what a resource manager declares about its storage is recorded, each optional: - #: without one, that declaration is not recorded, rather than being posted somewhere else. + #: The sensors to record the resource manager's declarations about its storage on. Each one is optional. + #: Without a sensor, that declaration is not recorded at all, instead of being posted somewhere else. _soc_minima_sensor_id: int | None _soc_maxima_sensor_id: int | None _usage_forecast_sensor_id: int | None _leakage_behaviour_sensor_id: int | None - #: Where what the operation modes say about converting drawn power into stored energy is recorded, - #: for the scheduler to plan the storage's conversion with. + #: The sensor to record the operation modes' conversion of drawn power into stored energy on. + #: The scheduler uses it to plan the storage's conversion. _charging_efficiency_sensor_id: int | None _resolution: timedelta _schedule_duration: timedelta @@ -196,16 +202,16 @@ class FRBCSimple(FRBC): _fill_level_scale: float _hard_bound_factor: float #: Whether the power sensor records consumption as positive, as S2 does. - #: Read from the sensor itself the first time a power value is posted, and remembered after that. - #: None until it has been read. + #: This is read from the sensor itself the first time a power value is posted, and remembered after that. + #: It is None until it has been read. _power_sensor_takes_consumption_as_positive: bool | None - #: When a power value was last recorded for the actuator, and for which moment, - #: so that a second status inside one event does not collide with the first. + #: For each moment, when a power value was last recorded for the actuator. + #: This prevents a second status within one event from colliding with the first. _power_recorded_at: dict[datetime, datetime] - #: The charging efficiency last recorded, and the window it was recorded for, - #: so that the same value for the same window is not recorded again on every schedule. + #: The last recorded charging efficiency, and the window it was recorded for. + #: This prevents recording the same value for the same window again on every schedule. _charging_efficiency_recorded: tuple[datetime, float] | None - #: How many times a schedule is asked for, and how long between two attempts. + #: How many times to request a schedule, and how long to wait between two attempts. _schedule_attempts: int _schedule_retry_interval: timedelta @@ -215,52 +221,55 @@ def __init__( soc_sensor_id: int, rm_discharge_sensor_id: int, price_sensor_id: int, - # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. - # Without one, both directions are priced by price_sensor_id. + # Use this if feeding power back is priced differently from drawing it, as with a feed-in tariff. + # Without it, both directions are priced by price_sensor_id. production_price_sensor_id: int | None = None, # Whether the site's connection capacity may be breached at a price (True) or never (False). - # Unset, it is left to FlexMeasures, which since v1.0 relaxes it at a high default breach price, - # under its relax-constraints umbrella, and before that did not. - # A home whose connection is a physical limit can pass False; - # a site whose capacity is a target set by others, as in a community steering its members, can pass True. + # If unset, FlexMeasures decides. + # Since v1.0, FlexMeasures relaxes it at a high default breach price, under its relax-constraints umbrella. + # Before v1.0, it did not relax it. + # A home whose connection is a physical limit can pass False. + # A site whose capacity is a target set by others, as in a community that steers its members, can pass True. relax_site_capacity: bool | None = None, - # Optional so existing callers keep working; when absent the actuator's fill rate is - # simply not recorded, rather than being posted onto the power sensor. + # Optional, so that existing callers keep working. + # Without it, the actuator's fill rate is not recorded at all, instead of being posted onto the power sensor. fill_rate_sensor_id: int | None = None, - # Likewise optional, for the three things a resource manager declares about its storage. + # Also optional: the sensors for three declarations a resource manager makes about its storage. soc_minima_sensor_id: int | None = None, soc_maxima_sensor_id: int | None = None, usage_forecast_sensor_id: int | None = None, - # Its resolution has to be the handler's resolution: a storage efficiency is what remains after one step, - # computed here for a step of the handler's resolution, - # and FlexMeasures reads it as one step of the sensor's own resolution. + # The resolution of this sensor must equal the handler's resolution. + # A storage efficiency is what remains after one step, and this handler computes it for a step of its own resolution. + # FlexMeasures reads it as one step of the sensor's resolution. leakage_behaviour_sensor_id: int | None = None, - # Likewise optional: without a sensor to record it on, no charging efficiency is declared, - # and the scheduler plans the storage as if the actuator converted power into stored energy one for one. + # Also optional. + # Without a sensor to record it on, no charging efficiency is declared. + # Then the scheduler plans the storage as if the actuator converted power into stored energy one to one. charging_efficiency_sensor_id: int | None = None, timezone: str = "UTC", schedule_duration: timedelta = timedelta(hours=12), - # How many times a schedule is asked for before the storage is left without an instruction, - # and how long between two attempts. - # One attempt means a failure is never answered by another, as it was before this was bounded at all. + # How many times to request a schedule before leaving the storage without an instruction, + # and how long to wait between two attempts. + # One attempt means a failure is never retried. + # Before this limit existed, a failure was always retried with another attempt. schedule_attempts: int = SCHEDULE_ATTEMPTS, schedule_retry_interval: timedelta = SCHEDULE_RETRY_INTERVAL, - # The resolution a schedule is planned on, which a schedule's start has to line up with. + # The resolution that schedules are planned on; a schedule's start must line up with it. resolution: timedelta = timedelta(minutes=15), max_size: int = 100, valid_from_shift: timedelta = timedelta(days=1), fill_level_scale: float = 1, - # How far above the declared fill-level range the hard upper bound of a schedule sits. - # Half again as high, so that a storage overshooting its declared range stays feasible. - # A deployment whose storage cannot overshoot at all can bring this down to 1. + # How far above the declared fill-level range a schedule's hard upper bound sits. + # The default is 1.5 times as high, so that a storage that overshoots its declared range stays feasible. + # A deployment whose storage cannot overshoot at all can lower this to 1. hard_bound_factor: float = 1.5, - # The unit a power value this handler posts is converted into and labelled with. - # Those values come from the operation modes' power ranges, which S2 states in watts. + # The unit that this handler converts posted power values into, and labels them with. + # These values come from the operation modes' power ranges, which S2 gives in watts. power_unit: str = "W", - # The unit a fill level is recorded in: a state of charge and the minima and maxima - # bounding it all go out in this one unit, so that they can be compared at all. - # Kilowatt hours, because S2 is a residential protocol and a home battery or heat buffer - # is of that order; a deployment at another scale says so here. + # The unit that fill levels are recorded in. + # A state of charge, and the minima and maxima that bound it, are all sent in this one unit, so that they can be compared. + # The default is kilowatt-hours, because S2 is a residential protocol, and a home battery or heat buffer is of that order. + # A deployment at another scale can set a different unit here. energy_unit: str = "kWh", ) -> None: super().__init__(max_size) @@ -289,14 +298,14 @@ def __init__( f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." ) if schedule_duration % resolution: - # FlexMeasures answers with whole steps of the power sensor's resolution only, - # so a duration between two whole numbers of steps comes back with fewer values than it spans, - # and the schedule's step, read as its duration over its number of values, would come out wrong. + # FlexMeasures only returns whole steps of the power sensor's resolution. + # So a duration that is not a whole number of steps comes back with fewer values than it spans. + # The schedule's step is computed as its duration divided by its number of values, so it would come out wrong. raise ValueError( f"schedule_duration has to be a whole number of resolution steps, but {schedule_duration} is not a multiple of {resolution}." ) if schedule_attempts < 1: - # A schedule is asked for at least once, or no schedule is asked for at all. + # A schedule must be requested at least once, or there is no schedule at all. raise ValueError( f"schedule_attempts has to be at least one, since it counts the attempts at a schedule rather than the retries, but {schedule_attempts} was given." ) @@ -304,9 +313,9 @@ def __init__( self._schedule_retry_interval = schedule_retry_interval self._timezone = ZoneInfo(timezone) if fill_level_scale <= 0: - # Every fill level this handler records is multiplied by the scale, - # and the hard bounds a schedule was planned against are divided back by it, - # so zero divides by zero and a negative scale turns the bounds upside down. + # Every fill level this handler records is multiplied by the scale. + # The hard bounds that a schedule was planned against are divided by it again. + # So a scale of zero would divide by zero, and a negative scale would turn the bounds upside down. raise ValueError( f"fill_level_scale has to be greater than zero, since it converts a fill level into the unit it is recorded in, but {fill_level_scale} was given." ) @@ -316,17 +325,17 @@ def __init__( self._soc_sensor_records_instants: bool | None = None self._planning = asyncio.Lock() if hard_bound_factor < 1: - # Below one, the hard upper bound sits inside the declared range, - # and FlexMeasures clips the band into the hard bounds, - # so the band would be narrowed to something the resource manager never asked for. + # Below one, the hard upper bound sits inside the declared range. + # FlexMeasures clips the band to the hard bounds. + # So the band would be narrowed to something the resource manager never asked for. raise ValueError( f"hard_bound_factor has to be at least 1, so that the hard upper bound sits at or above the declared fill-level range, but {hard_bound_factor} was given." ) self._hard_bound_factor = hard_bound_factor self.power_unit = power_unit if energy_unit not in WATTS_PER_ENERGY_UNIT_PER_SECOND: - # A fill rate is per second of this unit, and has to be compared with the watts S2 states a power in, - # so a unit that cannot be converted to watts cannot be planned with. + # A fill rate is in this unit per second, and must be compared with power values, which S2 gives in watts. + # So a unit that cannot be converted to watts cannot be planned with. raise ValueError( f"energy_unit has to be one of {', '.join(WATTS_PER_ENERGY_UNIT_PER_SECOND)}, but {energy_unit!r} was given." ) @@ -364,12 +373,14 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - # Floored on the moment's own clock, in the offset it carries, which is how FlexMeasures floors a series posted with it, - # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. - # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two that is not after the moment: taking the first pass every time put a boundary an hour or more too early. - # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. - # Compared as pandas timestamps, which compare instants: two datetimes on one local clock compare by wall time, - # whichever pass each is on. + # Floor in the moment's own timezone offset, which is how FlexMeasures floors a series posted with that offset. + # So an hourly step in a timezone that is half an hour off UTC, or a daily step, starts where FlexMeasures' steps start. + # During the repeated hour of an autumn DST change, a boundary occurs twice. + # The intended boundary is the later of the two that is not after the moment. + # Always taking the first pass put the boundary an hour or more too early. + # A boundary in a spring DST gap (which only a resolution of more than an hour can hit) moves to the end of the gap. + # Compare as pandas timestamps, which compare instants. + # Two datetimes in the same local timezone compare by wall time, whichever pass each is on. at = pd.Timestamp(moment) candidates = [ at.floor( @@ -381,15 +392,16 @@ def align_to_resolution(self, moment: datetime) -> datetime: return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted and floored to a boundary, but never before now. - - Never before now, since planning happens on every storage status while a description may be days old: - anchored to that description alone, every status in a long session was answered with the same window, - which slid into the past, so its instructions were all due already and arrived as one burst. - Where now is past that boundary, the schedule starts at the next one, not at the start of the step now is in. - A schedule is planned from the fill level reported now, and FlexMeasures takes that to be the level at its start, - so a schedule starting at a step already under way was planned as if the whole step were left, - and every replan within it asked for less power over what remained of it, delivering less than planned. + """Return the start of a schedule requested for valid_from: shifted and floored to a boundary, but never before now. + + Planning happens on every storage status, while a description may be days old. + If the start were based on that description alone, every status in a long session would get the same window. + That window slid into the past, so all its instructions were already due, and arrived as one burst. + Hence the start is never before now. + If now is past that boundary, the schedule starts at the next boundary, not at the start of the current step. + A schedule is planned from the fill level reported now, and FlexMeasures takes that as the fill level at the schedule's start. + So a schedule that started at a step already under way was planned as if the whole step remained. + Every replan within that step then asked for less power over the rest of it, and delivered less than planned. The step under way keeps the instruction it is already following. """ now = self.now() @@ -400,12 +412,13 @@ def schedule_start(self, valid_from: datetime) -> datetime: return current if current == now else current + self._resolution async def _soc_duration(self) -> timedelta: - """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. + """Return the duration to post a reported fill level with. - Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. - FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, - so on any other sensor, or one that cannot be read, it is posted over a minute, as before. - The sensor is asked until it answers, and its answer kept, since its resolution is a property of how it was set up. + If the state-of-charge sensor records instants, the duration is zero. + On such a sensor, a reading posted over the following minute was taken as a forecast of that minute. + However, FlexMeasures refuses a single value without duration on a sensor that has a resolution. + So on any other sensor, or on a sensor that cannot be read, the reading is posted over a minute, as before. + The sensor is queried until it responds, and the response is kept, because its resolution is fixed when the sensor is set up. """ if self._soc_sensor_records_instants is None: try: @@ -438,17 +451,18 @@ async def _soc_duration(self) -> timedelta: async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): - # Read by this handler's clock, and known as of then, which in a co-simulation is the simulated time: - # without a prior, FlexMeasures took the moment the request reached it, - # and a simulated reading was recorded as known years after it was taken. + # Use this handler's clock for both the reading time and the belief time (in a co-simulation, that is the simulated time). + # Without a prior, FlexMeasures used the moment the request reached it as the belief time. + # A simulated reading was then recorded as known years after it was taken. now = self.now() - # Guarded, because a failed post would otherwise unwind this method before the planning below, - # and a schedule starts from the state of charge this status reports, sent inline, not from the one recorded here. + # Catch failures here, because otherwise a failed post would exit this method before the planning below. + # Planning does not depend on this post: a schedule starts from the state of charge in this status, which is sent inline. try: await self._fm_client.post_measurements( self._soc_sensor_id, start=now, - # Scaled like every other fill level this handler records, or the level and the band around it would be in different scales wherever fill_level_scale is not 1. + # Scale this like every other fill level this handler records. + # Otherwise, if fill_level_scale is not 1, the fill level and the band around it would be on different scales. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, duration=await self._soc_duration(), @@ -462,10 +476,10 @@ async def send_storage_status( now, ) - # The state of charge this status reports is what a schedule starts from, so this is - # where a schedule is asked for. - # A system description is needed to turn that schedule into instructions. - # Without one yet, nothing is planned here, and the description plans against this status when it arrives. + # A schedule starts from the state of charge in this status, so this is where a schedule is requested. + # Turning that schedule into instructions requires a system description. + # If none has arrived yet, nothing is planned here. + # Then the description triggers planning from this status when it arrives. if not self._system_description_history: self._logger.warning( "Cannot ask for a schedule: no FRBC.SystemDescription has arrived yet." @@ -562,7 +576,7 @@ def _in_the_power_unit(self, power: float) -> float: @staticmethod def _latest(history: dict): - """The message a history holds that arrived last, or None while it holds none.""" + """Return the last message that arrived in a history, or None if the history is empty.""" return list(history.values())[-1] if history else None def _fill_level_band( @@ -572,14 +586,14 @@ def _fill_level_band( soc_unit: str, to_soc_unit: float, ) -> tuple[list[dict], list[dict]]: - """The fill levels to keep the storage between over the window planned, as time segments FlexMeasures takes. - - The declared fill-level range applies to every step, and the latest fill-level target profile, - held inside that range, applies where it covers a step. - The range used to reach FlexMeasures only through the band sensors, and only where a target profile was recorded on them, - so a storage without the sensors, or beyond the profile's end, was planned with no band at all, - down to empty and half again above its range. - Clipped at zero, since FlexMeasures takes no negative minima. + """Return the fill levels to keep the storage between over the planned window, as time segments for FlexMeasures. + + The declared fill-level range applies to every step. + The latest fill-level target profile, held inside that range, applies to the steps it covers. + The range used to reach FlexMeasures only through the band sensors, and only where a target profile was recorded on them. + So a storage without those sensors, or past the profile's end, was planned without any band. + It could then be planned down to empty, and up to half again above its range. + The values are clipped at zero, because FlexMeasures does not accept negative minima. """ steps = int( pd.Timedelta(self._schedule_duration) / pd.Timedelta(self._resolution) @@ -608,11 +622,12 @@ def _fill_level_band( ) def _on_the_window(self, profile: pd.Series, index: pd.DatetimeIndex) -> pd.Series: - """A target profile's values at the window's steps, by the instant each step starts at, where the profile covers it. + """Return a target profile's values at the window's steps, for the steps that the profile covers. - The profile is floored to steps on its own clock, and the window on the description's, - so in a time zone half an hour off the other the two sets of steps never share an instant, - and matching them by label dropped the whole profile without a word. + Steps are matched by the instant at which they start. + The profile is floored to steps in its own timezone, and the window in the description's timezone. + If the two timezones are half an hour apart, the two sets of steps never share an instant. + Matching them by label then silently dropped the whole profile. """ if profile.empty: return profile @@ -621,7 +636,7 @@ def _on_the_window(self, profile: pd.Series, index: pd.DatetimeIndex) -> pd.Seri return at_steps[(index >= profile.index[0]) & (index < covered_until)] def _as_segments(self, series: pd.Series, unit: str) -> list[dict]: - """A series on this handler's resolution as FlexMeasures' time segments, one per run of equal values.""" + """Convert a series on this handler's resolution into FlexMeasures time segments, one per run of equal values.""" segments: list[dict] = [] for moment, value in series.items(): end = moment + pd.Timedelta(self._resolution) @@ -641,7 +656,7 @@ def _as_segments(self, series: pd.Series, unit: str) -> list[dict]: return segments def _soc_unit(self) -> tuple[str, float]: - """The unit a state of charge goes to FlexMeasures in, and what a fill level is multiplied by to be in it.""" + """Return the unit for sending a state of charge to FlexMeasures, and the factor that converts a fill level into it.""" if self.energy_unit in ("kWh", "MWh"): return self.energy_unit, 1.0 return "kWh", ( @@ -650,7 +665,10 @@ def _soc_unit(self) -> tuple[str, float]: ) def _latest_fill_level(self) -> float | None: - """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" + """Return the fill level from the latest storage status, in the resource manager's own units. + + Return None if there is no storage status, or if its fill level is not a number. + """ if not self._storage_status_history: return None fill_level = list(self._storage_status_history.values())[-1].present_fill_level @@ -658,8 +676,8 @@ def _latest_fill_level(self) -> float | None: async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor - # After a reconnection an actuator status can come before any description, - # which this needs to tell what power the status stands for. + # After a reconnection, an actuator status can arrive before any description. + # This method needs a description to tell what power the status stands for. if not self._system_description_history: self._logger.warning( "Not recording the power of an FRBC.ActuatorStatus: no FRBC.SystemDescription has arrived yet to tell what its operation mode draws." @@ -697,13 +715,13 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): ) operation_mode = actuator.operation_modes[0] - # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, - # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. + # The element the actuator runs on depends on the fill level, just as a heat pump derates while its buffer warms up. + # So use the element that applies at the latest reported fill level, not the element that the mode happens to list first. fill_level = self._latest_fill_level() if fill_level is not None: element = element_for_fill_level(operation_mode.elements, fill_level) else: - # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. + # Assume the bottom of the range, which is as good a guess as any, and log below that it is a guess. element = element_for_fill_level(operation_mode.elements, float("-inf")) if len(operation_mode.elements) > 1: self._logger.warning( @@ -728,21 +746,19 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): start = status.transition_timestamp or self.now() - # A belief time that supersedes the one before it, rather than colliding with it. - # FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, - # and it floors the event to the sensor's resolution. - # Two statuses inside one resolution step, milliseconds apart, were refused in practice, - # with a duplicate key whose belief time had lost its sub-second part. - # Posting two such beliefs by hand does not reproduce that, - # so the exact rule is the server's to state. - # Either way, a second apart is unambiguous: the later status supersedes the earlier one, - # which is what a resource manager reporting twice in a second means. - # A resource manager sends a status for each instruction it applies, - # and this handler sends several at once, so that is the ordinary case rather than an edge. - # The comparison is on the event as FlexMeasures keys it, - # since two statuses seconds apart still land on the same resolution step. - # Kept per event, since statuses for different steps can interleave within a second, - # and a comparison with the latest status alone then missed the one before it about the same step. + # Pick a belief time that supersedes the previous one, instead of colliding with it. + # FlexMeasures keys a belief on its sensor, its source, the event, and the moment the belief was formed. + # It floors the event to the sensor's resolution. + # In practice, two statuses within one resolution step, milliseconds apart, were refused. + # The error was a duplicate key, with a belief time that had lost its sub-second part. + # Posting two such beliefs by hand does not reproduce that, so only the server can say what the exact rule is. + # Either way, belief times a second apart are unambiguous: the later status supersedes the earlier one. + # That is what a resource manager means when it reports twice within a second. + # A resource manager sends a status for each instruction it applies, and this handler sends several instructions at once. + # So this is the ordinary case, not an edge case. + # The comparison uses the event as FlexMeasures keys it, because two statuses seconds apart still land on the same resolution step. + # Keep the times per event, because statuses for different steps can interleave within a second. + # Comparing with the latest status alone then missed the earlier status for the same step. event = self.align_to_resolution(start) recorded_at = self.now() last_recorded_at = self._power_recorded_at.get(event) @@ -751,7 +767,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): ): recorded_at = last_recorded_at + timedelta(seconds=1) self._power_recorded_at[event] = recorded_at - # Steps more than a day older than this one are forgotten, to keep this bounded: a status that late is not expected. + # Forget steps more than a day older than this one, to keep this bounded. A status that late is not expected. for old_event in [ e for e in self._power_recorded_at if e < event - timedelta(days=1) ]: @@ -796,7 +812,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): fill_rate.start_of_range + (fill_rate.end_of_range - fill_rate.start_of_range) * factor ) * self._fill_level_scale - # Guarded like the power above, since this runs in a task nobody awaits. + # Catch failures, like for the power above, because this runs in a task that nobody awaits. try: await self._fm_client.post_sensor_data( self._fill_rate_sensor_id, @@ -820,18 +836,18 @@ async def _record_charging_efficiency( start: datetime, fill_level: float, ) -> bool: - """Record what the actuator converts drawn power into stored energy at, over the window a schedule covers. - - FlexMeasures reads a charging efficiency from a sensor rather than taking a value in the flex model, - so what the operation modes say about it is recorded here, by the call that is about to plan with it. - Recorded before the schedule is asked for, and not alongside it: - a schedule planned against a sensor holding nothing for its window is planned as if the actuator converted power one for one. - - :param start: where the schedule starts, which is where the value is recorded from. - :param fill_level: the fill level the schedule starts from, in the resource manager's own units, - which picks the element of each mode that answers. - :returns: whether that schedule may be planned against the sensor, - which it may not where nothing is recorded on it for this window. + """Record the efficiency at which the actuator converts drawn power into stored energy, over the window a schedule covers. + + FlexMeasures reads a charging efficiency from a sensor, instead of taking a value in the flex model. + So the efficiency that the operation modes imply is recorded here, by the call that is about to plan with it. + It is recorded before the schedule is requested, not at the same time. + A schedule planned against a sensor that holds nothing for its window is planned as if the actuator converted power one to one. + + :param start: where the schedule starts; the value is recorded from there. + :param fill_level: the fill level the schedule starts from, in the resource manager's own units. + It decides which element of each mode is used. + :returns: whether the schedule may be planned against the sensor. + It may not, if nothing is recorded on the sensor for this window. """ if self._charging_efficiency_sensor_id is None: return False @@ -839,8 +855,8 @@ async def _record_charging_efficiency( # One actuator, as everywhere else in this handler. actuator = system_description.actuators[0] efficiencies: dict[str, float] = {} - # The energy unit is one fill_rate_in_watts knows, which the constructor saw to. - # Only modes it may be instructed into under normal conditions, which are the ones the schedule is planned in. + # The constructor already checked that fill_rate_in_watts knows the energy unit. + # Only use the modes the actuator may be instructed into under normal conditions, because the schedule is planned in those. for operation_mode in normal_operation_modes(actuator): efficiency = operation_mode_conversion_efficiency( operation_mode, self._fill_level_scale, self.energy_unit, fill_level @@ -884,8 +900,8 @@ async def _record_charging_efficiency( # and for a heat pump, whose fill rate is the heat it delivers, it is well above one. unit="dimensionless", duration=self._schedule_duration, - # Remembered below as recorded, and so never posted again for this window: - # an ingestion still pending at the timeout is not taken as recorded. + # Below, the value is remembered as recorded, and is then never posted again for this window. + # So an ingestion that is still pending at the timeout must not count as recorded. confirm_ingestion=True, ) except (*REQUEST_FAILURES, NotImplementedError): @@ -921,14 +937,14 @@ async def _is_worth_asking_again( message = str(failure) if any(marker in message for marker in NON_RETRYABLE_SCHEDULE_FAILURES): return False - # A job nobody picked up in time, and a server that could not be reached, may both be there for the next attempt: - # a scheduling queue without a live worker, and a gateway restarting, are the likeliest of either. - # A timeout has already waited out the job's whole timeout, and a dead queue will not come back two seconds later, - # so it is asked about once more, not on every attempt allowed. + # A job that nobody picked up in time, or a server that could not be reached, may both work on the next attempt. + # The likeliest causes are a scheduling queue without a live worker, and a gateway that is restarting. + # A timeout has already waited for the job's whole timeout, and a dead queue will not come back two seconds later. + # So after a timeout, retry only once, not on every allowed attempt. if isinstance(failure, JobTimeoutError): return attempt == 1 - # The client wraps every HTTP error in a ConnectionError, - # and one answering 4xx, such as an endpoint an old server lacks, answers the same next time. + # The client wraps every HTTP error in a ConnectionError. + # A 4xx response, such as for an endpoint that an old server lacks, will be the same next time. cause = failure.__cause__ if isinstance(cause, ClientResponseError) and cause.status < 500: return False @@ -939,11 +955,11 @@ async def _is_worth_asking_again( return True if any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS): return True - # FlexMeasures before v0.33.0 reports a failed job only as a bare HTTP 400, - # which cannot be told apart from a refused trigger, so it is asked again, within the attempts allowed. + # FlexMeasures before v0.33.0 reports a failed job only as a bare HTTP 400. + # That cannot be told apart from a refused trigger, so retry it, within the allowed number of attempts. if message != BARE_REFUSAL: return False - # Looked up inside the handling of another failure, so a failure of its own must not take that one's place. + # This lookup happens while handling another failure, so a failure of the lookup itself must not replace that one. try: return not await self._fm_client.server_version_at_least("0.33.0") except REQUEST_FAILURES: @@ -956,22 +972,22 @@ async def _ask_for_a_schedule( flex_model: dict, request_number: int | None = None, ) -> dict | list[dict] | None: - """Ask FlexMeasures for a schedule, and ask again where the attempt is worth making again. - - One failed scheduling job otherwise leaves the storage without an instruction for its window. - The resource manager is told nothing, and goes on polling, - and since this runs as a task whose exception is logged and dropped, nothing else notices either. - That is what one failed job did in a co-simulation: - a worker crashed inside its solver, - and the resource manager it belonged to polled until its round timed out, every round, for the rest of the run. - - :param start: the window the schedule is asked for, which every attempt asks for again. - :param request_number: the planning request this answers; where a newer one arrives between attempts, - no further attempt is made and None is returned, so that the newer one is planned sooner. - :raises JobFailedError: where the last attempt worth making failed inside the server. - :raises JobTimeoutError: where the last attempt worth making was not picked up in time. - :raises ConnectionError: where the server could not be reached on the last attempt worth making. - :raises ValueError: where the trigger itself was refused. + """Ask FlexMeasures for a schedule, and retry if the failure is worth retrying. + + Without retries, one failed scheduling job leaves the storage without an instruction for its window. + The resource manager is not told, and keeps polling. + This runs as a task whose exception is logged and dropped, so nothing else notices either. + That happened in a co-simulation, after a worker crashed inside its solver. + The affected resource manager polled until its round timed out, in every round, for the rest of the run. + + :param start: the start of the window to request the schedule for; every attempt requests the same window. + :param request_number: the planning request this call serves. + If a newer request arrives between attempts, no further attempt is made, and None is returned. + That way, the newer request is planned sooner. + :raises JobFailedError: if the last attempt worth making failed inside the server. + :raises JobTimeoutError: if the last attempt worth making was not picked up in time. + :raises ConnectionError: if the server could not be reached on the last attempt worth making. + :raises ValueError: if the trigger itself was refused. """ attempt = 0 while True: @@ -1023,19 +1039,20 @@ async def _ask_for_a_schedule( def _warn_where_a_fallback_scheduler_answered( self, schedule: dict | list[dict], start: datetime ) -> None: - """Say so where a schedule was planned by a scheduler this handler did not ask for. + """Log an error if a schedule was planned by a scheduler that this handler did not ask for. - FlexMeasures answered an infeasible problem with a fallback scheduler until v1.0.0 retired it. - Against an older server, the schedule endpoint follows that fallback job rather than reporting the failure, - or redirects to it where the server sets FLEXMEASURES_FALLBACK_REDIRECT, which this client follows, - so either way a schedule nobody asked for arrives looking like the one that was. - It is a coarse charging policy rather than a plan, made without the band, the efficiencies and the power bands this handler asked with. + Until v1.0.0 retired it, FlexMeasures handled an infeasible problem with a fallback scheduler. + On an older server, the schedule endpoint follows that fallback job instead of reporting the failure. + Or, if the server sets FLEXMEASURES_FALLBACK_REDIRECT, it redirects to the fallback job, and this client follows the redirect. + Either way, a schedule that nobody asked for arrives looking like the requested one. + It is a coarse charging policy, not a plan. + It ignores the band, the efficiencies and the power bands that this handler sent with its request. - :param schedule: what came back, which names the scheduler that planned it. - :param start: the window it was asked for. + :param schedule: the response, which names the scheduler that planned it. + :param start: the start of the window it was requested for. """ if not isinstance(schedule, dict): - # A schedule per device, which only an asset-wide trigger is answered with. + # A schedule per device, which is only returned for an asset-wide trigger. return scheduler = str((schedule.get("scheduler_info") or {}).get("scheduler", "")) if "fallback" not in scheduler.lower(): @@ -1049,19 +1066,21 @@ def _warn_where_a_fallback_scheduler_answered( async def trigger_schedule( self, system_description_id: str, request_number: int | None = None ): - """Plan, one schedule at a time, and only for the newest request once it is this one's turn. + """Plan one schedule at a time, and when it is this request's turn, only plan if it is the newest request. Each storage status is a planning request. - One arriving while a schedule was being planned used to start a second solve beside it, whose answer replaced the first's, - so a resource manager reporting faster than FlexMeasures plans kept several solves running for nothing. - Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile; - the one in flight, once out of date, makes no further attempt after a failed one. - So a burst of statuses costs at most the attempt in flight and one plan more, for the newest, - which starts once that attempt is over rather than beside it. + A request that arrived while a schedule was being planned used to start a second solve in parallel. + The second result then replaced the first. + So a resource manager that reported faster than FlexMeasures could plan kept several solves running for nothing. + Now a request waits for the one in flight. + It is dropped if a newer request has arrived in the meantime. + Once the request in flight is out of date, it makes no further attempt after a failed one. + So a burst of statuses costs at most the attempt in flight, plus one more plan for the newest request. + That plan starts after the attempt in flight is over, not in parallel. :param system_description_id: system description to base the schedule on - :param request_number: which planning request this schedule answers, which is checked when its turn comes, - between attempts, and before the schedule is translated and sent + :param request_number: the planning request this schedule serves. + It is checked when its turn comes, between attempts, and before the schedule is translated and sent. """ async with self._planning: if request_number is not None and not self._is_latest_request( @@ -1080,7 +1099,7 @@ async def _plan( """Translates S2 System Description into FM API calls. :param system_description_id: system description to base the schedule on - :param request_number: which planning request this schedule answers + :param request_number: the planning request this schedule serves """ system_description: FRBCSystemDescription = self._system_description_history[ @@ -1122,49 +1141,50 @@ async def _plan( min(power_range.start_of_range, power_range.end_of_range), max(power_range.start_of_range, power_range.end_of_range), ) - # Only modes the actuator may be instructed into under normal conditions, - # since a band the scheduler plans in is a mode it may then be instructed into. + # Only use the modes the actuator may be instructed into under normal conditions. + # The actuator may be instructed into any band that the scheduler plans in. for operation_mode in normal_operation_modes(actuator) for element in operation_mode.elements for power_range in element.power_ranges ] - # The declared range is where a resource manager wants its storage kept, not where its storage can be: - # S2 calls it the range in which the fill level should remain, - # and has a resource manager free to ignore instructions while the fill level is outside it. - # So the range goes out as a band whose breach FlexMeasures prices (see _fill_level_band), - # rather than as the hard soc-min and soc-max, which made states a storage really reaches infeasible, - # and hard bounds are still sent, but wide ones. + # The declared range is where a resource manager wants its storage kept, not the limits of what its storage can reach. + # S2 calls it the range in which the fill level should remain. + # While the fill level is outside it, S2 lets a resource manager ignore instructions. + # So the range is sent as a band, and FlexMeasures puts a price on breaching it (see _fill_level_band). + # Sending the range as the hard soc-min and soc-max made states that a storage really reaches infeasible. + # Hard bounds are still sent, but wide ones. _, range_top = get_soc_min_max(system_description, self._fill_level_scale) # Zero below, since a stock cannot be negative and FlexMeasures accepts no lower floor. soc_min = 0.0 - # Multiplied, a top below zero, as S2 allows, would be pulled further down, - # but the bound widens below to take in the state of charge at the start, which is never below zero. + # S2 allows a top below zero, and multiplying it would pull it further down. + # But below, the bound widens to include the state of charge at the start, which is never below zero. soc_max = range_top * self._hard_bound_factor - # A state of charge outside the hard bounds is infeasible before the first step, - # so the bounds widen to take it in. - # A storage outside its declared range is not an error: - # a resource manager is free to report one, and a schedule still has to be planned from where it says its storage is. - # Upward only, since FlexMeasures takes no negative soc-min: - # a storage reported below zero is planned from zero, and said so, rather than refused with no schedule. + # A state of charge outside the hard bounds is infeasible before the first step. + # So the bounds widen to include it. + # A storage outside its declared range is not an error. + # A resource manager is free to report one, and a schedule must still be planned from the reported fill level. + # The bounds only widen upward, because FlexMeasures does not accept a negative soc-min. + # A storage reported below zero is planned from zero, with a warning, instead of being refused without a schedule. if soc_at_start < 0: self._logger.warning( "The storage is reported at %s, below zero, which FlexMeasures cannot plan from; planning from zero instead.", soc_at_start, ) soc_at_start = 0.0 - # And read back from zero, so the instructions follow the plan from where it started. + # Also translate the schedule from zero, so that the instructions follow the plan from where it started. fill_level_at_start = 0.0 soc_max = max(soc_max, soc_at_start) - # Where this schedule starts, which is both what it is asked for, - # and the window a charging efficiency has to be recorded over to be planned against. + # The start of this schedule. + # It is both the start of the requested window and the start of the window that a charging efficiency must be recorded over. start = self.schedule_start(system_description.valid_from) - # FlexMeasures takes a state of charge in kWh or MWh only. - # So a fill level recorded in another energy unit is converted here, at the edge, rather than refused, - # and the bounds read back below stay in the recorded unit. - # The usage and the leakage are read from sensors, which FlexMeasures converts itself, and the band is converted here. + # FlexMeasures only accepts a state of charge in kWh or MWh. + # So a fill level recorded in another energy unit is converted here, at the edge, instead of being refused. + # The bounds used for translating the schedule below stay in the recorded unit. + # The usage and the leakage are read from sensors, which FlexMeasures converts itself. + # The band is converted here. soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { "soc-unit": soc_unit, @@ -1173,13 +1193,13 @@ async def _plan( "soc-min": soc_min * to_soc_unit, "soc-max": soc_max * to_soc_unit, } - # The band the resource manager declared, sent with the request itself rather than named by sensor, - # so that it is there with or without the sensors it is also recorded on. + # Send the band the resource manager declared within the request itself, instead of referring to a sensor. + # That way, the band is included whether or not the sensors it is also recorded on exist. soc_minima, soc_maxima = self._fill_level_band( start, system_description, soc_unit, to_soc_unit ) - # FlexMeasures has the relax-soc-constraints that makes it a band from v0.26.0, - # and a flex context naming a field the server does not know is refused outright, leaving no schedule at all. + # FlexMeasures has relax-soc-constraints, which makes this a band, from v0.26.0 on. + # A server refuses a flex context with a field it does not know, which would leave no schedule at all. if await self._fm_client.server_version_at_least("0.26.0"): flex_model["soc-minima"] = soc_minima flex_model["soc-maxima"] = soc_maxima @@ -1189,9 +1209,9 @@ async def _plan( self._fm_client.server_version, ) if self._usage_forecast_sensor_id is not None: - # Not where the resource manager said it sends no usage forecast, - # since nothing would then be recorded on the sensor for the scheduler to read. - # Or where it sent one anyway, which is what it says it will do, whatever its flag says. + # Skip this if the resource manager says it provides no usage forecast, + # because then nothing is recorded on the sensor for the scheduler to read. + # But use it if the resource manager sent a forecast anyway, whatever its flag says. if ( system_description.storage.provides_usage_forecast or self._usage_forecast_history @@ -1229,16 +1249,17 @@ async def _plan( ) flex_model["production-capacity"] = f"{production_capacity} {S2_POWER_UNIT}" operation_modes = _operation_mode_power_bands(power_ranges, S2_POWER_UNIT) - # Only where the bands say more than the two capacities above already do. - # The capacities say the actuator can run at any power from minus its production capacity to its consumption capacity, - # so bands that together cover exactly that, without a gap, say nothing new, - # such as an idle mode at zero beside a mode running from zero up, and each band costs the solver a binary per step. - # Bands with a gap between them do: an actuator that runs at 200 W or not at all rules out everything in between, - # and so does one that only ever feeds back. - # And only to a server that has the field. - # FlexMeasures added operation-modes in v1.0.0, - # and a flex model naming a field the server does not know is refused outright, - # which would leave the resource manager with no schedule at all. + # Only send the bands if they add information to the two capacities above. + # The capacities say that the actuator can run at any power from minus its production capacity up to its consumption capacity. + # Bands that together cover exactly that range, without a gap, add nothing. + # An example is an idle mode at zero next to a mode running from zero up. + # Each band costs the solver a binary variable per step. + # Bands with a gap between them do add information. + # An actuator that runs at 200 W or not at all rules out everything in between. + # So does an actuator that only ever feeds back. + # Also, only send the bands to a server that has the field. + # FlexMeasures added operation-modes in v1.0.0. + # A server refuses a flex model with a field it does not know, which would leave the resource manager with no schedule at all. says_more_than_the_capacities = _merged(power_ranges) != [ (-production_capacity, consumption_capacity) ] @@ -1257,10 +1278,10 @@ async def _plan( "consumption-price": {"sensor": self._price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", } - # The band always goes out, and without this its minima and maxima are hard bounds, - # which makes states a storage really reaches infeasible. - # The breach price is left to FlexMeasures' own default, - # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. + # The band is always sent, and without this setting its minima and maxima are hard bounds. + # Hard bounds make states that a storage really reaches infeasible. + # The breach price is left to FlexMeasures' own default. + # The value of a comfort breach is for the deployment to judge, not for the protocol. if "soc-minima" in flex_model: flex_context["relax-soc-constraints"] = True if self._relax_site_capacity is not None: @@ -1273,8 +1294,8 @@ async def _plan( return self._warn_where_a_fallback_scheduler_answered(schedule, start) - # send_instruction_batch checks this once more, but translating a schedule follows every step of it, - # which is wasted on a request that is already out of date. + # send_instruction_batch checks this again, but translating a schedule goes through every step of it. + # That work is wasted on a request that is already out of date. if request_number is not None and not self._is_latest_request( STORAGE_STATUS_REQUESTS, request_number ): @@ -1297,10 +1318,10 @@ async def _plan( soc_min / self._fill_level_scale, soc_max / self._fill_level_scale, ), - # What the storage loses by itself and what is drawn from it, as the resource manager last said, - # so that the fill level each step is answered at falls as well as rises. + # Pass what the storage loses by itself and what is drawn from it, as the resource manager last reported. + # Then the fill level used for each step can fall as well as rise. leakage_behaviour=self._latest(self._leakage_behaviour_history), - # Without negative usage, as it was recorded and planned against (see send_usage_forecast). + # Leave out negative usage, as when the forecast was recorded and planned against (see send_usage_forecast). usage_forecast=_without_negative_usage( self._latest(self._usage_forecast_history) ), @@ -1361,8 +1382,8 @@ async def send_fill_level_target_profile( # that comes back begins there, not at the start_time the resource manager sent. Posting # it at the raw start_time would shift every value by the difference. start = self.align_to_resolution(fill_level_target_profile.start_time) - # Recorded to be seen, since the band a schedule is planned against goes out inline, - # so a failure is logged and the other bound still recorded, rather than raised in a task nobody awaits. + # These are recorded for inspection only, because the band that a schedule is planned against is sent inline. + # So a failure is logged, and the other bound is still recorded, instead of raising in a task that nobody awaits. for sensor_id, values in ( (self._soc_minima_sensor_id, soc_minima), (self._soc_maxima_sensor_id, soc_maxima), @@ -1397,10 +1418,10 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): strategy="mean", fill_level_scale=self._fill_level_scale, ) - # A negative usage is a gain the storage is forced to take, which FlexMeasures cannot relax: - # past the hard ceiling it makes every schedule for the window infeasible, so the storage gets no instruction at all. - # S2 allows the sign, but no storage in reach of this handler fills by being drawn from, - # so a negative value is taken as an artefact of the forecast, counted, and recorded as no usage. + # A negative usage is a gain that the storage is forced to take, and FlexMeasures cannot relax it. + # Above the hard ceiling, it makes every schedule for the window infeasible, so the storage gets no instruction at all. + # S2 allows the negative sign, but no storage that this handler controls fills up by being drawn from. + # So a negative value is treated as an artefact of the forecast: it is counted, and recorded as no usage. negative = translated < 0 if negative.any(): self._logger.warning( @@ -1424,10 +1445,12 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): ) async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): - """Keep a resource manager's leakage behaviour for planning, which records it when it asks for a schedule. + """Keep a resource manager's leakage behaviour for planning; it is recorded when a schedule is requested. - Recorded on arrival instead, it covered the hours after the message rather than the window being planned, - at a fill level nobody had reported yet, and planning went on without it once those hours had passed. + It used to be recorded on arrival. + Then it covered the hours after the message, instead of the window being planned. + It was also computed at a fill level that nobody had reported yet. + And once those hours had passed, planning went on without it. """ if self._leakage_behaviour_sensor_id is None: self._logger.info( @@ -1437,9 +1460,9 @@ async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): async def _record_storage_efficiency( self, start: datetime, fill_level: float ) -> bool: - """Record what the storage keeps of itself per step, over the window a schedule covers, from where it starts. + """Record the share of its stock that the storage keeps per step, over the window a schedule covers, from the schedule's start. - :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. + :returns: whether the schedule may be planned against the sensor. It may not, if nothing is recorded on the sensor. """ leakage_behaviour = self._latest(self._leakage_behaviour_history) if ( @@ -1448,17 +1471,19 @@ async def _record_storage_efficiency( or not self._system_description_history ): return False - # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate only at one fill level, and the loss scales with the stock away from it. - # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: - # from a near-empty start, a share of the start level would let almost nothing remain per step, - # and the loss would be overstated many times over as soon as the plan filled the storage. - # Below the middle the loss is understated instead, by at most the ratio of the middle to the stock, which is the milder error. + # S2 gives leakage as a rate, but FlexMeasures takes a share of the stock. + # So one share matches the rate at only one fill level; at other fill levels, the loss scales with the stock. + # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range. + # From a near-empty start, a share of the start level would let almost nothing remain per step. + # The loss would then be overstated many times over, as soon as the plan filled the storage. + # Below the middle, the loss is understated instead, by at most the ratio of the middle to the stock. + # That is the milder error. storage_range = list(self._system_description_history.values())[ -1 ].storage.fill_level_range middle = (storage_range.start_of_range + storage_range.end_of_range) / 2 - # What remains after one step of this handler's resolution, - # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. + # The share that remains after one step of this handler's resolution. + # FlexMeasures reads it as one step of the leakage sensor's resolution, so the two resolutions must be equal. efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, @@ -1476,7 +1501,7 @@ async def _record_storage_efficiency( values=[efficiency], unit="%", duration=self._schedule_duration, - # As for the charging efficiency: remembered below as recorded, so only once it is. + # As for the charging efficiency: the value is remembered below as recorded, so it must really be recorded. confirm_ingestion=True, ) except REQUEST_FAILURES: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 23babaf3..5d2b0a14 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -169,11 +169,11 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - # By this handler's clock, so that a co-simulation's timers run on simulated time. + # Use this handler's clock, so that a co-simulation's timers run on simulated time. now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + # A timer without a timezone used to be compared with the host's local time, so keep reading it as local time. due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period @@ -191,8 +191,8 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) - # On a period boundary the ceiling is the moment itself, - # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + # On a period boundary, the ceiling equals the current moment. + # A clock that stands still (as a simulated clock does between messages) would then find the timer due on every call. if next_due <= now: next_due += self._minimum_measurement_period self._timers[name] = next_due @@ -315,9 +315,9 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - # It takes neither argument the base class declares: TUNES plans from its latest description, - # and does not take part in dropping superseded requests. - # Accepting them and ignoring them would turn a caller's mistake into a silent one, so the mismatch stays. + # This override takes neither of the arguments that the base class declares. + # TUNES plans from its latest description, and does not take part in dropping superseded requests. + # Accepting the arguments and ignoring them would hide a caller's mistake, so the signatures stay different. async def trigger_schedule(self): # type: ignore[override] """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager @@ -485,8 +485,9 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) - # Posted per hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, - # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. + # send_usage_forecast posts the usage forecast per hour, scaled to FlexMeasures' units, so that is how it is read back. + # The schedule translation needs a usage rate per second, in the resource manager's own units, as in S2. + # So convert it back here. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" ) / ((pd.Timedelta(hours=1) / pd.Timedelta(seconds=1)) * self._fill_level_scale) @@ -512,7 +513,7 @@ async def trigger_schedule(self): # type: ignore[override] system_description, initial_fill_level=soc_at_start / self._fill_level_scale, logger=self._logger, - # What the storage loses by itself, so that the running fill level falls as well as rises. + # Pass what the storage loses by itself, so that the running fill level can fall as well as rise. leakage_behaviour=( list(self._leakage_behaviour_history.values())[-1] if self._leakage_behaviour_history @@ -693,9 +694,9 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): fill_level_scale=self._fill_level_scale, ) - # From per second, as S2 states a usage rate, to per hour, which is what a value labelled MW means: - # FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour. - # Posted per quarter hour instead, it was planned as a quarter of the real draw. + # Convert from per second (as S2 gives a usage rate) to per hour, which is what a value in MW means. + # FlexMeasures reads soc-usage as a power, and multiplies it by the step's fraction of an hour. + # When this was posted per quarter hour instead, the scheduler planned for a quarter of the real draw. scale = timedelta(hours=1) / timedelta(seconds=1) scaled_usage_forecast = usage_forecast_series * scale @@ -703,7 +704,7 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), - unit=POWER_UNIT, # a rate per hour, as MW is + unit=POWER_UNIT, # a rate per hour, like MW duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index aef2c27a..0b597962 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -37,16 +37,17 @@ def compute_next_fill_level( leakage_behaviour: FRBCLeakageBehaviour | None = None, usage_rate: float = 0.0, ) -> float: - """The fill level one step later. + """Return the fill level one step later. - This is the unit contract for every rate it takes, and it is the one S2 states: - each is a change in fill level per second, in the resource manager's own fill-level units. - A fill rate raises the fill level, while a usage rate and a leakage rate both lower it where they are positive. - The step is a duration, rather than a number of some unit, so that it cannot be taken in the wrong one. + This function sets the unit contract for every rate it takes, and that contract is the one S2 uses. + Each rate is a change in fill level per second, in the resource manager's own fill-level units. + A fill rate raises the fill level. + A positive usage rate or a positive leakage rate lowers it. + The step is a duration, not a number in some unit, so that it cannot be given in the wrong unit. - :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; - none where the resource manager has not said. - :param usage_rate: what is drawn from the storage, held over this step. + :param leakage_behaviour: what the storage loses by itself; the element that applies at this fill level is used. + None if the resource manager has not reported it. + :param usage_rate: what is drawn from the storage, constant over this step. """ leakage_rate = 0.0 if leakage_behaviour is not None: @@ -56,23 +57,25 @@ def compute_next_fill_level( return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() -#: How close two factors are to count as the same instruction. -#: It absorbs the float noise of converting a schedule from MW to W, which is not a new instruction. +#: How close two factors must be to count as the same instruction. +#: It absorbs the float noise from converting a schedule from MW to W, which should not lead to a new instruction. FACTOR_TOLERANCE = 1e-6 class UsageProfile: - """What a usage forecast expects, per second, at any moment: piecewise constant, looked up by bisection. + """The usage rate per second that a usage forecast expects at any moment. - A forecast is a run of elements, each lasting its own duration from where the one before it ended. - Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. - Built once per schedule, since a forecast may hold hundreds of elements and a schedule asks about it often. + The rate is piecewise constant, and is looked up by bisection. + A forecast is a sequence of elements. + Each element starts where the previous one ended, and lasts its own duration. + Outside that sequence, nothing is forecast, and nothing is assumed: the rate is zero. + Build this once per schedule, because a forecast may hold hundreds of elements, and a schedule looks up the rate often. """ def __init__( self, usage_forecast: FRBCUsageForecast | None = None, rate: float | None = None ): - #: The moments the rate changes at, and the rate from each of them on; one constant rate has none. + #: The moments at which the rate changes, and the rate from each moment on. A constant rate has no changes. self._changes: list[pd.Timestamp] = [] self._rates: list[float] = [] self._before = 0.0 if rate is None else rate @@ -98,14 +101,15 @@ def next_change_after(self, moment: pd.Timestamp) -> pd.Timestamp | None: def usage_rate_at( usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp ) -> float: - """The usage rate a forecast expects at a moment, per second, as S2 states it (see UsageProfile).""" + """Return the usage rate that a forecast expects at a moment, per second as in S2 (see UsageProfile).""" return UsageProfile(usage_forecast).rate_at(moment) def _sides(elements, fill_level: float): - """The element that applies just below a fill level, and the one just above it. + """Return the element that applies just below a fill level, and the element that applies just above it. - Inside an element both are that element; at a boundary two elements share, they are the two neighbours. + Inside an element, both are that element. + At a boundary between two elements, they are the two neighbouring elements. """ above = element_for_fill_level(elements, fill_level) below_candidates = [ @@ -133,18 +137,20 @@ def follow_operation_mode( usage: UsageProfile | None = None, bounds: tuple[float, float] | None = None, ) -> float: - """The fill level after following one operation mode at one factor for a step, as a resource manager does. - - An instruction fixes the mode and the factor, not the element: - within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, - and the usage a forecast expects can change. - Every rate is constant between such moments, so the step is followed exactly, from one of them to the next. - A level that reaches a boundary its far side would push it back from, as where a heat pump cannot heat above it, - stays at that boundary, as a real storage does, rather than flipping across it. - So does a level that reaches one of the bounds. - - :param usage: what is drawn from the storage, per second; none where nothing is known. - :param bounds: the fill levels the storage is held between. + """Return the fill level after following one operation mode at one factor for one step, as a resource manager does. + + An instruction fixes the mode and the factor, but not the element. + Within the step, the fill level can pass from one element into another, of the mode or of the leakage behaviour. + Also, the usage that a forecast expects can change. + Every rate is constant between such moments. + So the step is followed exactly, from one such moment to the next. + Sometimes a fill level reaches a boundary where the rate on the far side would push it back. + For example, a heat pump may be unable to heat above that boundary. + Then the fill level stays at that boundary, as in a real storage, instead of flipping back and forth across it. + The same holds for a fill level that reaches one of the bounds. + + :param usage: what is drawn from the storage, per second; None if nothing is known. + :param bounds: the fill levels between which the storage is held. """ usage = usage or UsageProfile() leakage_elements = leakage_behaviour.elements if leakage_behaviour else [] @@ -183,7 +189,7 @@ def net_rate(mode_element, leakage_element, moment) -> float: elif falling < 0 and fill_level > lowest: rate = falling else: - rate = 0.0 # held where it is, at a boundary or a bound, until something changes + rate = 0.0 # stay at the boundary or bound until something changes horizon = end change = usage.next_change_after(moment) if change is not None and change < horizon: @@ -194,7 +200,7 @@ def net_rate(mode_element, leakage_element, moment) -> float: elif rate < 0: target = next((b for b in reversed(boundaries) if b < fill_level), None) if target is not None: - # Compared in seconds first: a slow rate can put a boundary further off than a timedelta can say. + # Compare in seconds first: at a slow rate, a boundary can be further away than a timedelta can represent. seconds_to_target = (target - fill_level) / rate if seconds_to_target < (horizon - moment).total_seconds(): fill_level = target @@ -206,11 +212,11 @@ def net_rate(mode_element, leakage_element, moment) -> float: def normal_operation_modes(actuator) -> list[FRBCOperationMode]: - """The operation modes an actuator may be instructed into under normal conditions. + """Return the operation modes an actuator may be instructed into under normal conditions. - S2 marks a mode abnormal_condition_only where it is reserved for an abnormal condition, such as an emergency boost, - and a resource manager refuses an instruction into one that is not marked abnormal itself, - which every instruction a schedule is turned into is not. + S2 marks a mode as abnormal_condition_only if it is reserved for an abnormal condition, such as an emergency boost. + A resource manager refuses an instruction into such a mode, unless the instruction itself is marked abnormal. + The instructions made from a schedule are never marked abnormal. So such a mode is neither planned in nor instructed into. """ return [ @@ -229,31 +235,33 @@ def fm_schedule_to_instructions( ) -> List[FRBCInstruction]: """Turn a FlexMeasures schedule into the instructions that follow it. - Each step is answered with the operation mode, and the factor within it, that draws the power the schedule asks for - at the fill level the storage is then at, so a running fill level is kept alongside, under the contract of compute_next_fill_level. - An instruction is sent where that answer changes, which can happen at a constant power too, - where the fill level passes from one element of a mode into another. + For each step, this picks the operation mode, and the factor within it, that draw the power the schedule asks for. + That choice depends on the fill level the storage is at during the step. + So a running fill level is tracked alongside, under the unit contract of compute_next_fill_level. + An instruction is sent whenever the chosen mode or factor changes. + That can happen even at constant power, when the fill level passes from one element of a mode into another. :param schedule: a schedule as FlexMeasures returns it, or a frame indexed by time with a "schedule" column. - A frame may also carry a "usage_forecast" column, per second as S2 states it, - which then takes the place of usage_forecast. - :param fill_level_bounds: the fill levels this conversion holds its own running fill level between, + A frame may also have a "usage_forecast" column, per second as in S2. + If it does, that column is used instead of usage_forecast. + :param fill_level_bounds: the fill levels between which this conversion holds its own running fill level, in the resource manager's units. - These have to be the hard bounds the schedule was planned against, - or a step is answered with the operation mode for a fill level the scheduler never planned. - Without them the declared fill-level range is used, - which is right only for a caller that asked for a schedule against that same range. - :param leakage_behaviour: what the storage loses by itself, as the resource manager last said. + These must be the hard bounds that the schedule was planned against. + Otherwise, a step may get the operation mode for a fill level that the scheduler never planned. + Without them, the declared fill-level range is used. + That is only right for a caller that asked for a schedule against that same range. + :param leakage_behaviour: what the storage loses by itself, as the resource manager last reported it. :param usage_forecast: what is drawn from the storage, as the resource manager last forecast it. - The step is the schedule's own: a schedule from FlexMeasures states it as its duration over its number of values, - which is the resolution of the sensor it was asked for, whatever resolution the caller plans on. + The step comes from the schedule itself. + A schedule from FlexMeasures gives a duration and a number of values, and the step is the duration divided by that number. + That step is the resolution of the sensor the schedule was requested for, whatever resolution the caller plans on. """ if fill_level_bounds is not None and fill_level_bounds[0] > fill_level_bounds[1]: - # A mistake in the caller, which computes these, rather than anything a resource manager sent: - # held between a lower bound above its upper one, the running fill level would sit at the lower one, - # and every step would be answered with the operation mode for a fill level the storage is never at. + # This is a mistake by the caller, which computes these bounds, not something a resource manager sent. + # With a lower bound above the upper bound, the running fill level would stay at the lower bound. + # Then every step would get the operation mode for a fill level the storage is never at. raise ValueError( f"fill_level_bounds runs from its lower bound to its upper one, but {fill_level_bounds} was given." ) @@ -270,7 +278,7 @@ def fm_schedule_to_instructions( idx = pd.date_range(start=start, periods=len(values), freq=step) schedule = pd.Series(values, index=idx, name="schedule").to_frame() else: - # A frame of one row has no later step for its fill level to matter to. + # A frame with one row has no later step, so the fill level after it does not matter, and a zero step will do. step = ( pd.Timedelta(schedule.index[1] - schedule.index[0]) if len(schedule) > 1 @@ -290,9 +298,9 @@ def fm_schedule_to_instructions( ) modes_by_id = {str(mode.id): mode for mode in operation_modes} - # Held between the hard bounds the schedule was planned against, - # so that each step is answered with the operation mode for the fill level the scheduler planned. - # Falling back to the declared range keeps every other caller as it was. + # Hold the running fill level between the hard bounds that the schedule was planned against. + # Then each step gets the operation mode for the fill level that the scheduler planned. + # Falling back to the declared range keeps the behaviour unchanged for every other caller. if fill_level_bounds is None: fill_level_range: NumberRange = system_description.storage.fill_level_range fill_level_min = fill_level_range.start_of_range @@ -309,8 +317,8 @@ def fm_schedule_to_instructions( for timestamp, row in schedule.iterrows(): power = row["schedule"] - # The mode and factor for this power at this fill level, every step, - # since the element that applies can change while the power does not. + # Pick the mode and factor for this power at this fill level in every step, + # because the element that applies can change while the power stays the same. results = [ (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) for om in operation_modes @@ -353,10 +361,10 @@ def fm_schedule_to_instructions( explain_choice(results, best), ) - # The factor is the one that draws the power asked for, taken from the power range itself. - # It used to be worked back out of the fill rate instead, - # which fails wherever the fill-rate range is a single value and the power range is not: - # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. + # Take the factor that draws the requested power from the power range itself. + # It used to be derived from the fill rate instead. + # That fails wherever the fill-rate range is a single value but the power range is not. + # For example, given a request for 1200 W from a mode that runs at 800 to 1200 W with a single fill rate, the old code instructed 800 W. answer = (str(best_operation_mode.id), operation_mode_factor) if last_instructed is None or not ( answer[0] == last_instructed[0] @@ -377,8 +385,9 @@ def fm_schedule_to_instructions( instructions.append(instruction) last_instructed = answer - # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: - # a NaN fill level would be held at the top of the bounds, and answered there from then on. + # FlexMeasures returns a gap in a usage column for a moment it holds no data for. + # Treat such a gap as no known usage, not as NaN. + # A NaN fill level would be held at the top of the bounds, and every later step would be chosen from there. usage = forecast_usage if has_usage_column: usage = UsageProfile( @@ -400,7 +409,7 @@ def fm_schedule_to_instructions( ) if logger.isEnabledFor(logging.DEBUG): - # The whole batch in one line, by mode name, which the per-step lines above do not give. + # Log the whole batch in one line, by mode name, which the per-step log lines above do not show. logger.debug( "Instructions: %s", [ @@ -429,9 +438,9 @@ def get_soc_min_max( return soc_min, soc_max -#: What one unit of energy per second is in watts, for the units a fill level can be recorded in. -#: A joule per second is a watt by definition, -#: and an hour's worth of a watt is 3600 times as much energy as a watt-second. +#: The number of watts in one unit of energy per second, for each unit a fill level can be recorded in. +#: A joule per second is a watt, by definition. +#: A watt-hour is 3600 times as much energy as a watt-second. WATTS_PER_ENERGY_UNIT_PER_SECOND = { "J": 1.0, "kJ": 1e3, @@ -443,15 +452,15 @@ def get_soc_min_max( def fill_rate_in_watts(fill_rate: float, energy_unit: str) -> float: - """A fill rate, stated per second in the unit a fill level is recorded in, as watts. + """Convert a fill rate to watts, given the unit that the fill level is recorded in. - S2 states a fill rate as fill level per second, so what it is in watts depends on the unit the fill level itself is in: - a kilowatt hour per second is 3.6 million watts. - The conversion is needed wherever a fill rate is compared against a power value, - which S2 always states in watts. + S2 gives a fill rate as fill level per second. + So its value in watts depends on the unit of the fill level itself. + For example, one kilowatt-hour per second is 3.6 million watts. + This conversion is needed wherever a fill rate is compared with a power value, which S2 always gives in watts. - :raises NotImplementedError: for an energy unit this does not know, - rather than returning a number that is wrong by a factor of thousands. + :raises NotImplementedError: for an unknown energy unit, + instead of returning a number that is wrong by a factor of thousands. """ try: return fill_rate * WATTS_PER_ENERGY_UNIT_PER_SECOND[energy_unit] @@ -468,41 +477,42 @@ def operation_mode_conversion_efficiency( energy_unit: str, fill_level: float, ) -> float | None: - """How much energy an operation mode puts into the storage for each unit of power it draws, at a fill level. - - An operation mode states a fill rate and a power range, - so the ratio between the two is what the mode converts drawn power into stored energy at: - a heat pump's coefficient of performance, or a battery's charging efficiency. - Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, - and for a thermal actuator that ratio is well above one. - - A mode may convert differently at different fill levels, as a heat pump does when its buffer warms, - and FlexMeasures takes one charging efficiency for a whole schedule. - The element that applies at the fill level the schedule starts from answers, - since the start of a schedule is the part that runs before the next storage status asks for a new one. - Not the lowest element over the whole range: - that overstates what the actuator needs to fill the storage, so the plan overshoots its upper band and buys more than it uses, - and against a usage the plan cannot relax it can turn a problem the actuator can meet into one the scheduler finds infeasible. - - The ratio is taken at the top of that element's range, rather than as the slope over it. - An actuator may well declare degenerate ranges, - where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), - and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. + """Return how much energy an operation mode puts into the storage per unit of power it draws, at a fill level. + + An operation mode gives a fill rate and a power range. + The ratio between the two is the rate at which the mode converts drawn power into stored energy. + Examples are a heat pump's coefficient of performance and a battery's charging efficiency. + Both sides are taken in watts, because FlexMeasures expects a charging efficiency as a dimensionless ratio. + For a thermal actuator, that ratio is well above one. + + A mode may convert differently at different fill levels, as a heat pump does when its buffer warms up. + But FlexMeasures takes a single charging efficiency for a whole schedule. + So this uses the element that applies at the fill level where the schedule starts. + The start of a schedule is the part that runs before the next storage status triggers a new schedule. + The lowest element over the whole range would be the wrong choice. + It overstates what the actuator needs to fill the storage. + The plan then overshoots its upper band, and buys more than it uses. + And if the plan cannot relax the usage, the scheduler may find a problem infeasible that the actuator can actually meet. + + The ratio is taken at the top of that element's range, not as the slope over the range. + An actuator may well declare degenerate ranges, where a mode runs at only one power and one fill rate. + For example, the heaters in our own smoke tests declare their on mode as [883.7, 883.7] W. + Over such a range the slope is zero, while the ratio is the efficiency the mode really runs at. :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. - :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :param fill_level: where the schedule starts, in the resource manager's own units, as the elements' ranges are. - :returns: the efficiency, or None where the element that applies converts no drawn power into stored energy. - It converts nothing where it draws no power, as an idle mode does, - where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, - or where it draws power while the storage does not fill, as a standby or circulation mode does: - no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. + :param energy_unit: the unit a fill level is recorded in; the fill rate is in this unit per second. + :param fill_level: where the schedule starts, in the resource manager's own units, like the elements' ranges. + :returns: the efficiency, or None if the element that applies converts no drawn power into stored energy. + That is the case in three situations. + The element draws no power, as in an idle mode. + The element only feeds power back, which FlexMeasures handles with a separate discharging efficiency. + Or the element draws power while the storage does not fill, as in a standby or circulation mode. + No single ratio can express that, and a ratio of zero would tell the scheduler that the storage cannot be filled at all. """ element = element_for_fill_level(operation_mode.elements, fill_level) - # todo: an element states one power range per commodity quantity, - # and no distinction is made between them here, - # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). + # todo: an element gives one power range per commodity quantity, and this makes no distinction between them. + # Neither does the rest of this handler (see power_to_fill_rate_with_metrics). power = element.power_ranges[0].end_of_range fill_rate = element.fill_rate.end_of_range * fill_level_scale if power <= 0 or fill_rate <= 0: @@ -562,7 +572,7 @@ def power_to_fill_rate_with_metrics( power_penalty: float efficiency: float element: FRBCOperationModeElement that produced fill_rate - factor: the operation mode factor that draws this power, which is what an instruction asks for + factor: the operation mode factor that draws this power; an instruction asks for this factor """ # --- 1. Select the element that applies at this fill level --- best_element = element_for_fill_level(operation_mode.elements, fill_level) @@ -586,9 +596,10 @@ def power_to_fill_rate_with_metrics( pr_end = power_range.end_of_range if pr_end == pr_start: - # Every factor draws this one power, so the power cannot say which factor is meant. - # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. - # Where the fill-rate range is a single value too, the factor makes no difference and stays at zero. + # Every factor draws this same power, so the power cannot tell which factor is meant. + # Ask for the top of the fill-rate range. + # That is where operation_mode_conversion_efficiency declares the efficiency that FlexMeasures plans with, so the plan and the instruction agree. + # If the fill-rate range is a single value too, the factor makes no difference, and it stays at zero. fr_start = best_element.fill_rate.start_of_range fr_end = best_element.fill_rate.end_of_range factor = 0.0 if fr_end == fr_start else 1.0 @@ -624,23 +635,24 @@ def clamp_distance(value: float, start: float, end: float) -> float: def element_for_fill_level( elements: Sequence[ElementWithFillLevelRange], fill_level: float ) -> ElementWithFillLevelRange: - """The element that applies at a fill level, whatever order the elements were listed in. + """Return the element that applies at a fill level, whatever order the elements are listed in. S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, in any order. - s2python checks that an operation mode's elements are contiguous, on a sorted copy, and leaves the list as it was sent; - it checks nothing of the kind for a leakage behaviour, whose elements may leave gaps or overlap. + s2python checks that an operation mode's elements are contiguous, using a sorted copy, and leaves the list in the order it was sent. + It does not check this for a leakage behaviour, whose elements may leave gaps or overlap. - Two neighbouring elements share a boundary, which belongs to the upper of the two: - of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. - A range that is a single point therefore applies only at the very top. - The standard says a range starts below where it ends, so such a range is outside it, though s2python lets it through. - Two elements with the same range are ambiguous, which overlapping elements always are, and the first listed answers. + Two neighbouring elements share a boundary, which belongs to the upper of the two. + Of all elements whose range holds the fill level, the one that starts highest applies. + If two of them start at the same level, the one that reaches higher applies. + So a range that is a single point only applies at the very top. + The standard says a range starts below where it ends, so such a range breaks the standard, but s2python lets it through. + Two elements with the same range are ambiguous (as overlapping elements always are), and then the first one listed applies. - A fill level outside the range the elements cover is answered as the nearest end of that range is, - since a storage may be outside its declared range, and the answer should not jump as it crosses that end. - A fill level in a gap between elements is answered by the nearer, and on a tie the lower. + A fill level outside the range the elements cover is treated as the nearest end of that range. + That is because a storage may be outside its declared range, and the result should not jump when the fill level crosses that end. + For a fill level in a gap between elements, the nearer element applies; on a tie, the lower one applies. - :raises ValueError: for a fill level that is not a number, which no element applies at. + :raises ValueError: for a fill level that is not a number, since no element applies there. """ if math.isnan(fill_level): raise ValueError("No element applies at a fill level that is not a number.") diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index c708c2b2..f2504ffa 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -44,18 +44,20 @@ async def send_message(self, message: BaseModel) -> None: await self._sending_queue.put(message) async def close(self): - """Stop this handler's work in flight, when the CEM closes as its websocket tears down. + """Stop this handler's work in flight. - Defined here so that CEM.close(), which calls close() on every registered handler, works for any of them: - a handler lacking close() raised AttributeError inside CEM.close(), - killing the CEM's request handler and hanging the resource manager. + The CEM calls this when it closes, which happens when its websocket tears down. + CEM.close() calls close() on every registered handler, so every handler needs this method. + It is defined here for that reason. + Before, a handler without close() raised AttributeError inside CEM.close(). + That killed the CEM's request handler and left the resource manager hanging. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") - # Work still in flight would otherwise go on after the session has ended, - # planning for a resource manager that is no longer there and racing the session that replaces it. - # That includes recording what the resource manager last reported, which is stopped too: - # a reading cut off mid-post is reported again by a resource manager that reconnects. - # Not every handler keeps background tasks, so a handler without them has nothing to stop. + # Otherwise, work in flight would continue after the session has ended. + # It would plan for a resource manager that is gone, and race the session that replaces it. + # Recording the resource manager's last reports is stopped too. + # That is safe: a resource manager that reconnects reports a reading again if its post was cut off. + # Not every handler keeps background tasks; a handler without them has nothing to stop. for task in list(getattr(self, "background_tasks", ())): task.cancel() diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index e61744db..54bc0158 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -58,10 +58,10 @@ def leakage_behaviour_to_storage_efficiency( """ - # FlexMeasures takes one storage efficiency here, for the whole window it covers. - # Where the fill level is known, the element that applies at it answers, as for a charging efficiency. - # Where it is not, the leakiest answers, looked for rather than assumed to come last, - # since S2 does not fix the order the elements are listed in, and the leakiest is not always the top one. + # FlexMeasures takes a single storage efficiency here, for the whole window it covers. + # If the fill level is known, use the element that applies at that fill level, as for a charging efficiency. + # If not, use the leakiest element. + # Search for it instead of assuming it comes last: S2 does not fix the order of the elements, and the leakiest one is not always the top one. if fill_level is not None: element = element_for_fill_level(message.elements, fill_level) else: @@ -70,13 +70,15 @@ def leakage_behaviour_to_storage_efficiency( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) leakage = element.leakage_rate * fill_level_scale - # S2 states a leakage rate per second, so what remains after one second is compounded over the seconds in a step. - # This used to compound it over the hours in a step instead, - # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, - # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. - # FlexMeasures applies a storage efficiency to the stock as it stands, so the loss is a share of a fill level: - # of share_of where given, as the fill level a schedule starts from is, - # and otherwise of the top of the range, which understates it at lower fill levels. + # S2 gives a leakage rate per second. + # So the share that remains after one second is compounded over the number of seconds in a step. + # This used to be compounded over the number of hours in a step instead. + # Take a buffer that loses 1.5 kWh of 10 kWh a day: over twelve hours, it loses about 7%. + # The old calculation declared that it lost only 0.002% over those twelve hours. + # Because of that, the scheduler pre-charged the buffer in a cheap hour. + # FlexMeasures applies a storage efficiency to the current stock, so the loss is a share of a fill level. + # That fill level is share_of, if given, such as the fill level a schedule starts from. + # Otherwise, it is the top of the range, which understates the loss at lower fill levels. reference = ( share_of * fill_level_scale if share_of is not None and share_of > 0 diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py index 5f154514..3f173f36 100644 --- a/src/flexmeasures_client/s2/measurements.py +++ b/src/flexmeasures_client/s2/measurements.py @@ -1,4 +1,7 @@ -"""Recording what a resource manager measures on FlexMeasures sensors, one value per step of each sensor's resolution.""" +"""Record what a resource manager measures on FlexMeasures sensors. + +Each sensor gets one value per step of its resolution. +""" from __future__ import annotations @@ -19,17 +22,17 @@ _LOGGER = getLogger(__name__) -#: The unit S2 states every power in, measured ones included. +#: The unit of every power value in S2, including measured power. S2_POWER_UNIT = "W" -#: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. +#: How long a recorded step is remembered. A step that a resource manager sends again within this time is not recorded twice. REMEMBERED_FOR = timedelta(days=7) -#: How often a step is tried before it is given up on, and how long to wait between tries. +#: How many times to try recording a step before giving up, and how long to wait between tries. ATTEMPTS = 3 RETRY_AFTER = timedelta(seconds=30) -#: What a post can fail with, short of a defect in this code. +#: The exceptions a post can raise when there is no defect in this code. POST_FAILURES = ( ValueError, ConnectionError, @@ -41,24 +44,30 @@ class MeasurementRecorder: - """Records measured power on a sensor as the average of what was measured in each step of its resolution. - - A step is recorded once it is over by the CEM's clock, as it stands when the step's last measurement arrives, - with the moment it ended as the moment it became known, - so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. - Steps are taken by the time each measurement is stamped with, not by when it arrives, - since a co-simulation's resource manager sends a window's measurements together once the window has passed, - each stamped at the start of its quarter hour, while a live one sends them as they are taken, - in which case a step is recorded with the first measurement after it is over. - - Recording happens in the background, so that a resource manager is not kept waiting for FlexMeasures - before its measurement is acknowledged, one post at a time per sensor. - A step that could not be recorded is tried again a while later, and given up on after a few tries. - - Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, - which a co-simulation makes every round, and so remembers what was recorded: - a step sent again is not recorded again, and the first value recorded for a step stands. - A sensor is meant to be one resource manager's: one that several share has their measurements averaged together. + """Record measured power on a sensor, as the average of the measurements in each step of its resolution. + + A step is recorded once it is over, according to the CEM's clock when the step's last measurement arrives. + The recorded belief time is the moment the step ended. + So a measurement is recorded as a belief formed at the end of the period it measures, not as a forecast of it. + + Each measurement is assigned to a step by its timestamp, not by its arrival time. + This matters because resource managers send measurements in different ways. + A co-simulation's resource manager sends a window's measurements together, once the window has passed. + It stamps each one at the start of its quarter hour. + A live resource manager sends each measurement when it is taken. + In that case, a step is recorded when the first measurement after the step arrives. + + Recording happens in the background, so the resource manager does not wait for FlexMeasures before its measurement is acknowledged. + Per sensor, only one post runs at a time. + If a step could not be recorded, it is tried again a while later. + After a few failed tries, it is given up on. + + The server creates one recorder and hands it to each CEM. + So the recorder outlives the reconnections of a resource manager, which a co-simulation makes every round. + The recorder therefore remembers what it recorded. + A step that is sent again is not recorded again, and the first value recorded for a step stands. + Each sensor is meant to belong to one resource manager. + If several resource managers share a sensor, their measurements are averaged together. """ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): @@ -67,16 +76,17 @@ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): self._resolutions: dict[int, timedelta] = {} self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) self._attempts: dict[int, dict[datetime, int]] = defaultdict(dict) - #: By sensor, the steps recorded, in the order they were, which is roughly that of time. + #: The recorded steps per sensor, in recording order, which is roughly chronological. self._recorded: dict[int, dict[datetime, None]] = defaultdict(dict) self._locks: dict[int, asyncio.Lock] = defaultdict(asyncio.Lock) self._tasks: set[asyncio.Task] = set() async def resolution(self, sensor_id: int) -> timedelta: - """The sensor's resolution, asked once, since it is a property of how the sensor was set up. + """Return the sensor's resolution. - A caller taking several measurements at once asks this for each sensor first, - so that none is taken where one of them cannot be. + The resolution is fetched from FlexMeasures only once, because it is fixed when the sensor is set up. + A caller that takes several measurements at once should first call this for each sensor. + Then, if any one sensor fails, no measurement is taken at all. """ if sensor_id not in self._resolutions: sensor = await self._fm_client.get_sensor(sensor_id) @@ -93,7 +103,7 @@ async def resolution(self, sensor_id: int) -> timedelta: async def add( self, sensor_id: int, measured_at: datetime, watts: float, now: datetime ) -> None: - """Take one measurement, and record in the background every step of this sensor that is over by now.""" + """Take one measurement, then record every step of this sensor that is over by now, in the background.""" resolution = await self.resolution(sensor_id) step = step_of(measured_at, resolution) if step in self._recorded[sensor_id]: @@ -105,10 +115,11 @@ async def add( self._in_background(self.record(sensor_id, now)) async def record(self, sensor_id: int, now: datetime) -> None: - """Record every pending step of this sensor that is over by now, each in a request of its own. + """Record every pending step of this sensor that is over by now, each in its own request. - Each with its own end as the moment it became known, which one request for several steps cannot say. - One at a time per sensor, so a step is not posted twice, nor taken from under a post in flight. + Each step needs its own end as its belief time, and a single request for several steps cannot express that. + Per sensor, only one call runs at a time. + This way, a step is never posted twice, and no step is removed while a post for it is in flight. """ async with self._locks[sensor_id]: resolution = await self.resolution(sensor_id) @@ -159,7 +170,7 @@ async def _retry(self, sensor_id: int, now: datetime) -> None: await self.record(sensor_id, now) async def settle(self) -> None: - """Wait for every recording in the background to finish, retries included.""" + """Wait for every background recording to finish, including retries.""" while self._tasks: await asyncio.gather(*list(self._tasks), return_exceptions=True) @@ -176,7 +187,10 @@ def _done(self, task: asyncio.Task) -> None: ) def _forget_before(self, sensor_id: int, cutoff: datetime) -> None: - """Forget recorded steps from before the cutoff, oldest first, so that this costs little when nothing is old.""" + """Forget recorded steps from before the cutoff. + + Steps are checked oldest first, so this costs little when no step is old. + """ recorded = self._recorded[sensor_id] while recorded: oldest = next(iter(recorded)) @@ -186,10 +200,12 @@ def _forget_before(self, sensor_id: int, cutoff: datetime) -> None: def step_of(moment: datetime, resolution: timedelta) -> datetime: - """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it. + """Return the start of the step that a moment falls in. - In the repeated hour of an autumn DST change a boundary exists twice, and the step is the later of the two - that is not after the moment; taken from the moment's pass alone, flooring cannot always be done at all. + The step start is in the moment's own timezone, floored the way FlexMeasures floors it. + During the repeated hour of an autumn DST change, a step boundary occurs twice. + The step then starts at the later of the two boundaries that is not after the moment. + Flooring based only on which of the two passes the moment is in does not always work. """ if not resolution: return moment diff --git a/src/flexmeasures_client/s2/script/__init__.py b/src/flexmeasures_client/s2/script/__init__.py index c777fdcf..35f6224c 100644 --- a/src/flexmeasures_client/s2/script/__init__.py +++ b/src/flexmeasures_client/s2/script/__init__.py @@ -1 +1,4 @@ -"""Example scripts: a CEM server and a resource manager that talk S2 to each other, run by path as the CEM documentation shows.""" +"""Example scripts: a CEM server and a resource manager that talk S2 to each other. + +Run them by path, as the CEM documentation shows. +""" diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index b73ce4c9..6edbbf69 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -1,12 +1,12 @@ -"""An example resource manager: a heat pump keeping a hot-water buffer warm, speaking S2's FRBC to a CEM. +"""An example resource manager: a heat pump that keeps a hot-water buffer warm, and talks S2's FRBC to a CEM. Run it by path, next to the example CEM server, as the CEM documentation shows. -It declares what the heat pump can do and what its buffer needs, then simulates that buffer: -it follows each instruction from the moment it is due, and reports its fill level every so often, -which is what asks the CEM for a new schedule. +It declares what the heat pump can do and what its buffer needs, and then simulates that buffer. +The simulation follows each instruction from the moment it is due. +Every so often, it reports its fill level, which triggers the CEM to plan a new schedule. -The buffer is simulated from S2's own definitions, and not with the CEM's code, -so that it is a second opinion on where the instructions take the storage rather than a copy of the first. +The buffer is simulated from S2's own definitions, not with the CEM's code. +That way, the simulation is an independent check on where the instructions take the storage, not a copy of the CEM's own calculation. """ from __future__ import annotations @@ -58,20 +58,20 @@ #: The range the buffer's fill level should stay in, in kilowatt hours of stored heat. FILL_LEVEL_RANGE = (2.0, 10.0) -#: Below this fill level the water is cold enough for the heat pump to run at a COP of 4; above it, 3. +#: Below this fill level, the water is cold enough for the heat pump to run at a COP of 4. Above it, the COP is 3. COLD_BELOW = 6.0 -#: What the buffer holds when the simulation starts, and what it can physically hold at most. +#: The buffer's fill level when the simulation starts, and the most it can physically hold. STARTING_FILL_LEVEL = 5.0 PHYSICAL_CAPACITY = 12.0 #: How often the simulated buffer reports its fill level, in seconds of wall-clock time. STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) -#: How often the forecasts are sent again, each covering the day from then on, so that planning never runs out of them. +#: How often the forecasts are sent again, each covering the next day, so that planning never runs out of them. FORECAST_INTERVAL = float(os.getenv("S2_EXAMPLE_FORECAST_INTERVAL", "3600")) ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC def heat(watts: float) -> float: - """A rate of heat in watts as a fill rate in kilowatt hours per second, which is how S2 states a fill rate.""" + """Convert a heat rate in watts into a fill rate in kilowatt-hours per second, which is how S2 expresses a fill rate.""" return watts / 3.6e6 @@ -101,7 +101,7 @@ def _element( def operation_modes() -> list[FRBCOperationMode]: - """Idle, or heating at anything from 400 to 1500 W, at a COP of 4 while the buffer is cold and 3 once it is warm.""" + """Return two modes: idle, and heating at 400 to 1500 W, with a COP of 4 while the buffer is cold and 3 once it is warm.""" idle = FRBCOperationMode( id=get_unique_id(), diagnostic_label="idle", @@ -123,8 +123,8 @@ def operation_modes() -> list[FRBCOperationMode]: def system_description( modes: list[FRBCOperationMode], valid_from: datetime ) -> FRBCSystemDescription: - # S2 lists the transitions an actuator can make, so the heat pump says it can go from either mode to the other, - # at once and at no cost. + # S2 lists the transitions an actuator can make. + # This heat pump declares that it can switch from either mode to the other, immediately and at no cost. transitions = [ Transition( id=get_unique_id(), @@ -165,7 +165,7 @@ def system_description( def leakage_behaviour(valid_from: datetime) -> FRBCLeakageBehaviour: - """A warmer buffer loses more: 0.8 kWh a day while cold, 1.5 kWh a day once warm.""" + """Return the leakage behaviour: a warmer buffer loses more, 0.8 kWh a day while cold, and 1.5 kWh a day once warm.""" return FRBCLeakageBehaviour( message_id=get_unique_id(), valid_from=valid_from, @@ -185,12 +185,12 @@ def leakage_behaviour(valid_from: datetime) -> FRBCLeakageBehaviour: def expected_usage(hour: int) -> float: - """Hot water drawn in a given hour of the day, in kWh: showers in the morning, washing up in the evening.""" + """Return the hot water drawn in a given hour of the day, in kWh: showers in the morning, washing up in the evening.""" return {7: 3.0, 19: 1.0, 20: 1.0}.get(hour, 0.1) def usage_forecast(start: datetime) -> FRBCUsageForecast: - """A day of hot-water use, hour by hour, from the hour start falls in.""" + """Return a forecast for a day of hot-water use, hour by hour, from the start of the hour that start falls in.""" start = start.replace(minute=0, second=0, microsecond=0) return FRBCUsageForecast( message_id=get_unique_id(), @@ -206,7 +206,7 @@ def usage_forecast(start: datetime) -> FRBCUsageForecast: def fill_level_target_profile(start: datetime) -> FRBCFillLevelTargetProfile: - """Kept within the declared range, and at least 6 kWh in the hour before the morning showers.""" + """Return a target profile within the declared range: at least 3 kWh, and at least 6 kWh in the hour before the morning showers.""" start = start.replace(minute=0, second=0, microsecond=0) return FRBCFillLevelTargetProfile( message_id=get_unique_id(), @@ -226,12 +226,12 @@ def fill_level_target_profile(start: datetime) -> FRBCFillLevelTargetProfile: @dataclass class Buffer: - """The simulated buffer: where its fill level goes, following S2's definitions. + """The simulated buffer, whose fill level changes according to S2's definitions. - S2 states every rate per second, in the fill level's own units: - a fill rate raises the fill level, and usage and leakage lower it. - An operation mode's fill rate is interpolated between the ends of its element's range by the factor, - from the element whose fill-level range holds the fill level. + S2 gives every rate per second, in the fill level's own units. + A fill rate raises the fill level, and usage and leakage lower it. + An operation mode's fill rate comes from the element whose fill-level range holds the fill level. + It is interpolated between the ends of that element's fill-rate range, using the factor. """ modes: dict[str, FRBCOperationMode] @@ -243,7 +243,11 @@ class Buffer: @staticmethod def _holding(elements, fill_level: float): - """The element whose range holds the fill level, the upper one at a shared boundary, the nearest outside.""" + """Return the element whose range holds the fill level. + + At a shared boundary, the upper element applies. + Outside all ranges, the nearest element applies. + """ lowest = min(e.fill_level_range.start_of_range for e in elements) highest = max(e.fill_level_range.end_of_range for e in elements) level = min(max(fill_level, lowest), highest) @@ -262,7 +266,7 @@ def _holding(elements, fill_level: float): ) def rates(self, moment: datetime) -> tuple[float, float, float]: - """Fill, usage and leakage, each per second, at this fill level and moment.""" + """Return the fill, usage and leakage rates, each per second, at this fill level and moment.""" fill = 0.0 if self.mode_id is not None: element = self._holding(self.modes[self.mode_id].elements, self.fill_level) @@ -273,7 +277,7 @@ def rates(self, moment: datetime) -> tuple[float, float, float]: return fill, usage, leakage def advance(self, until: datetime, step: timedelta = timedelta(minutes=1)) -> None: - """Move the fill level forward to a moment, a minute at a time, so that an element boundary is crossed in time.""" + """Move the fill level forward to a moment, a minute at a time, so that element boundaries are crossed at the right time.""" while self.at < until: dt = min(step, until - self.at) fill, usage, leakage = self.rates(self.at) @@ -290,7 +294,10 @@ def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: async def send_instruction_status( ws, instruction_id, status: InstructionStatus ) -> None: - """Tell the CEM where an instruction stands, which is what it revokes by: only one not yet carried out.""" + """Send the CEM the status of an instruction. + + The CEM uses these statuses to decide what to revoke: it only revokes instructions that are not yet carried out. + """ await ws.send_str( InstructionStatusUpdate( message_id=get_unique_id(), @@ -310,14 +317,14 @@ async def apply_at_execution_time( ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. - An instruction carries an execution_time, - which is when the resource manager is to be in that mode, not when the instruction happens to arrive. - A resource manager that answers on arrival reports a mode it is not in yet, - and a customer energy manager recording that status has a measurement of a power the device is not drawing, + An instruction has an execution_time. + That is when the resource manager should be in that mode, not when the instruction happens to arrive. + A resource manager that acts on arrival reports a mode it is not in yet. + A customer energy manager that records that status then has a measurement of a power the device is not drawing, timestamped now. - The mode it reports leaving is the one the last instruction carried out put it in, kept in applied, - rather than whatever the instruction before it asked for, which may have been revoked and never carried out. + The mode it reports leaving is the mode the last executed instruction put it in, which is kept in applied. + It is not the mode the previous instruction asked for, because that instruction may have been revoked and never carried out. """ delay = (instruction.execution_time - datetime.now(timezone.utc)).total_seconds() @@ -325,7 +332,7 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) - # From here on it is being carried out, so a revocation arriving now is too late to stop it. + # From here on, the instruction is being carried out, so a revocation that arrives now is too late to stop it. if on_due is not None: on_due() now = datetime.now(timezone.utc) @@ -351,8 +358,9 @@ async def apply_at_execution_time( async def answer_instructions(ws, buffer: Buffer | None = None) -> None: """Acknowledge every instruction, and report the transition each one asks for when it is due. - An instruction the CEM revokes, because a newer plan replaces the one it belonged to, is not followed: - without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. + The CEM revokes an instruction when a newer plan replaces the plan it belonged to. + A revoked instruction is not followed. + Otherwise, the buffer would end up in whatever mode the last instruction to fire said, across every plan ever sent. """ applied: dict = {"mode": None} @@ -405,7 +413,7 @@ def forget(done: asyncio.Task, key: str = key) -> None: async def report_fill_level( ws, buffer: Buffer, interval: float = STATUS_INTERVAL ) -> None: - """Report where the buffer is every interval, which is what asks the CEM for a new schedule.""" + """Report the buffer's fill level every interval, which triggers the CEM to plan a new schedule.""" while True: await asyncio.sleep(interval) buffer.advance(datetime.now(timezone.utc)) @@ -420,9 +428,9 @@ async def report_fill_level( async def refresh_forecasts( ws, leakage: FRBCLeakageBehaviour, interval: float = FORECAST_INTERVAL ) -> None: - """Send the leakage, the usage forecast and the comfort profile again every interval, each from now on. + """Send the leakage behaviour, the usage forecast and the comfort profile again every interval, each starting now. - Sent only once, they would cover the first day, and the CEM would plan without them from then on. + If they were sent only once, they would only cover the first day, and after that the CEM would plan without them. """ while True: await asyncio.sleep(interval) @@ -439,7 +447,7 @@ async def refresh_forecasts( async def receive_until(ws, message_type: str) -> dict: - """The next message of a type, past the reception statuses that arrive in between.""" + """Return the next message of a type, skipping the reception statuses that arrive in between.""" while True: message = (await ws.receive()).json() print("RECEIVED: ", message) @@ -450,10 +458,10 @@ async def receive_until(ws, message_type: str) -> dict: async def run_resource_manager( ws, interval: float = STATUS_INTERVAL, forecast_interval: float = FORECAST_INTERVAL ) -> Buffer: - """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. + """Introduce the heat pump, declare what it can do and what its buffer needs, then follow instructions and report. - In the order S2 suggests: the system description first, then the storage's status. - Returns the simulated buffer, for whoever wants to look at where the instructions took it. + The messages follow the order that S2 suggests: the system description first, then the storage's status. + Returns the simulated buffer, so that a caller can inspect where the instructions took it. """ await ws.send_str( Handshake( @@ -469,7 +477,7 @@ async def run_resource_manager( message_id=get_unique_id(), resource_id=get_unique_id(), name="heat pump with a hot-water buffer", - # It draws electricity and stores heat, so it is a consumer of the commodity it is planned in. + # It draws electricity and stores heat, so it consumes the commodity it is planned in. roles=[ Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY) ], @@ -478,8 +486,8 @@ async def run_resource_manager( ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, ], - # It sends no PowerForecast. It could measure its power, which is what the next field says, - # and S2 requires at least one type there. + # It sends no PowerForecast. + # It could measure its power, which is what the next field declares, and S2 requires at least one type there. provides_forecast=False, provides_power_measurement_types=[ELECTRIC_POWER], ).model_dump_json() diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 591540bd..4768f4ba 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -35,8 +35,8 @@ SITE_NAME = "My CEM" SITE_TIMEZONE = "Europe/Amsterdam" -# Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, -# with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. +# Enough days of demo prices to cover the CEM's planning horizon, which is twelve hours from now. +# Seeding is repeated every SEED_PRICES_EVERY, so that a demo left running never runs out of prices. SEED_PRICE_DAYS = 3 SEED_PRICES_EVERY = timedelta(hours=12) @@ -47,17 +47,17 @@ "yes", ) -#: Each sensor the site needs, which is the library's table of what an FRBC site records. +#: The sensors the site needs, taken from the library's table of sensors for an FRBC site. SITE_SENSORS = FRBC_SITE_SENSORS -#: What every site asset is made with: a building, whose connection takes three phases of 25 A at 230 V. +#: The specs to create every site asset with: a building with a three-phase connection of 25 A per phase at 230 V. SITE_ASSET_SPECS: dict[str, Any] = dict( generic_asset_type_id=6, flex_model={"power-capacity": f"{3 * 25 * 230} VA"}, ) -#: Whether each resource manager gets a site of its own, named after it, as in a co-simulation of a community, -#: rather than all of them sharing the one site this server sets up at startup. +#: Whether each resource manager gets its own site, named after it, as in a co-simulation of a community. +#: If not, they all share the one site that this server sets up at startup. SITE_PER_RESOURCE_MANAGER = os.getenv("S2_SITE_PER_RESOURCE_MANAGER", "").lower() in ( "1", "true", @@ -92,8 +92,8 @@ async def rm_details_watchdog(ws, cem: CEM): cem._logger.debug(f"CONTROL TYPE: {cem._control_type}") - # From here on, a schedule is asked for on each storage status, - # and on a system description that arrives after one. + # From here on, each storage status triggers a schedule request. + # So does a system description that arrives after a storage status. async def websocket_producer(ws, cem: CEM): @@ -168,22 +168,22 @@ async def serve_resource_manager( ) -> None: """Act as the CEM for one resource manager connected over ws, until it disconnects. - On the site whose sensors are given, or, with a configurator, on a site of the resource manager's own, - which is set up once its details arrive. + The CEM uses the site whose sensors are given. + With a configurator, it uses the resource manager's own site instead, which is set up once its details arrive. """ cem = CEM( fm_client=fm_client, logger=LOGGER, site_configurator=configurator, - # The server's, which outlives a connection, so a resource manager that reconnects is not recorded twice. + # The server's recorder outlives a connection, so a resource manager that reconnects is not recorded twice. measurement_recorder=recorder, - # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. + # The sensor to record the resource manager's measurements on, for the fixed site. A configured site wires its own. power_sensor_id=( {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} if sensors is not None and "measured-power" in sensors else None ), - # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. + # A co-simulation's resource manager wraps its messages with the simulated time, and this sets the clock to that time. accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) if sensors is not None: @@ -192,13 +192,13 @@ async def serve_resource_manager( def fixed_site_handler(sensors: dict[str, dict]) -> FRBCSimple: - """The handler for a resource manager on the one site this server sets up at startup.""" + """Return the handler for a resource manager on the one site that this server sets up at startup.""" frbc = FRBCSimple( power_sensor_id=sensors["power"]["id"], price_sensor_id=sensors["price"]["id"], soc_sensor_id=sensors["state of charge"]["id"], rm_discharge_sensor_id=sensors["RM discharge"]["id"], - # Where the resource manager's declarations are recorded, so that each can be seen next to the schedule planned against it. + # Record the resource manager's declarations, so that each can be inspected next to the schedule planned against it. fill_rate_sensor_id=sensors["fill rate"]["id"], soc_minima_sensor_id=sensors["soc minima"]["id"], soc_maxima_sensor_id=sensors["soc maxima"]["id"], @@ -206,20 +206,21 @@ def fixed_site_handler(sensors: dict[str, dict]) -> FRBCSimple: leakage_behaviour_sensor_id=sensors["storage efficiency"]["id"], charging_efficiency_sensor_id=sensors["charging efficiency"]["id"], timezone=SITE_TIMEZONE, - # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. + # Plan from now, not from a day after the description, so that a demo shows its instructions being followed. valid_from_shift=timedelta(0), ) - # A newer plan revokes what is left of the one before, so the resource manager does not follow both. + # A newer plan revokes the remainder of the previous plan, so the resource manager does not follow both. frbc.send_revocations = True return frbc async def serve(ws, cem: CEM) -> None: """Serve one resource manager with this CEM, for as long as its connection lasts.""" - # The session lasts as long as the resource manager's messages do. - # When they end, the CEM is closed, which stops what its handlers still have in flight, - # and the producer and the watchdog stop with it: left running, they outlived the connection, - # and a resource manager that reconnected was served by two sessions planning for it at once. + # The session lasts as long as the resource manager keeps sending messages. + # When the messages end, the CEM is closed, which stops any work its handlers still have in flight. + # The producer and the watchdog stop too. + # When they were left running, they outlived the connection. + # A resource manager that reconnected was then served by two sessions, both planning for it at once. consumer = asyncio.create_task(websocket_consumer(ws, cem)) side_tasks = [ asyncio.create_task(websocket_producer(ws, cem)), @@ -244,7 +245,7 @@ async def configure_site( asset_specs=SITE_ASSET_SPECS, sensors=SITE_SENSORS, timezone=SITE_TIMEZONE, - # The demo's own graphs, on every start, so changes made to them in the UI do not outlive a restart. + # Reset the demo's graphs on every start, so that changes made to them in the UI do not survive a restart. reset_graphs=True, ) await seed_prices(fm_client, sensors["price"]["id"]) @@ -252,12 +253,12 @@ async def configure_site( async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: - """Demo prices, so that a site has something to schedule against. + """Post demo prices, so that a site has something to schedule against. - One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - plans ahead of the moment a resource manager reports its storage: - a fixed date here goes stale the day after it is written, and the scheduler then fails the - job with UnknownPricesException rather than returning a schedule. + The prices repeat one day's hourly shape over SEED_PRICE_DAYS, starting this morning. + The CEM plans ahead of the moment a resource manager reports its storage. + A fixed date here would go stale the day after it was written. + The scheduler would then fail the job with UnknownPricesException, instead of returning a schedule. """ one_day = [ 0.10, 0.11, 0.12, 0.15, 0.18, 0.17, 0.11, 0.09, @@ -277,9 +278,10 @@ async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> No async def keep_prices_seeded(app: web.Application): - """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them. + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so that a demo never runs out of them. - Not where each resource manager has a site of its own, whose prices are whoever runs them to post. + This is skipped if each resource manager has its own site. + Then whoever runs those resource managers is responsible for posting their prices. """ if SITE_PER_RESOURCE_MANAGER: yield @@ -324,16 +326,16 @@ async def configure_site_once(app: web.Application) -> None: app[FM_CLIENT] = fm_client app[RECORDER] = MeasurementRecorder(fm_client, logger=LOGGER) if SITE_PER_RESOURCE_MANAGER: - # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. + # Use one configurator for the server's lifetime, so that a resource manager that reconnects finds its site again by id. app[CONFIGURATOR] = FRBCSiteConfigurator( fm_client, asset_type_id=SITE_ASSET_SPECS["generic_asset_type_id"], asset_specs={"flex_model": SITE_ASSET_SPECS["flex_model"]}, timezone=SITE_TIMEZONE, - # The co-simulation's resource managers are heat pumps, which cannot feed power back: - # FlexMeasures then keeps their production at zero even where constraints are relaxed. + # The co-simulation's resource managers are heat pumps, which cannot feed power back. + # With this attribute, FlexMeasures keeps their production at zero, even where constraints are relaxed. power_sensor_attributes={"is_strictly_non_positive": True}, - # A community's capacity is a target set by its members' steering, breached at a price, as #188 asked for. + # A community's capacity is a target for steering its members, so it may be breached at a price, as #188 asked for. handler_options={"relax_site_capacity": True}, logger=LOGGER, ) diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py index cf3f783a..56279f9f 100644 --- a/src/flexmeasures_client/s2/site.py +++ b/src/flexmeasures_client/s2/site.py @@ -1,8 +1,9 @@ -"""A FlexMeasures site for an S2 resource manager: its asset, its sensors, its graphs, and the handler wired to them. +"""Set up a FlexMeasures site for an S2 resource manager. +A site consists of an asset, its sensors and its graphs, plus a handler wired to those sensors. The example server keeps one site for whichever resource manager connects (see ensure_site). -A CEM serving many resource managers, as a co-simulation of a community does, gives each one a site of its own, -named after it, with FRBCSiteConfigurator. +A CEM can also serve many resource managers, as in a co-simulation of a community. +Such a CEM uses FRBCSiteConfigurator to give each resource manager its own site, named after it. """ from __future__ import annotations @@ -29,7 +30,7 @@ @dataclass(frozen=True) class SiteSensor: - """What a site's sensor is made with: its unit, its resolution, and any attributes.""" + """The settings for creating a site's sensor: its unit, its resolution, and any attributes.""" unit: str resolution: str @@ -37,18 +38,18 @@ class SiteSensor: #: The sensors an FRBC site needs, by name. -#: Their units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. -#: The names are a contract with whoever reads the site, a co-simulation among them, -#: which looks up "price", "production price", "power", "state of charge" and "measured-power" by name. +#: Their units are the units the CEM records in: fill levels in kWh, and fill rates per second, as in S2. +#: The names are a contract with any reader of the site, such as a co-simulation. +#: A co-simulation looks up "price", "production price", "power", "state of charge" and "measured-power" by name. FRBC_SITE_SENSORS: dict[str, SiteSensor] = { "price": SiteSensor("EUR/kWh", "PT15M"), "production price": SiteSensor("EUR/kWh", "PT15M"), - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. + # In S2, consumed power is positive. + # This sensor holds both an actuator's power and the schedule planned for it. + # Without this attribute, FlexMeasures treats production as positive on a power sensor. + # It would then flip the sign of an incoming schedule, so the power and the schedule would point in opposite directions. "power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), - # What the resource manager measures its site drawing, kept apart from the schedule on "power". + # The power the resource manager measures its site drawing, kept separate from the schedule on "power". "measured-power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), "state of charge": SiteSensor("kWh", "PT0M"), "RM discharge": SiteSensor("dimensionless", "PT15M"), @@ -62,13 +63,14 @@ class SiteSensor: def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: - """A site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + """Return a site's graphs, top to bottom, in the format of an asset's sensors_to_show in FlexMeasures. - In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, - rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. - A graph whose sensors the site does not have is left out. + Each graph has a title and a list of sensors. + Every FlexMeasures version reads this format. + The newer format with plots draws the same graphs, but needs FlexMeasures v0.32 or later. + A graph is left out if the site has none of its sensors. """ - # Titled as #188 titled them, since a co-simulation's controller adds to the graph titled "Power". + # Keep the titles from #188, because a co-simulation's controller adds to the graph titled "Power". graphs = [ ("Prices", ["price", "production price"]), ("Power", ["power", "measured-power"]), @@ -93,16 +95,17 @@ async def ensure_site( known_asset_id: int | None = None, reset_graphs: bool = False, ) -> tuple[dict, dict[str, dict]]: - """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. + """Find or create the site asset with this name, with its specs, all of these sensors, and its graphs. - It is safe to run again: what exists is kept, and what is missing is made. - A flex model in the specs is merged into the one the asset has, since others write to it too, - such as a co-simulation's controller setting what its stock is worth at the end. - Graphs are set where the asset has none, since others add to them too, unless asked to reset them. + Running this again is safe: existing parts are kept, and missing parts are created. + A flex model in the specs is merged into the asset's existing flex model, because others write to it too. + For example, a co-simulation's controller sets the value of the asset's stock at the end. + Graphs are only set if the asset has none, because others add to them too. + Pass reset_graphs to set them anyway. - :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. - :param reset_graphs: whether to set the graphs even where the asset has some. - :returns: the asset, and its sensors by name. + :param known_asset_id: The asset's id from an earlier lookup, which avoids listing every asset. + :param reset_graphs: Whether to set the graphs even if the asset already has some. + :returns: The asset, and its sensors by name. """ site_asset = None if known_asset_id is not None: @@ -115,7 +118,7 @@ async def ensure_site( if candidate is not None and candidate.get("name") == name: site_asset = candidate if site_asset is None: - # The newest of any of that name, as a co-simulation picks it too. + # Pick the newest asset with that name, as a co-simulation does too. named = [ asset for asset in await fm_client.get_assets(parse_json_fields=True) @@ -163,20 +166,20 @@ async def ensure_site( @dataclass class ConfiguredSite: - """What a CEM needs to serve one resource manager: its handler, and where its measured power goes.""" + """What a CEM needs to serve one resource manager: its handler, and the sensors to record its measured power on.""" handler: FRBCSimple - #: The sensor each commodity quantity's measured power is recorded on, as the CEM's power_sensor_id takes it. + #: Per commodity quantity, the sensor to record measured power on, in the format of the CEM's power_sensor_id. power_sensor_ids: dict[str, int] asset_id: int class FRBCSiteConfigurator: - """Gives each resource manager a site of its own, named after it, and an FRBC handler wired to its sensors. + """Give each resource manager its own site, named after it, and an FRBC handler wired to the site's sensors. - Made once per server and handed to each CEM it makes (see CEM's site_configurator), - so that a resource manager that reconnects finds its site by the id remembered here, - rather than by a listing of every asset on the server. + The server creates one configurator and hands it to each CEM it creates (see the CEM's site_configurator). + The configurator remembers each site's asset id. + So when a resource manager reconnects, its site is found by that id, without listing every asset on the server. """ def __init__( @@ -193,15 +196,15 @@ def __init__( logger: Logger | None = None, ) -> None: """ - :param asset_type_id: the FlexMeasures asset type each site is made with. - :param timezone: the time zone of each site's sensors, and of its handler. - :param asset_specs: anything else each site asset is made or updated with, such as a flex model. - :param sensors: the sensors each site gets; FRBC_SITE_SENSORS by default. - :param power_sensor_attributes: attributes added to the "power" sensor's, which a device's nature decides, + :param asset_type_id: The FlexMeasures asset type to create each site with. + :param timezone: The timezone of each site's sensors, and of its handler. + :param asset_specs: Other specs to create or update each site asset with, such as a flex model. + :param sensors: The sensors each site gets; FRBC_SITE_SENSORS by default. + :param power_sensor_attributes: Extra attributes for the "power" sensor that depend on the device, such as that it cannot produce. - :param handler_options: keyword arguments for each FRBCSimple beyond its sensors. - By default it plans from now, over 24 hours, as a co-simulation steps through days. - :param send_revocations: whether a newer plan revokes what is left of the one before. + :param handler_options: Keyword arguments for each FRBCSimple, other than its sensors. + By default, it plans from now over 24 hours, because a co-simulation steps through days. + :param send_revocations: Whether a newer plan revokes the remainder of the previous plan. """ self._fm_client = fm_client self._timezone = timezone diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 0da31bcb..004fc7e9 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -28,14 +28,14 @@ def __init__(self, *args, max_size=100, **kwargs): super(SizeLimitOrderedDict, self).__init__(*args, **kwargs) self._max_size = max_size - # deleting values to make the dictionary of length _max_size at most, oldest first + # Delete the oldest entries until the dictionary holds at most _max_size entries. while len(self) > self._max_size: self.popitem(last=False) def __setitem__(self, __key: KT, __value: VT) -> None: - # The oldest entry makes room, not the newest, which popitem() takes by default: - # a full history otherwise kept a session's first entries and lost every later one but the last. - # A key already present takes no room of its own. + # Evict the oldest entry, not the newest (which is what popitem() removes by default). + # Otherwise, a full history kept a session's first entries and lost all later ones except the last. + # A key that is already present needs no extra room. if ( self._max_size is not None and __key not in self @@ -44,7 +44,8 @@ def __setitem__(self, __key: KT, __value: VT) -> None: self.popitem(last=False) super().__setitem__(__key, __value) - # Set again, an entry is the latest, which is what reading the last entry means everywhere this is used. + # An entry that is set again becomes the latest entry. + # Everywhere this class is used, reading the last entry means reading the latest one. self.move_to_end(__key) @@ -72,7 +73,7 @@ def get_message_id(message: pydantic.BaseModel) -> str | None: class _HasMessageId(Protocol): - """Any S2 message: every one of them carries a message id.""" + """Any S2 message, since every S2 message carries a message id.""" message_id: Any @@ -86,7 +87,7 @@ def get_reception_status( This function returns a ReceptionStatus message for the subject message `subject_message`. By default, the status ReceptionStatusValues.OK is sent. - :param diagnostic_label: why, for a status other than OK, which the resource manager can log. + :param diagnostic_label: For a status other than OK, the reason, which the resource manager can log. """ return ReceptionStatus( subject_message_id=str(subject_message.message_id), From 9f031885bbbc4acea2d9c76cff1671d72bec3727 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 8 Nov 2026 15:54:24 +0100 Subject: [PATCH 80/87] docs(S2): keep what the plain rewrite of the comments got wrong A check of every rewritten docstring and comment against the code found one false claim (that a failure was always retried before retries were bounded, when there were no retries at all), a capacity comment that read as a community-level capacity rather than each member site's, and smaller shifts: a condition made of a fact, a lost "because", a lost example, a tense slip, a softened "cannot", and a line over the length limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 3 ++- .../s2/control_types/FRBC/__init__.py | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 14 +++++++------- .../s2/control_types/FRBC/utils.py | 2 +- src/flexmeasures_client/s2/measurements.py | 2 +- .../s2/script/websockets_server.py | 2 +- src/flexmeasures_client/s2/utils.py | 2 +- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index eb3707a5..34122e77 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -126,7 +126,8 @@ def __init__( and set the clock to that time (see flexmeasures_client.s2.wrapper.S2Wrapper). This is off by default, because the wrapper is not part of S2, and a live CEM should not let a resource manager set its time. - When it is off, a resource manager that wraps its messages from its handshake on is refused with INVALID_MESSAGE, and cannot connect. + When it is off, a resource manager that wraps its messages from its handshake on is refused with INVALID_MESSAGE, + and cannot connect. When it is on, the clock must be a SimulatedClock. If no clock is given, a SimulatedClock is created. """ diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 140bd290..258889eb 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -213,7 +213,7 @@ def handle_system_description( # Storing it again would push an older description out of the bounded history for no reason. # Also, a handler that acts on a description would act on it twice. # The acknowledgement still matters, because without it the resource manager retries. - # Ignoring a resend is safe because planning is triggered by storage statuses, not by descriptions. + # Ignoring a resend is safe because planning is normally triggered by storage statuses, and a resend never differs from the previous description. # Before that was the case, ignoring a resend would have dropped a planning request. # The TUNES handler has always fingerprinted its descriptions this way. # It can adopt this shared helper separately. diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index bfaad55a..748c6702 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -250,8 +250,8 @@ def __init__( schedule_duration: timedelta = timedelta(hours=12), # How many times to request a schedule before leaving the storage without an instruction, # and how long to wait between two attempts. - # One attempt means a failure is never retried. - # Before this limit existed, a failure was always retried with another attempt. + # One attempt means a failure is never retried, + # which is how this handler behaved before retries were added. schedule_attempts: int = SCHEDULE_ATTEMPTS, schedule_retry_interval: timedelta = SCHEDULE_RETRY_INTERVAL, # The resolution that schedules are planned on; a schedule's start must line up with it. @@ -396,7 +396,7 @@ def schedule_start(self, valid_from: datetime) -> datetime: Planning happens on every storage status, while a description may be days old. If the start were based on that description alone, every status in a long session would get the same window. - That window slid into the past, so all its instructions were already due, and arrived as one burst. + That window would slide into the past, so all its instructions would already be due, and arrive as one burst. Hence the start is never before now. If now is past that boundary, the schedule starts at the next boundary, not at the start of the current step. A schedule is planned from the fill level reported now, and FlexMeasures takes that as the fill level at the schedule's start. @@ -592,7 +592,7 @@ def _fill_level_band( The latest fill-level target profile, held inside that range, applies to the steps it covers. The range used to reach FlexMeasures only through the band sensors, and only where a target profile was recorded on them. So a storage without those sensors, or past the profile's end, was planned without any band. - It could then be planned down to empty, and up to half again above its range. + It could then be planned down to empty, and up to the hard upper bound (by default, half again above its range). The values are clipped at zero, because FlexMeasures does not accept negative minima. """ steps = int( @@ -715,7 +715,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): ) operation_mode = actuator.operation_modes[0] - # The element the actuator runs on depends on the fill level, just as a heat pump derates while its buffer warms up. + # The element the actuator runs on depends on the fill level; for example, a heat pump derates while its buffer warms up. # So use the element that applies at the latest reported fill level, not the element that the mode happens to list first. fill_level = self._latest_fill_level() if fill_level is not None: @@ -1142,7 +1142,7 @@ async def _plan( max(power_range.start_of_range, power_range.end_of_range), ) # Only use the modes the actuator may be instructed into under normal conditions. - # The actuator may be instructed into any band that the scheduler plans in. + # This is because the actuator may then be instructed into any band that the scheduler plans in. for operation_mode in normal_operation_modes(actuator) for element in operation_mode.elements for power_range in element.power_ranges @@ -1211,7 +1211,7 @@ async def _plan( if self._usage_forecast_sensor_id is not None: # Skip this if the resource manager says it provides no usage forecast, # because then nothing is recorded on the sensor for the scheduler to read. - # But use it if the resource manager sent a forecast anyway, whatever its flag says. + # But use it if the resource manager sent a forecast anyway, since sending one says it provides one, whatever its flag says. if ( system_description.storage.provides_usage_forecast or self._usage_forecast_history diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 0b597962..4e8fb54f 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -492,7 +492,7 @@ def operation_mode_conversion_efficiency( The lowest element over the whole range would be the wrong choice. It overstates what the actuator needs to fill the storage. The plan then overshoots its upper band, and buys more than it uses. - And if the plan cannot relax the usage, the scheduler may find a problem infeasible that the actuator can actually meet. + And because the plan cannot relax the usage, the scheduler may find a problem infeasible that the actuator can actually meet. The ratio is taken at the top of that element's range, not as the slope over the range. An actuator may well declare degenerate ranges, where a mode runs at only one power and one fill rate. diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py index 3f173f36..b5c9f443 100644 --- a/src/flexmeasures_client/s2/measurements.py +++ b/src/flexmeasures_client/s2/measurements.py @@ -205,7 +205,7 @@ def step_of(moment: datetime, resolution: timedelta) -> datetime: The step start is in the moment's own timezone, floored the way FlexMeasures floors it. During the repeated hour of an autumn DST change, a step boundary occurs twice. The step then starts at the later of the two boundaries that is not after the moment. - Flooring based only on which of the two passes the moment is in does not always work. + Flooring based only on which of the two passes the moment is in cannot always be done. """ if not resolution: return moment diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 4768f4ba..80d736ec 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -335,7 +335,7 @@ async def configure_site_once(app: web.Application) -> None: # The co-simulation's resource managers are heat pumps, which cannot feed power back. # With this attribute, FlexMeasures keeps their production at zero, even where constraints are relaxed. power_sensor_attributes={"is_strictly_non_positive": True}, - # A community's capacity is a target for steering its members, so it may be breached at a price, as #188 asked for. + # Each member's site capacity is a target set by the community that steers it, so it may be breached at a price, as #188 asked for. handler_options={"relax_site_capacity": True}, logger=LOGGER, ) diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 004fc7e9..86e6f7d9 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -34,7 +34,7 @@ def __init__(self, *args, max_size=100, **kwargs): def __setitem__(self, __key: KT, __value: VT) -> None: # Evict the oldest entry, not the newest (which is what popitem() removes by default). - # Otherwise, a full history kept a session's first entries and lost all later ones except the last. + # Popping the newest instead made a full history keep a session's first entries, and lose all later ones except the last. # A key that is already present needs no extra room. if ( self._max_size is not None From ef5739f9370836f840e5770bb5ceaab13488c933 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 9 Nov 2026 16:48:46 +0100 Subject: [PATCH 81/87] fix(FRBC): record an actuator status that arrived before any system description After a reconnection, a resource manager can send its actuator status before its system description. Only a description says what power an operation mode stands for, so such a status was logged and skipped, and the power sensor showed nothing until the next status. It is now kept, the latest per actuator, and recorded once a description arrives. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/__init__.py | 12 ++++++++++++ .../s2/control_types/FRBC/frbc_simple.py | 6 ++++-- tests/s2/test_frbc_actuator_measurement.py | 17 +++++++++++++---- 4 files changed, 30 insertions(+), 7 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 12887b18..a19b7df5 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -123,7 +123,7 @@ Fixed * A full message history now keeps the newest messages. It used to make room by evicting the newest entry, so it kept a session's first messages and lost all later ones except the last. A message stored again under the same id now counts as the latest [`#274 `__] * ``Handler.is_revoked()`` now finds objects the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] -* An FRBC actuator status that arrives before any system description is logged and skipped, instead of raising an error in a background task. This can happen after a reconnection [`#274 `__] +* An FRBC actuator status that arrives before any system description, which can happen after a reconnection, is kept, and recorded once a system description arrives. Only the latest per actuator is kept, since a status describes the actuator's present state. It used to raise an error in a background task [`#274 `__, `#PRNUM `__] * A CEM records measured power by the time it was measured. It records one average per step of the sensor's resolution (15 minutes on the example's sites, previously five), in watts, as S2 states it. A step is recorded with the first measurement that arrives after the step has ended by the CEM's clock. Before, measurements were grouped by the CEM's current time, which dropped every measurement sent after the fact, as co-simulations do. They were also labelled as kilowatts instead of watts, and kept in a buffer shared by every CEM in the process. Recording happens in the background, with a few retries if it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records L1 measurements on sensor 357. That was whatever sensor had that id on the server, on any site, and the CEM reported the measurement as recorded [`#265 `__] * A CEM that is selecting a control type now takes a message of that control type as confirmation of the selection, because a resource manager only sends such messages once it uses that control type. Before, the CEM stayed without a control type, and answered each such message with a temporary error, which the resource manager retried; co-simulations whose resource managers reconnect showed this. A message of any other control type activates nothing [`#263 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 258889eb..70857762 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -55,6 +55,8 @@ class FRBC(ControlTypeHandler): _timer_status_history: SizeLimitOrderedDict[str, FRBCTimerStatus] _actuator_status_history: SizeLimitOrderedDict[str, FRBCActuatorStatus] + #: Per actuator, the latest status that arrived before any system description, to be handled once one arrives. + _statuses_awaiting_a_description: dict[str, FRBCActuatorStatus] _storage_status_history: SizeLimitOrderedDict[str, FRBCStorageStatus] background_tasks: set _sent_instructions: dict[str, FRBCInstruction] @@ -75,6 +77,7 @@ def __init__(self, max_size: int = 100) -> None: self._timer_status_history = SizeLimitOrderedDict(max_size=max_size) self._actuator_status_history = SizeLimitOrderedDict(max_size=max_size) + self._statuses_awaiting_a_description = {} self._storage_status_history = SizeLimitOrderedDict(max_size=max_size) self._system_description_history = SizeLimitOrderedDict(max_size=max_size) @@ -243,6 +246,15 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message + # A status that arrived before any description could not be handled then, since only a description says what its operation mode stands for. + # A status describes the actuator's present state, so the latest per actuator is handled now. + awaiting = list(self._statuses_awaiting_a_description.values()) + self._statuses_awaiting_a_description.clear() + for status in awaiting: + task = asyncio.create_task(self.send_actuator_status(status)) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) + # Warn once per description, because nothing else here would mention it. # An instruction can ask for a switch that a running timer or a missing transition forbids, and then the device refuses it. actuator = message.actuators[0] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 748c6702..442272a3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -679,8 +679,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # After a reconnection, an actuator status can arrive before any description. # This method needs a description to tell what power the status stands for. if not self._system_description_history: - self._logger.warning( - "Not recording the power of an FRBC.ActuatorStatus: no FRBC.SystemDescription has arrived yet to tell what its operation mode draws." + # Kept instead, and recorded once a description arrives (see handle_system_description). + self._statuses_awaiting_a_description[str(status.actuator_id)] = status + self._logger.info( + "Not recording the power of an FRBC.ActuatorStatus yet: no FRBC.SystemDescription has arrived to tell what its operation mode draws, so it is recorded once one does." ) return sd: FRBCSystemDescription = list(self._system_description_history.values())[-1] diff --git a/tests/s2/test_frbc_actuator_measurement.py b/tests/s2/test_frbc_actuator_measurement.py index 3913182b..c2cbf431 100644 --- a/tests/s2/test_frbc_actuator_measurement.py +++ b/tests/s2/test_frbc_actuator_measurement.py @@ -11,6 +11,7 @@ from __future__ import annotations +import asyncio import logging from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock @@ -351,15 +352,23 @@ async def test_a_unit_nothing_converts_between_is_reported(system_description, c @pytest.mark.asyncio -async def test_an_actuator_status_before_any_description_is_said_and_skipped( +async def test_an_actuator_status_before_any_description_waits_for_one( system_description, caplog ): """After a reconnection one can arrive first; indexing the empty history raised an IndexError in a task nobody awaited.""" + caplog.set_level(logging.INFO) frbc = _frbc(system_description, consumption_as_positive=True) - status = _status(system_description, 0) frbc._system_description_history.clear() - await frbc.send_actuator_status(status) + await frbc.send_actuator_status(_status(system_description, 0)) + await frbc.send_actuator_status(_status(system_description, 1)) assert _posts_to_the_power_sensor(frbc) == [] - assert "no FRBC.SystemDescription has arrived yet" in caplog.text + assert "no FRBC.SystemDescription has arrived" in caplog.text + + await frbc.handle_system_description(system_description.to_dict()) + await asyncio.gather(*frbc.background_tasks) + + # Only the latest status describes the actuator's present state, so only that one is recorded. + assert [post["values"] for post in _posts_to_the_power_sensor(frbc)] == [[ON_W]] + assert frbc._statuses_awaiting_a_description == {} From d7d9f6c86109e9ad5b47d3a93663114abbffdc85 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 9 Nov 2026 16:49:14 +0100 Subject: [PATCH 82/87] docs(changelog): link #287 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index a19b7df5..79813170 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -123,7 +123,7 @@ Fixed * A full message history now keeps the newest messages. It used to make room by evicting the newest entry, so it kept a session's first messages and lost all later ones except the last. A message stored again under the same id now counts as the latest [`#274 `__] * ``Handler.is_revoked()`` now finds objects the resource manager revoked. The CEM and its handlers stored a revoked object's id as a ``UUID``, so a lookup by its string id always missed [`#257 `__] -* An FRBC actuator status that arrives before any system description, which can happen after a reconnection, is kept, and recorded once a system description arrives. Only the latest per actuator is kept, since a status describes the actuator's present state. It used to raise an error in a background task [`#274 `__, `#PRNUM `__] +* An FRBC actuator status that arrives before any system description, which can happen after a reconnection, is kept, and recorded once a system description arrives. Only the latest per actuator is kept, since a status describes the actuator's present state. It used to raise an error in a background task [`#274 `__, `#287 `__] * A CEM records measured power by the time it was measured. It records one average per step of the sensor's resolution (15 minutes on the example's sites, previously five), in watts, as S2 states it. A step is recorded with the first measurement that arrives after the step has ended by the CEM's clock. Before, measurements were grouped by the CEM's current time, which dropped every measurement sent after the fact, as co-simulations do. They were also labelled as kilowatts instead of watts, and kept in a buffer shared by every CEM in the process. Recording happens in the background, with a few retries if it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records L1 measurements on sensor 357. That was whatever sensor had that id on the server, on any site, and the CEM reported the measurement as recorded [`#265 `__] * A CEM that is selecting a control type now takes a message of that control type as confirmation of the selection, because a resource manager only sends such messages once it uses that control type. Before, the CEM stayed without a control type, and answered each such message with a temporary error, which the resource manager retried; co-simulations whose resource managers reconnect showed this. A message of any other control type activates nothing [`#263 `__] From 58f9a1574ac05935c59af0e18039b71ee9596d56 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 9 Nov 2026 16:50:03 +0100 Subject: [PATCH 83/87] docs(changelog): say that one sensor holds the charging efficiency of all operation modes As asked in the review of #283. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 12887b18..04b9d484 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -26,9 +26,9 @@ Added * ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge, the fill-level band, the fill rate and the usage forecast are recorded in this unit (the last two per second). Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] * ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule into instructions keeps the resource manager's own units, because it matches fill levels against the operation modes, which are stated in those units. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which an actuator's power is converted into and posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] * ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a hard upper bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: it records there what the operation modes say about converting drawn power into stored energy, and requests a schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. +* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: before each schedule request, it records there one efficiency for all operation modes, over the window of the schedule, from what they say about converting drawn power into stored energy, and requests the schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. - - FlexMeasures takes one efficiency per device. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. If modes differ, the lowest is used, and the choice is logged. + - FlexMeasures takes one efficiency per device, so the modes share one sensor. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. If modes differ, the lowest is used, and the choice is logged. Recording each mode's efficiency apart only helps once FlexMeasures can plan with an efficiency per mode (`FlexMeasures #2832 `__). - A mode that draws power without filling the storage, such as standby or circulation, does not count. Counting it as zero would tell the scheduler that the storage cannot be filled at all. Modes reserved for abnormal conditions do not count either. - The efficiency is recorded once per window. It is recorded again for the next schedule if FlexMeasures did not confirm storing it. If it cannot be recorded, the schedule is requested without it, and this is logged. From 8299241d9c3b555357829393134d6d8804644db7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 9 Nov 2026 19:32:35 +0100 Subject: [PATCH 84/87] docs(changelog): split the charging efficiency entry into short sentences Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 04b9d484..b3eb4f7e 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -26,9 +26,9 @@ Added * ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge, the fill-level band, the fill rate and the usage forecast are recorded in this unit (the last two per second). Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] * ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule into instructions keeps the resource manager's own units, because it matches fill levels against the operation modes, which are stated in those units. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which an actuator's power is converted into and posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] * ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a hard upper bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` can take the actuator's charging efficiency into account. Give it the new ``charging_efficiency_sensor_id``: before each schedule request, it records there one efficiency for all operation modes, over the window of the schedule, from what they say about converting drawn power into stored energy, and requests the schedule with it as ``charging-efficiency``. For a heat pump, this ratio is its coefficient of performance. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. +* ``FRBCSimple`` can take the actuator's charging efficiency into account: how much stored energy the drawn power becomes. For a heat pump, this is its coefficient of performance. Give it the new ``charging_efficiency_sensor_id``. Before each schedule request, it records the efficiency there, over the window of the schedule, and requests the schedule with it as ``charging-efficiency``. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. - - FlexMeasures takes one efficiency per device, so the modes share one sensor. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. If modes differ, the lowest is used, and the choice is logged. Recording each mode's efficiency apart only helps once FlexMeasures can plan with an efficiency per mode (`FlexMeasures #2832 `__). + - FlexMeasures takes one efficiency per device, so all operation modes share one sensor and one value. If modes differ, the lowest is used, and the choice is logged. If a mode's efficiency depends on the fill level, the value where the schedule starts is used. A sensor per mode would only help once FlexMeasures can plan with an efficiency per mode (`FlexMeasures #2832 `__). - A mode that draws power without filling the storage, such as standby or circulation, does not count. Counting it as zero would tell the scheduler that the storage cannot be filled at all. Modes reserved for abnormal conditions do not count either. - The efficiency is recorded once per window. It is recorded again for the next schedule if FlexMeasures did not confirm storing it. If it cannot be recorded, the schedule is requested without it, and this is logged. From ebdc615ca4ae5116d411121710cd0d5f74ed2e91 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Nov 2026 00:09:00 +0100 Subject: [PATCH 85/87] feat(FRBC): record rates of fill level in a fill_rate_unit, by default W S2 states fill rates and usage per second, which in kWh/s puts a heat buffer's rates near 1e-5. Ported from #289 without its leakage-as-usage work, which stays with #286. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 55 ++++++++++------ .../s2/control_types/FRBC/utils.py | 5 ++ tests/s2/test_frbc_declared_profiles.py | 62 ++++++++++++------- tests/s2/test_frbc_soft_band.py | 2 +- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 442272a3..1331a499 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -36,6 +36,7 @@ from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( WATTS_PER_ENERGY_UNIT_PER_SECOND, + WATTS_PER_FILL_RATE_UNIT, clip_fill_level_target_profile, element_for_fill_level, fm_schedule_to_instructions, @@ -271,6 +272,10 @@ def __init__( # The default is kilowatt-hours, because S2 is a residential protocol, and a home battery or heat buffer is of that order. # A deployment at another scale can set a different unit here. energy_unit: str = "kWh", + # The unit that rates of fill level are recorded in: the fill rate and the usage forecast. + # S2 states them per second, which in kWh/s makes tiny numbers, so they are recorded in a power unit: W, kW or MW. + # The default is watts, in which a residential heat buffer's rates read as such. + fill_rate_unit: str = "W", ) -> None: super().__init__(max_size) self._power_sensor_id = power_sensor_id @@ -340,24 +345,34 @@ def __init__( f"energy_unit has to be one of {', '.join(WATTS_PER_ENERGY_UNIT_PER_SECOND)}, but {energy_unit!r} was given." ) self.energy_unit = energy_unit + if fill_rate_unit not in WATTS_PER_FILL_RATE_UNIT: + raise ValueError( + f"fill_rate_unit has to be one of {', '.join(WATTS_PER_FILL_RATE_UNIT)}, but {fill_rate_unit!r} was given." + ) + self._fill_rate_unit = fill_rate_unit # delay the start of the schedule from the time `valid_from` # of the FRBC.SystemDescription. self._valid_from_shift = valid_from_shift def fill_rate_unit(self) -> str: - """The unit a rate of change of the fill level is recorded in. + """The unit a rate of change of the fill level is recorded in: the fill_rate_unit given, by default W. - S2 states a fill rate, and a usage forecast, as fill level per second, so this follows - whatever unit the fill level itself is recorded in. It is not the power unit: a fill rate - is what an actuator delivers into its buffer, which for a heat buffer is not electricity - at all, and which even for a battery differs from kilowatts by the seconds in an hour. + S2 states a fill rate and a usage forecast as fill level per second. + Per second, a residential heat buffer's rates are tiny numbers in kWh/s, so they are recorded in a power unit instead (see _per_second_in_fill_rate_unit). + It is not the power unit: a fill rate is what an actuator delivers into its buffer, which for a heat buffer is not electricity at all. - A method rather than a property, because S2 Handler's discover() walks dir(self) and - getattrs every name during __init__, before a subclass has assigned its own attributes. + A method rather than a property, because S2 Handler's discover() walks dir(self) and getattrs every name during __init__, before a subclass has assigned its own attributes. A property would be evaluated there and raise on the energy_unit it reads. """ - return f"{self.energy_unit}/s" + return self._fill_rate_unit + + def _per_second_in_fill_rate_unit(self, rate): + """Return a rate of fill level per second, as S2 states it, in the fill-rate unit; a number or a series.""" + return rate * ( + WATTS_PER_ENERGY_UNIT_PER_SECOND[self.energy_unit] + / WATTS_PER_FILL_RATE_UNIT[self._fill_rate_unit] + ) def align_to_resolution(self, moment: datetime) -> datetime: """The last resolution boundary at or before that moment. @@ -810,10 +825,13 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # useful quantity - what the actuator delivers into its buffer - it simply is not power. if self._fill_rate_sensor_id is not None: fill_rate = element.fill_rate - rate = ( - fill_rate.start_of_range - + (fill_rate.end_of_range - fill_rate.start_of_range) * factor - ) * self._fill_level_scale + rate = self._per_second_in_fill_rate_unit( + ( + fill_rate.start_of_range + + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + ) + * self._fill_level_scale + ) # Catch failures, like for the power above, because this runs in a task that nobody awaits. try: await self._fm_client.post_sensor_data( @@ -1414,11 +1432,13 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): ) return - translated = translate_usage_forecast_to_fm( - usage_forecast, - self._resolution, - strategy="mean", - fill_level_scale=self._fill_level_scale, + translated = self._per_second_in_fill_rate_unit( + translate_usage_forecast_to_fm( + usage_forecast, + self._resolution, + strategy="mean", + fill_level_scale=self._fill_level_scale, + ) ) # A negative usage is a gain that the storage is forced to take, and FlexMeasures cannot relax it. # Above the hard ceiling, it makes every schedule for the window infeasible, so the storage gets no instruction at all. @@ -1441,7 +1461,6 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): start=self.align_to_resolution(usage_forecast.start_time), prior=self.now(), values=translated.tolist(), - # translate_usage_forecast_to_fm returns fill level per second, scaled. unit=self.fill_rate_unit(), duration=pd.Timedelta(self._resolution) * len(translated), ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 4e8fb54f..0c30ca9f 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -451,6 +451,11 @@ def get_soc_min_max( } +#: The units a rate of fill level can be recorded in, and the number of watts in one of each. +#: S2 states rates per second, which in kWh/s puts a heat buffer's rates of some tens of watts near 1e-5, so they are recorded in a power unit instead, by default W. +WATTS_PER_FILL_RATE_UNIT: dict[str, float] = {"W": 1.0, "kW": 1e3, "MW": 1e6} + + def fill_rate_in_watts(fill_rate: float, energy_unit: str) -> float: """Convert a fill rate to watts, given the unit that the fill level is recorded in. diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index a1489358..f8dca1b5 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -260,7 +260,7 @@ async def test_usage_forecast_is_recorded_from_a_resolution_boundary( assert posted["start"] == frbc.align_to_resolution(usage_forecast.start_time) assert posted["start"].minute % 15 == 0 and posted["start"].second == 0 # A usage forecast is a rate of fill level, not a power. - assert posted["unit"] == frbc.fill_rate_unit() == "kWh/s" + assert posted["unit"] == frbc.fill_rate_unit() == "W" @pytest.mark.asyncio @@ -546,31 +546,44 @@ async def test_the_instruction_translation_stays_in_the_resource_managers_units( assert translate.call_args.kwargs["initial_fill_level"] == 30.0 -def test_a_fill_rate_is_not_recorded_as_a_power(): - """S2 states a fill rate as fill level per second, which is not the power unit. - - For a heat buffer the quantity is not electricity at all, and even for a battery kWh/s - differs from kW by the number of seconds in an hour. - """ - frbc = FRBCSimple( +def _handler(**units) -> FRBCSimple: + return FRBCSimple( power_sensor_id=1, price_sensor_id=2, soc_sensor_id=3, rm_discharge_sensor_id=4, - energy_unit="kWh", - power_unit="W", + **units, ) - assert frbc.fill_rate_unit() == "kWh/s" - assert frbc.fill_rate_unit() != frbc.power_unit - megawatt_hours = FRBCSimple( - power_sensor_id=1, - price_sensor_id=2, - soc_sensor_id=3, - rm_discharge_sensor_id=4, - energy_unit="MWh", - ) - assert megawatt_hours.fill_rate_unit() == "MWh/s" + +@pytest.mark.parametrize( + "units, fill_rate_unit, recorded", + [ + # 0.01 kWh/s is 36 kW. + ({}, "W", 36000.0), + ({"fill_rate_unit": "kW"}, "kW", 36.0), + ({"energy_unit": "MWh", "fill_rate_unit": "MW"}, "MW", 36.0), + # A joule per second is a watt. + ({"energy_unit": "kJ"}, "W", 10.0), + ({"energy_unit": "J", "fill_rate_unit": "kW"}, "kW", 0.00001), + ], +) +def test_a_fill_rate_is_recorded_in_the_fill_rate_unit_whatever_the_power_unit( + units, fill_rate_unit, recorded +): + """S2 states a fill rate as fill level per second, which in kWh/s makes tiny numbers, so it is recorded in a power unit, by default W. + + That is not the power unit, which says how an actuator's electric power is posted: for a heat buffer, a fill rate is not electricity at all. + """ + frbc = _handler(power_unit="kW", **units) + assert frbc.fill_rate_unit() == fill_rate_unit + assert frbc._per_second_in_fill_rate_unit(0.01) == pytest.approx(recorded) + + +@pytest.mark.parametrize("fill_rate_unit", ["kWh/s", "kWh", "GW"]) +def test_a_fill_rate_unit_that_is_no_power_unit_is_refused(fill_rate_unit): + with pytest.raises(ValueError, match="fill_rate_unit has to be one of W, kW, MW"): + _handler(fill_rate_unit=fill_rate_unit) @pytest.mark.asyncio @@ -609,8 +622,13 @@ async def test_an_actuator_s_fill_rate_is_posted_in_the_fill_rate_unit( ) ) - assert ( - _posted(frbc, fill_rate_sensor_id)["unit"] == frbc.fill_rate_unit() == "kWh/s" + assert _posted(frbc, fill_rate_sensor_id)["unit"] == frbc.fill_rate_unit() == "W" + # Halfway through the mode's fill rate, from kWh/s into W. + fill_rate = actuator.operation_modes[0].elements[0].fill_rate + halfway = (fill_rate.start_of_range + fill_rate.end_of_range) / 2 + assert halfway > 0 + assert _posted(frbc, fill_rate_sensor_id)["values"] == pytest.approx( + [halfway * 3.6e6] ) # The power sensor really does carry a power, and is untouched by this. assert _posted(frbc, 1)["unit"] == frbc.power_unit diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index f160d3cc..26462c9c 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -557,7 +557,7 @@ async def test_a_negative_usage_forecast_is_recorded_as_no_usage_and_said_so( await frbc.send_usage_forecast(forecast) posted = frbc._fm_client.post_sensor_data.await_args.kwargs assert min(posted["values"]) == 0.0 - assert posted["values"][0] == pytest.approx(0.01) + assert posted["values"][0] == pytest.approx(36000.0) # 0.01 kWh/s, in W assert "negative usage on 2 of 4 steps" in caplog.text From c983c0e887584e9abe17ab2a9ce1dce6ade09a96 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Nov 2026 00:09:01 +0100 Subject: [PATCH 86/87] feat(site): create a site's power and rate sensors in W Consistent with the handler's defaults, and readable at residential scale. Sensors an existing site already has are reused by name and keep their units. Ported from #289. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/site.py | 10 +++++----- tests/s2/test_example_end_to_end.py | 4 ++-- tests/s2/test_site.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py index 56279f9f..b7648518 100644 --- a/src/flexmeasures_client/s2/site.py +++ b/src/flexmeasures_client/s2/site.py @@ -38,7 +38,7 @@ class SiteSensor: #: The sensors an FRBC site needs, by name. -#: Their units are the units the CEM records in: fill levels in kWh, and fill rates per second, as in S2. +#: Their units suit a residential site, and match the handler's defaults: energy and fill levels in kWh, prices per kWh, and powers and rates of fill level in W. #: The names are a contract with any reader of the site, such as a co-simulation. #: A co-simulation looks up "price", "production price", "power", "state of charge" and "measured-power" by name. FRBC_SITE_SENSORS: dict[str, SiteSensor] = { @@ -48,15 +48,15 @@ class SiteSensor: # This sensor holds both an actuator's power and the schedule planned for it. # Without this attribute, FlexMeasures treats production as positive on a power sensor. # It would then flip the sign of an incoming schedule, so the power and the schedule would point in opposite directions. - "power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + "power": SiteSensor("W", "PT15M", {"consumption_is_positive": True}), # The power the resource manager measures its site drawing, kept separate from the schedule on "power". - "measured-power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + "measured-power": SiteSensor("W", "PT15M", {"consumption_is_positive": True}), "state of charge": SiteSensor("kWh", "PT0M"), "RM discharge": SiteSensor("dimensionless", "PT15M"), - "fill rate": SiteSensor("kWh/s", "PT15M"), + "fill rate": SiteSensor("W", "PT15M"), "soc minima": SiteSensor("kWh", "PT15M"), "soc maxima": SiteSensor("kWh", "PT15M"), - "usage forecast": SiteSensor("kWh/s", "PT15M"), + "usage forecast": SiteSensor("W", "PT15M"), "storage efficiency": SiteSensor("%", "PT15M"), "charging efficiency": SiteSensor("dimensionless", "PT15M"), } diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 12a64aa3..ae23ecb7 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -326,7 +326,7 @@ async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_toda made = {call.kwargs["name"]: call.kwargs for call in fm.add_sensor.await_args_list} assert "price" not in made assert made["power"]["attributes"] == {"consumption_is_positive": True} - assert made["fill rate"]["unit"] == "kWh/s" + assert made["fill rate"]["unit"] == "W" assert made["soc minima"]["unit"] == "kWh" prices = fm.post_sensor_data.await_args.kwargs assert prices["sensor_id"] == 50 @@ -539,7 +539,7 @@ def test_the_site_sensors_are_what_the_cem_records_in(): server.SITE_SENSORS["state of charge"].resolution, ) == ("kWh", "PT0M") assert server.SITE_SENSORS["storage efficiency"].unit == "%" - assert server.SITE_SENSORS["usage forecast"].unit == "kWh/s" + assert server.SITE_SENSORS["usage forecast"].unit == "W" @pytest.mark.asyncio diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py index 69676bb3..cb0ac08e 100644 --- a/tests/s2/test_site.py +++ b/tests/s2/test_site.py @@ -84,6 +84,25 @@ async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): assert specs_update.args == (7, {"generic_asset_type_id": 6}) +def test_a_site_s_sensors_suit_a_residential_site(): + """Powers and rates of fill level in W, energy in kWh: in kWh/s, a heat buffer's usage of some tens of watts would read about 0.00001.""" + units = {name: spec.unit for name, spec in FRBC_SITE_SENSORS.items()} + assert units == { + "price": "EUR/kWh", + "production price": "EUR/kWh", + "power": "W", + "measured-power": "W", + "state of charge": "kWh", + "RM discharge": "dimensionless", + "fill rate": "W", + "soc minima": "kWh", + "soc maxima": "kWh", + "usage forecast": "W", + "storage efficiency": "%", + "charging efficiency": "dimensionless", + } + + @pytest.mark.asyncio async def test_a_known_site_is_fetched_by_id_rather_than_found_in_a_listing(): """A resource manager reconnects every round of a co-simulation; a listing of every asset each time is slow.""" From e102532cd3b3b01b63da034a4e4441434e45ce24 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Nov 2026 00:12:48 +0100 Subject: [PATCH 87/87] docs(changelog): fill_rate_unit and the site sensors' units (#290) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 2f961304..59a3e66e 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -19,11 +19,12 @@ Added * ``FRBCSimple`` can record what a resource manager declares about its storage, so it can be seen and planned with. Give it any of the new ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``; a declaration without a sensor is not recorded. - A fill-level target profile is recorded on the minima and maxima sensors, if both are given. It is recorded on the handler's ``resolution`` steps, starting from the step its start falls in. It is kept inside the declared fill-level range: a target past either end, such as a night setback below the range, is cut back to the range, and a target band wholly outside the range is recorded at its nearest end, with a warning. If one of the two bounds cannot be recorded, this is logged and the other is still recorded. - - A usage forecast is recorded in ``/s``. A negative expected usage is recorded as zero, with a warning. + - A usage forecast is recorded in ``fill_rate_unit`` (see below). A negative expected usage is recorded as zero, with a warning. - A leakage behaviour is recorded as a storage efficiency, each time a schedule is requested (see Changed). [`#244 `__, `#259 `__, `#271 `__, `#274 `__] -* ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge, the fill-level band, the fill rate and the usage forecast are recorded in this unit (the last two per second). Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] +* ``FRBCSimple`` takes an ``energy_unit`` (default ``kWh``). The state of charge and the fill-level band are recorded in this unit. Allowed units are ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``; any other unit is refused when the handler is created. In a flex model, the state of charge is stated in ``kWh`` or ``MWh``, converted from the other units, because FlexMeasures accepts no others [`#244 `__, `#260 `__] +* ``FRBCSimple`` takes a ``fill_rate_unit`` (default ``W``; also ``kW`` or ``MW``). Rates of fill level, which S2 states per second, are recorded in it: the fill rate and the usage forecast. In ``kWh/s``, a heat buffer's rates of some tens of watts would read about 0.00001. Any other unit is refused when the handler is created [`#290 `__] * ``FRBCSimple`` takes a ``fill_level_scale`` (default 1). Every fill level and fill rate sent to FlexMeasures is multiplied by it: the state of charge, the flex model, the fill-level band, the fill rate, the usage forecast and the storage efficiency. Translating a schedule into instructions keeps the resource manager's own units, because it matches fill levels against the operation modes, which are stated in those units. A scale of zero or below is refused. ``FRBCSimple`` also takes a ``power_unit`` (default ``W``), which an actuator's power is converted into and posted in [`#236 `__, `#244 `__, `#245 `__, `#252 `__] * ``FRBCSimple`` takes a ``hard_bound_factor`` (default 1.5). It sets how far above the declared fill-level range a schedule's hard upper bound lies. If a schedule starts from a state of charge above that bound, the bound is raised to include it. A factor below 1 is refused: FlexMeasures clips the band to the hard bounds, so a hard upper bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] * ``FRBCSimple`` can take the actuator's charging efficiency into account: how much stored energy the drawn power becomes. For a heat pump, this is its coefficient of performance. Give it the new ``charging_efficiency_sensor_id``. Before each schedule request, it records the efficiency there, over the window of the schedule, and requests the schedule with it as ``charging-efficiency``. Without it, the scheduler assumes power converts into stored energy one to one, and plans to reach fill-level targets at a power the actuator cannot deliver. @@ -48,10 +49,10 @@ Added * An FRBC handler tracks the instruction batch the resource manager is working from, and can revoke instructions it no longer wants carried out. Set its ``send_revocations`` attribute to ``True``. When a new batch replaces that one, the handler revokes the old batch's pending instructions: those without a reported status, or reported as ``NEW`` or ``ACCEPTED``. This is off by default, because a resource manager does not have to honour revocations [`#237 `__] * New ``FRBC`` methods for handlers. ``filter_instruction_transitions()`` drops instructions that repeat the previous one. ``send_instruction_batch()`` sends a batch that replaces the previous one, unless a newer request has arrived in the meantime. ``system_description_fingerprint()`` tells a changed system description from a re-sent one [`#237 `__] * ``ControlTypeHandler.send_message()``. Handlers now use it to send messages, instead of putting them on the CEM's sending queue themselves [`#237 `__] -* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there. The fill rate is recorded in ``/s``, not as a power: S2 states a fill rate as fill level per second, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] +* ``FRBCSimple`` takes an optional ``fill_rate_sensor_id``, and records the actuator's fill rate there. The fill rate is recorded in ``fill_rate_unit``, not in ``power_unit``: a fill rate is what the actuator delivers into its buffer, which for a heat buffer is not electricity at all [`#236 `__, `#244 `__] * ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``. The clock sets the time for the CEM and for every handler it registers. That includes their timers, and the time a recorded state of charge is known from [`#264 `__, `#268 `__] * ``CEM(accept_simulation_metadata=True)`` accepts messages from a co-simulation that are wrapped with the simulated time (``S2Wrapper``), and sets the CEM's clock from them. It is off by default, and then a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] -* ``flexmeasures_client.s2.site``, with ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``. It gives each resource manager its own FlexMeasures site, named after it, with an FRBC handler connected to that site's sensors [`#265 `__] +* ``flexmeasures_client.s2.site``, with ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``. It gives each resource manager its own FlexMeasures site, named after it, with an FRBC handler connected to that site's sensors. The sensors' units suit a residential site and match the handler's defaults: powers and rates of fill level in ``W``, energy and fill levels in ``kWh``, and prices per ``kWh``. For sites at a larger scale, such as charge points, pass ``sensors`` with other units, and the handler's ``power_unit`` and ``fill_rate_unit`` in ``handler_options`` [`#265 `__, `#290 `__] * ``CEM(site_configurator=...)`` sets up a resource manager's site once its details arrive. Until then, the CEM answers its messages with a temporary error. The example server enables this with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] * ``examples/S2-co-simulation``: a FlexMeasures stack with one CEM per simulated home, for co-simulations whose resource managers connect over websockets [`#265 `__, `#267 `__] * ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)`` record a resource manager's measured power. Use one recorder per server, so that a resource manager that reconnects is not recorded twice [`#266 `__]