From 43bb171b1e49653cf779962fa13ece0b44195341 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:10:53 +0100 Subject: [PATCH 01/59] fix(TUNES): record a leakage behaviour instead of refusing it FillRateBasedControlTUNES.send_leakage_behaviour read its fill-level scale from self._fill_level_scalefill_level_scale, an attribute name pasted in twice, which has been there since #158. The AttributeError that raised was caught by the handler's broad except as if the post had failed, so every FRBC.LeakageBehaviour was answered with PERMANENT_ERROR and none was ever recorded. mypy could not see it: the type-check task only looks at files that contain `from typing import`, and the FRBC base class is not one of them, so the attribute lookup went to Any. The next commit widens that. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_tunes.py | 2 +- tests/s2/test_frbc_tunes.py | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 3b6b956c..29a1e4f7 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -239,7 +239,7 @@ async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): leakage_behaviour_to_storage_efficiency( message=leakage, resolution=timedelta(minutes=15), - fill_level_scale=self._fill_level_scalefill_level_scale, + fill_level_scale=self._fill_level_scale, ) ], unit=PERCENTAGE, diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 645692c9..cd43fb9b 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -288,3 +288,44 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert third_call["values"][0] == frbc._nes_fill_rate_sensor_id await cem.close() + + +@pytest.mark.asyncio +async def test_a_leakage_behaviour_is_recorded_rather_than_refused( + cem_in_frbc_control_type, +): + """A leakage behaviour is posted as a storage efficiency, and not answered as a permanent error. + + The handler read its fill-level scale from an attribute whose name was pasted in twice, + and the AttributeError that raised was caught as if the post had failed, + so every leakage behaviour was refused and none was recorded. + """ + from s2python.common import NumberRange + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc.send_message = AsyncMock() + + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + await frbc.send_leakage_behaviour(leakage) + + sent = [call.args[0] for call in frbc.send_message.await_args_list] + assert not any( + getattr(message, "status", None) == ReceptionStatusValues.PERMANENT_ERROR + for message in sent + ), sent + posted_to = [ + call.args[0] if call.args else call.kwargs.get("sensor_id") + for call in fm_client.post_sensor_data.await_args_list + ] + assert 15 in posted_to, posted_to From 30dc81e8f7e1001d3bdd6a1bedeae62cbfecffbb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:12:22 +0100 Subject: [PATCH 02/59] chore(types): check every source file, including untyped function bodies The type-check task fed mypy only the files that contain `from typing import`, which were seven of eighteen, and left out frbc_simple.py, both control-type __init__.py files and the example scripts. With --follow-imports skip, anything those files defined then looked like Any to the files that were checked, which is how the previous commit's misspelt attribute went unnoticed since #158. It now checks all of src, and with --check-untyped-defs, so the bodies of functions without annotations are checked too. What that surfaced, besides the attribute fixed in the previous commit: - get_access_token stored a token whose type could be None and then used it as a header value; it now returns the token, so the caller holds a str. - get_access_token and get_versions indexed a response typed as a dict or a list; each now says so plainly when the server answers with something other than an object, instead of failing on the lookup. - The S2 translations annotated their resolution as a string while frbc_simple passes a timedelta, which pandas accepts as readily; the annotation now says so. - FillRateBasedControlTUNES overrode trigger_schedule without the arguments the base class declares. It calls its own override without them and ignores them, so it now accepts and ignores them. - The example server indexed an account that can be None and spread a dict of mixed values into add_asset; it now says so when the demo user has no organisation, and types the dict. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- pyproject.toml | 7 +-- src/flexmeasures_client/client.py | 23 +++++++--- .../s2/control_types/FRBC/frbc_tunes.py | 6 ++- .../s2/control_types/translations.py | 6 +-- .../s2/script/websockets_server.py | 7 ++- tests/client/test_init.py | 46 +++++++++++++++++++ 6 files changed, 79 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3407ad3d..33c29c8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,11 +89,8 @@ help = "Run S2 tests." cmd = "pytest tests/s2" [tool.poe.tasks.type-check] -help = "Run mypy on files with type hints." -shell = """ - files=$(find src -name \\*.py | xargs grep -l "from typing import") - mypy --follow-imports skip --ignore-missing-imports $files -""" +help = "Run mypy on every source file, including the bodies of functions without type hints." +cmd = "mypy --follow-imports skip --ignore-missing-imports --check-untyped-defs src" [tool.poe.tasks.lint] help = "Run pre-commit hooks on all files." diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c344d900..14de2cc4 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -563,9 +563,10 @@ async def get_headers(self, include_auth: bool) -> dict: headers = dict(CONTENT_TYPE_HEADERS) | {CLIENT_VERSION_HEADER: client_version} if include_auth: - if self.access_token is None: - await self.get_access_token() - headers |= {"Authorization": self.access_token} + access_token = self.access_token + if access_token is None: + access_token = await self.get_access_token() + headers |= {"Authorization": access_token} return headers def build_url(self, uri: str, path: str = path) -> URL: @@ -577,8 +578,8 @@ def build_url(self, uri: str, path: str = path) -> URL: ) return url - async def get_access_token(self): - """Get access token and store it on the FlexMeasuresClient.""" + async def get_access_token(self) -> str: + """Get access token, store it on the FlexMeasuresClient, and return it.""" response, _status = await self.request( uri="requestAuthToken", path="/api/", @@ -588,7 +589,13 @@ async def get_access_token(self): }, include_auth=False, ) - self.access_token = response["auth_token"] + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from requestAuthToken, but got a {type(response).__name__}." + ) + access_token: str = response["auth_token"] + self.access_token = access_token + return access_token async def get_versions(self): """Get the FlexMeasures version and the supported API versions of the FlexMeasures server. @@ -601,6 +608,10 @@ async def get_versions(self): method="GET", include_auth=False, ) + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from the API root, but got a {type(response).__name__}." + ) # Check whether the API version used by the client is supported by the server server_api_versions = response["versions"] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 29a1e4f7..45245d89 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -310,7 +310,11 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - async def trigger_schedule(self): + async def trigger_schedule( + self, + system_description_id: str | None = None, + request_number: int | None = None, + ): """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager """ diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a51a14be..a5ccea7c 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -75,7 +75,7 @@ def unevenly_ts_to_evenly( start: pd.Timestamp, values: list[float], durations: list[pd.Timedelta], - target_resolution: str, + target_resolution: str | timedelta, strategy="mean", ) -> pd.Series: """ @@ -132,7 +132,7 @@ def unevenly_ts_to_evenly( def translate_usage_forecast_to_fm( usage_forecast: FRBCUsageForecast, - resolution: str = "1h", + resolution: str | timedelta = "1h", strategy: str = "mean", fill_level_scale: float = 1, ) -> pd.Series: @@ -166,7 +166,7 @@ def translate_usage_forecast_to_fm( def translate_fill_level_target_profile( fill_level_target_profile: FRBCFillLevelTargetProfile, - resolution: str = "1h", + resolution: str | timedelta = "1h", fill_level_scale: float = 1, ) -> tuple[pd.Series, pd.Series]: """ diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 354adb18..108a475b 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime, timedelta +from typing import Any from urllib.parse import urlparse from zoneinfo import ZoneInfo @@ -133,6 +134,10 @@ async def configure_site( ) -> tuple[dict, dict, dict, dict]: """Create or update the site asset and its sensors, and seed it with demo prices.""" account = await fm_client.get_account() + if account is None: + raise ValueError( + "The demo user belongs to no organisation, so there is nowhere to create the site." + ) assets = await fm_client.get_assets(parse_json_fields=True) site_asset = None @@ -141,7 +146,7 @@ async def configure_site( site_asset = asset break - site_asset_specs = dict( + site_asset_specs: dict[str, Any] = dict( latitude=0, longitude=0, generic_asset_type_id=6, # Building asset type diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 0647b16e..138f9809 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -957,3 +957,49 @@ async def test_server_version_at_least_looks_the_version_up() -> None: assert await flexmeasures_client.server_version_at_least("0.27.0") is True assert await flexmeasures_client.server_version_at_least("1.1.0") is False await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_an_auth_token_answer_that_is_not_an_object_is_refused() -> None: + """A list where the token should be says what came back, instead of failing on the lookup.""" + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload=["auth_token"], + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + with pytest.raises(ValueError, match="requestAuthToken, but got a list"): + await flexmeasures_client.get_access_token() + assert flexmeasures_client.access_token is None + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_the_access_token_is_returned_as_well_as_stored() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + assert await flexmeasures_client.get_access_token() == "a-token" + assert flexmeasures_client.access_token == "a-token" + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_version_answer_that_is_not_an_object_is_refused() -> None: + with aioresponses() as m: + m.get("http://localhost:5000/api/", status=200, payload=["v3_0"]) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ValueError, match="API root, but got a list"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() From ff1304c95985da8089010e94f4d62396bfe2128d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:12:32 +0100 Subject: [PATCH 03/59] ci: measure the coverage of the S2 tests None of the S2 integration had a coverage number, for two reasons that each hid the other. The S2 tests run in a CI job of their own, since they need the s2 extra, and that job never asked for coverage. And .coveragerc omitted src/flexmeasures_client/s2/* outright, since #183, so asking for it would have reported nothing about S2 while looking as though it did. The omission is gone, and the S2 job now reports and uploads its own partial report, which Coveralls merges with the other job's in the finalize step, as it already does across the test matrix. The poe task that runs the S2 tests locally collects it as well. The job that skips tests/s2 will now list the S2 files as unexecuted in its own report; the merged figure is the one that counts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .coveragerc | 3 --- .github/workflows/ci.yml | 14 +++++++++++++- pyproject.toml | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5d068fcb..9cda9250 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,9 +2,6 @@ [run] branch = True source = flexmeasures_client -omit = - src/flexmeasures_client/s2/* - */site-packages/flexmeasures_client/s2/* [paths] source = diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb624446..188fa1bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,19 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra s2 --group test - name: Run S2 tests - run: uv run pytest tests/s2 -rFEx --durations 10 --color yes + run: uv run pytest tests/s2 -rFEx --durations 10 --color yes --cov + - name: Show coverage report + run: uv run coverage report + - name: Generate coverage report + run: uv run coverage lcov -o coverage.lcov + - name: Upload partial coverage report + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + path-to-lcov: coverage.lcov + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: s2 - py${{ matrix.python }} + parallel: true finalize: needs: [ test, test-s2] diff --git a/pyproject.toml b/pyproject.toml index 33c29c8b..8f1552e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,8 @@ help = "Run the test suite (excluding S2) and generate coverage report." cmd = "pytest --ignore=tests/s2 --cov --cov-report=" [tool.poe.tasks.test-s2] -help = "Run S2 tests." -cmd = "pytest tests/s2" +help = "Run S2 tests and generate coverage report." +cmd = "pytest tests/s2 --cov --cov-report=" [tool.poe.tasks.type-check] help = "Run mypy on every source file, including the bodies of functions without type hints." From 372a39a827bc5249237b01ebf4b273818d3e37d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:13:53 +0100 Subject: [PATCH 04/59] chore(lint): no print() in the S2 code that runs unattended A CEM's handlers run for days with nobody watching standard output, so anything they print is lost to a deployment, which keeps the log instead. FRBCSimple gave up on a schedule that way when no storage status had arrived, which is the very failure, a storage left without an instruction, that the handler otherwise goes to some length to say out loud. It now logs a warning. A pygrep hook keeps it that way for s2/control_types and s2/cem.py. The example scripts print on purpose, as do the client's interactive delete confirmations, which answer an input() prompt on the console, so the hook leaves those alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 8 +++++++ .../s2/control_types/FRBC/frbc_simple.py | 4 +++- tests/s2/test_frbc_simple_planning.py | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09dd0ef1..536c9366 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,3 +66,11 @@ repos: Put screenshots in the FlexMeasures/screenshots repo, or leave them untracked and attach them to the PR/issue manually. files: '\.(png|jpe?g|gif|bmp|tiff?|webp|ico|psd)$' + - id: no-print-in-unattended-code + name: No print() in code that runs unattended + # A CEM's handlers run for days without anyone watching standard output, + # so what they have to say goes to the log, where a deployment keeps it. + # The example scripts and the client's interactive confirmations print on purpose, and are left alone. + language: pygrep + entry: '^\s*print\(' + files: '^src/flexmeasures_client/s2/(control_types/|cem\.py$)' diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 351f8f97..720854e3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -726,7 +726,9 @@ async def trigger_schedule( -1 ].present_fill_level else: - print("Can't trigger schedule without knowing the status of the storage...") + self._logger.warning( + "Cannot ask for a schedule: no FRBC.StorageStatus has arrived yet, so there is no state of charge to plan from." + ) return # Scaled at the FlexMeasures edge, like every bound and level below it. diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index dab9231c..5d5e5c77 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -195,3 +195,25 @@ async def test_the_schedule_is_asked_for_from_an_aligned_start(): start = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs["start"] assert start == datetime(2022, 12, 3, 9, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_planning_without_a_storage_status_is_logged_not_printed(capsys, caplog): + """A handler runs unattended, so giving up on a schedule goes to the log, where it can be found. + + It used to go to standard output, which a deployment does not keep. + """ + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + system_description_id = list(frbc._system_description_history)[-1] + + with caplog.at_level("WARNING", logger="FRBCSimple"): + await frbc.trigger_schedule(system_description_id) + + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + assert capsys.readouterr().out == "" + assert any( + record.levelname == "WARNING" and "FRBC.StorageStatus" in record.getMessage() + for record in caplog.records + ), caplog.records From 9ae5a5cd7e7a053dbaaff9300828ebf694630466 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:18:24 +0100 Subject: [PATCH 05/59] docs(changelog): FillRateBasedControlTUNES records a leakage behaviour Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 502eea76..1bcc43a4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,6 +41,7 @@ Changed Fixed ----- +* ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] * ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] From f791bdbb8f2fb5673e9a50826a21c0842e0b4455 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:38:47 +0100 Subject: [PATCH 06/59] chore: close the gaps an adversarial review found in the instruments - The print ban now matches a print anywhere on a line and with a space before its parenthesis, plus pprint and writes to stdout or stderr, and covers all of s2/ but the example scripts. It only caught an indented `print(` at the start of a line before, in two directories, and missed s2/__init__.py, utils.py and wrapper.py, which run inside the CEM as much as the handlers do. - The example scripts get an __init__.py. Without one, coverage never listed websockets_server.py at all, since no test imports it, so the S2 report left it out instead of showing it at 0%. Running them by path is unaffected. - The TUNES leakage test now pins what is posted, its value and its unit, rather than only that something reached the sensor. - A client without a token now has a test that it authorises with the one it fetches. - FillRateBasedControlTUNES.trigger_schedule is back to taking no arguments, with the override mismatch marked for mypy instead. Accepting the base class's arguments and ignoring them would have turned a caller's mistake, such as expecting its superseded-request protection, into a silent one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 9 ++++++--- .../s2/control_types/FRBC/frbc_tunes.py | 9 ++++----- src/flexmeasures_client/s2/script/__init__.py | 1 + tests/client/test_init.py | 16 ++++++++++++++++ tests/s2/test_frbc_tunes.py | 19 ++++++++++++++++--- 5 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/flexmeasures_client/s2/script/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 536c9366..a53e82e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -70,7 +70,10 @@ repos: name: No print() in code that runs unattended # A CEM's handlers run for days without anyone watching standard output, # so what they have to say goes to the log, where a deployment keeps it. - # The example scripts and the client's interactive confirmations print on purpose, and are left alone. + # The example scripts print on purpose, and so do the client's interactive confirmations, which live outside s2/. + # Matched anywhere on a line, not only at its start, and with a space before the parenthesis too, + # so that `if x: print(...)`, `return print(...)`, `print (...)`, pprint and writes to stdout or stderr are caught. language: pygrep - entry: '^\s*print\(' - files: '^src/flexmeasures_client/s2/(control_types/|cem\.py$)' + entry: '\b(print|pprint)\s*\(|sys\.std(out|err)\.write' + files: '^src/flexmeasures_client/s2/.*\.py$' + exclude: '^src/flexmeasures_client/s2/script/' diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 45245d89..4dae13ca 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -310,11 +310,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - async def trigger_schedule( - self, - system_description_id: str | None = None, - request_number: int | None = None, - ): + # It takes neither argument the base class declares: TUNES plans from its latest description, + # and does not take part in dropping superseded requests. + # Accepting them and ignoring them would turn a caller's mistake into a silent one, so the mismatch stays. + async def trigger_schedule(self): # type: ignore[override] """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager """ diff --git a/src/flexmeasures_client/s2/script/__init__.py b/src/flexmeasures_client/s2/script/__init__.py new file mode 100644 index 00000000..c777fdcf --- /dev/null +++ b/src/flexmeasures_client/s2/script/__init__.py @@ -0,0 +1 @@ +"""Example scripts: a CEM server and a resource manager that talk S2 to each other, run by path as the CEM documentation shows.""" diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 138f9809..52a02af7 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -1003,3 +1003,19 @@ async def test_a_version_answer_that_is_not_an_object_is_refused() -> None: with pytest.raises(ValueError, match="API root, but got a list"): await flexmeasures_client.get_versions() await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_client_without_a_token_authorises_with_the_one_it_fetches() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + headers = await flexmeasures_client.get_headers(include_auth=True) + assert headers["Authorization"] == "a-token" + await flexmeasures_client.close() diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index cd43fb9b..4086b25b 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -324,8 +324,21 @@ async def test_a_leakage_behaviour_is_recorded_rather_than_refused( getattr(message, "status", None) == ReceptionStatusValues.PERMANENT_ERROR for message in sent ), sent - posted_to = [ - call.args[0] if call.args else call.kwargs.get("sensor_id") + posts = [ + call for call in fm_client.post_sensor_data.await_args_list + if (call.args[0] if call.args else call.kwargs.get("sensor_id")) == 15 ] - assert 15 in posted_to, posted_to + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + # What the translation makes of this message on a quarter-hour grid, which is what TUNES says it posts. + # The fill-level scale is not exercised here: it multiplies both the leakage and the range it is a share of, and cancels. + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + assert posts[0].kwargs["values"] == [ + leakage_behaviour_to_storage_efficiency( + message=leakage, resolution=timedelta(minutes=15), fill_level_scale=1 + ) + ] + assert posts[0].kwargs["unit"] == "%" From 5ff631deafb2457b6c7a1ff51aeb04271056af8f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:32:45 +0100 Subject: [PATCH 07/59] fix(FRBC): answer by the element that applies, whatever order S2 lists them in S2 lets the elements of an operation mode, and of a leakage behaviour, cover the fill-level range in pieces, and s2python checks that they are contiguous on a sorted copy, leaving the list in whatever order it was sent. Four places took an element by its position instead, and answered three different ways: - the measured power of an actuator status took elements[0], so a heat pump that derates as its buffer warms was recorded at its cold power when warm: 2000 W where it draws at most 1200 W; - the charging efficiency took elements[-1] as though it were the top of the range, so a mode listed high element first planned against the wrong ratio; - the storage efficiency took the leakage rate of elements[-1], on the same assumption, though the leakiest element need not be the top one; - the schedule translation selected by fill level, but settled a shared boundary by list order. element_for_fill_level now answers wherever a fill level is known: the element whose range holds it, the upper of two at a shared boundary, and past either end of the range the element that applies at that end, so the answer does not jump there. The measured power and the schedule translation use it. Before any storage status there is no fill level, so the measured power takes the bottom of the range and says in the log that it guessed. Where FlexMeasures takes one number for a whole schedule, it does not know which fill levels the schedule will pass through, so the most cautious element answers, which is the rule #253 already applies across modes: the lowest conversion efficiency, and the leakiest element. That is also independent of order. The efficiency now also skips an element that draws power while the storage does not fill, such as a standby or circulation mode. Before, such a mode gave a ratio of zero or below, which then won the minimum across modes, went onto the charging-efficiency sensor, and told the scheduler that no power could fill the storage at all; with the fill-level band priced rather than hard, never running the heat pump was then a feasible plan. Two guards were dropped rather than tested, because S2 makes them unreachable: every operation mode and leakage behaviour has at least one element, and every element at least one power range. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 21 +- .../s2/control_types/FRBC/utils.py | 100 +++-- .../s2/control_types/translations.py | 8 +- tests/s2/test_frbc_element_selection.py | 415 ++++++++++++++++++ 4 files changed, 506 insertions(+), 38 deletions(-) create mode 100644 tests/s2/test_frbc_element_selection.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 720854e3..a62f3c8e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -30,8 +30,10 @@ from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( clip_fill_level_target_profile, + element_for_fill_level, fm_schedule_to_instructions, get_soc_min_max, + om_label, operation_mode_conversion_efficiency, ) from flexmeasures_client.s2.control_types.translations import ( @@ -426,7 +428,24 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): status.actuator_id, ) operation_mode = actuator.operation_modes[0] - element = operation_mode.elements[0] + + # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, + # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. + if self._storage_status_history: + fill_level = list(self._storage_status_history.values())[ + -1 + ].present_fill_level + element = element_for_fill_level(operation_mode.elements, fill_level) + else: + # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. + element = element_for_fill_level(operation_mode.elements, float("-inf")) + if len(operation_mode.elements) > 1: + self._logger.warning( + "FRBC.ActuatorStatus for operation mode %s arrived with no FRBC.StorageStatus yet, " + "so its power is recorded as the element at the bottom of the fill-level range states it, " + "which need not be the element the actuator runs on.", + om_label(operation_mode), + ) # Interpolate the element's POWER range, not its fill rate. A fill rate is the thermal # output a heat pump delivers into its buffer; the power range is what it draws diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 4acd8d8c..d6f78386 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,7 +1,7 @@ import logging from datetime import timedelta from math import isclose -from typing import List +from typing import List, Sequence, TypeVar import numpy as np import pandas as pd @@ -11,6 +11,7 @@ from s2python.frbc import ( FRBCInstruction, FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCSystemDescription, ) @@ -327,32 +328,38 @@ def operation_mode_conversion_efficiency( Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, and for a thermal actuator that ratio is well above one. - The ratio is taken at the top of the last element's range, rather than as the slope over that range. + Each element's ratio is taken at the top of its range, rather than as the slope over that range. An actuator may well declare degenerate ranges, where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. - Where an efficiency varies over a mode's range, this is the one it reaches at full tilt. + + A mode whose elements convert differently, as a heat pump does when its buffer warms, states the lowest of them. + FlexMeasures takes one charging efficiency for a whole schedule, and does not know which fill levels it will pass through, + so the most cautious element answers, for the same reason the lowest mode answers across modes: + a plan that converts less than the actuator really does reaches its target early, where one that converts more reaches it late. + That also makes the answer independent of the order the elements were listed in, which S2 does not fix. :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :returns: the efficiency, or None where the mode states no power to convert. - An idle mode draws nothing, and a mode that only feeds power back is discharging, - which FlexMeasures takes as a separate discharging efficiency that this does not state. + :returns: the efficiency, or None where no element of the mode converts drawn power into stored energy. + An element converts nothing where it draws no power, as an idle mode does, + where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, + or where it draws power while the storage does not fill, as a standby or circulation mode does: + no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. """ - if not operation_mode.elements: - return None - element = operation_mode.elements[-1] - if not element.power_ranges: - return None - # todo: an element states one power range per commodity quantity, - # and no distinction is made between them here, - # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). - power = element.power_ranges[0].end_of_range - if power <= 0: - return None - fill_rate = element.fill_rate.end_of_range * fill_level_scale - return fill_rate_in_watts(fill_rate, energy_unit) / power + efficiencies = [] + # S2 gives every operation mode at least one element, and every element at least one power range. + for element in operation_mode.elements: + # todo: an element states one power range per commodity quantity, + # and no distinction is made between them here, + # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). + power = element.power_ranges[0].end_of_range + fill_rate = element.fill_rate.end_of_range * fill_level_scale + if power <= 0 or fill_rate <= 0: + continue + efficiencies.append(fill_rate_in_watts(fill_rate, energy_unit) / power) + return min(efficiencies) if efficiencies else None def clip_fill_level_target_profile( @@ -408,22 +415,12 @@ def power_to_fill_rate_with_metrics( efficiency: float element: FRBCOperationModeElement that produced fill_rate """ - elements = operation_mode.elements - - # --- 1. Select element with minimal fill-level penalty --- - best_element, fill_penalty = min( - ( - ( - e, - clamp_distance( - fill_level, - e.fill_level_range.start_of_range, - e.fill_level_range.end_of_range, - ), - ) - for e in elements - ), - key=lambda x: x[1], + # --- 1. Select the element that applies at this fill level --- + best_element = element_for_fill_level(operation_mode.elements, fill_level) + fill_penalty = clamp_distance( + fill_level, + best_element.fill_level_range.start_of_range, + best_element.fill_level_range.end_of_range, ) # --- 2. Select power range with minimal power penalty --- @@ -465,6 +462,39 @@ def clamp_distance(value: float, start: float, end: float) -> float: return 0.0 +ElementWithFillLevelRange = TypeVar( + "ElementWithFillLevelRange", FRBCOperationModeElement, FRBCLeakageBehaviourElement +) + + +def element_for_fill_level( + elements: Sequence[ElementWithFillLevelRange], fill_level: float +) -> ElementWithFillLevelRange: + """The element that applies at a fill level, whatever order the elements were listed in. + + S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, + and s2python checks that they are contiguous on a sorted copy, so the list itself may arrive in any order. + Two neighbouring elements share a boundary, which belongs to the upper of the two: + of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. + That includes an element whose range is a single point at the top, which no half-open rule could ever select. + A fill level outside the range the elements cover is answered as the nearest end of that range is, + since a storage may be outside its declared range, and the answer should not jump as it crosses that end. + Elements with a gap between them, which S2 rules out, answer a fill level in the gap with the nearer, and on a tie the lower. + """ + def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: + return e.fill_level_range.start_of_range, e.fill_level_range.end_of_range + + covered_from = min(bounds(e)[0] for e in elements) + covered_to = max(bounds(e)[1] for e in elements) + clamped = min(max(fill_level, covered_from), covered_to) + holding = [e for e in elements if bounds(e)[0] <= clamped <= bounds(e)[1]] + if holding: + return max(holding, key=bounds) + return min( + elements, key=lambda e: (clamp_distance(clamped, *bounds(e)), *bounds(e)) + ) + + def om_label(operation_mode: FRBCOperationMode) -> str: label = getattr(operation_mode, "diagnostic_label", None) return label if isinstance(label, str) else operation_mode.id diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a5ccea7c..8b02b15e 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -56,11 +56,15 @@ def leakage_behaviour_to_storage_efficiency( """ - last_element = message.elements[-1] + # FlexMeasures takes one storage efficiency here, and does not know which fill levels a schedule will pass through, + # so the leakiest element answers, as the most cautious: a plan that expects to lose more reaches its target early. + # It is looked for rather than assumed to come last, since S2 does not fix the order the elements are listed in, + # and the leakiest is not always the top one. + leakiest = max(message.elements, key=lambda e: e.leakage_rate) max_fill_level = max( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) - leakage = last_element.leakage_rate * fill_level_scale + leakage = leakiest.leakage_rate * fill_level_scale base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) # Discuss conversions diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py new file mode 100644 index 00000000..426b7d10 --- /dev/null +++ b/tests/s2/test_frbc_element_selection.py @@ -0,0 +1,415 @@ +"""Which element of an operation mode, or of a leakage behaviour, answers a question. + +S2 lets those elements cover the fill-level range in pieces, and lets them arrive in any order: +s2python's own contiguity check sorts a copy of them and leaves the list as it was sent. +So no answer may depend on the order they are listed in. + +Where a fill level is known, the element that applies at it answers, +and a boundary two elements share belongs to the upper one. +Where FlexMeasures takes one number for a whole schedule, the most cautious element answers: +the one converting drawn power into stored energy least, or the one leaking most. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest +from s2python.common import Commodity, CommodityQuantity, NumberRange, PowerRange +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, +) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + element_for_fill_level, + operation_mode_conversion_efficiency, + power_to_fill_rate_with_metrics, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +POWER_SENSOR = 1 +CHARGING_EFFICIENCY_SENSOR = 7 +RANGE_TOP = 80.0 +KWH_PER_SECOND_IN_WATTS = 3.6e6 + + +def _element( + lowest: float, + highest: float, + power: float, + heat_in_watts: float, +) -> FRBCOperationModeElement: + """A stretch of fill level over which a mode runs from nothing up to a power, delivering up to some heat.""" + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=0, end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS + ), + power_ranges=[ + PowerRange( + start_of_range=0, + end_of_range=power, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +#: A heat pump that derates as its buffer warms: 2000 W at a COP of 4 below 40, 1200 W at a COP of 3 above. +COLD = _element(0, 40, power=2000.0, heat_in_watts=8000.0) +WARM = _element(40, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + + +def _mode(*elements: FRBCOperationModeElement, label: str = "heat") -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _range(element) -> tuple[float, float]: + return ( + element.fill_level_range.start_of_range, + element.fill_level_range.end_of_range, + ) + + +# --------------------------------------------------------------------------- +# The selector itself +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fill_level", [-5.0, 0.0, 20.0, 40.0, 60.0, 80.0, 95.0]) +def test_the_element_for_a_fill_level_does_not_depend_on_the_order_listed(fill_level): + assert _range(element_for_fill_level([COLD, WARM], fill_level)) == _range( + element_for_fill_level([WARM, COLD], fill_level) + ) + + +@pytest.mark.parametrize( + "fill_level, expected", + [ + (0.0, (0, 40)), + (39.9, (0, 40)), + # A boundary two elements share belongs to the upper one, + (40.0, (40, RANGE_TOP)), + # except at the very top, where there is no upper one. + (RANGE_TOP, (40, RANGE_TOP)), + # Outside the range, the nearest element answers: a storage may be outside its declared range. + (-5.0, (0, 40)), + (95.0, (40, RANGE_TOP)), + ], +) +def test_the_element_that_applies_at_a_fill_level(fill_level, expected): + for listed in ([COLD, WARM], [WARM, COLD]): + assert _range(element_for_fill_level(listed, fill_level)) == expected + + +def test_an_element_that_is_a_single_point_applies_at_that_point(): + """S2 accepts an element whose range is one fill level, as long as the elements stay contiguous. + + At the top of a buffer, a heat pump may run at its minimum only, and say so with such an element. + It is the upper of the two at their shared boundary, so it applies there. + """ + below = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ([below, at_the_top], [at_the_top, below]): + assert element_for_fill_level(listed, RANGE_TOP) is at_the_top + assert element_for_fill_level(listed, RANGE_TOP - 1) is below + + +def test_past_either_end_the_answer_is_that_at_the_end(): + """A storage outside its declared range is answered as at the nearest end, so the answer does not jump there.""" + at_the_bottom = _element(0, 0, power=0.0, heat_in_watts=0.0) + middle = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ( + [at_the_bottom, middle, at_the_top], + [at_the_top, middle, at_the_bottom], + ): + assert element_for_fill_level(listed, RANGE_TOP + 15) is at_the_top + assert element_for_fill_level(listed, -5.0) is element_for_fill_level( + listed, 0.0 + ) + # At the bottom, the point element shares its only fill level with the element above it, which is upper. + assert element_for_fill_level(listed, 0.0) is middle + + +def test_a_fill_level_in_a_gap_is_answered_by_the_nearer_element_and_on_a_tie_the_lower(): + """S2 rules gaps out, but the selector takes raw elements and should not depend on their order there either.""" + low = _element(0, 30, power=2000.0, heat_in_watts=8000.0) + high = _element(50, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + for listed in ([low, high], [high, low]): + assert element_for_fill_level(listed, 35.0) is low + assert element_for_fill_level(listed, 45.0) is high + assert element_for_fill_level(listed, 40.0) is low + + +def test_leakage_elements_are_selected_the_same_way(): + low = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=1e-5 + ) + high = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=RANGE_TOP), + leakage_rate=3e-5, + ) + assert element_for_fill_level([high, low], 10.0) is low + assert element_for_fill_level([low, high], 40.0) is high + + +# --------------------------------------------------------------------------- +# Where a fill level is known: the measured power, and the schedule translation +# --------------------------------------------------------------------------- + + +def _system_description(*modes: FRBCOperationMode) -> FRBCSystemDescription: + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=list(modes), + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=False, + provides_fill_level_target_profile=False, + provides_usage_forecast=False, + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + ), + ) + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=POWER_SENSOR, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + # What the power sensor means by a sign is not the question here. + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +async def _measured_power(frbc: FRBCSimple, mode: FRBCOperationMode, factor: float): + actuator = list(frbc._system_description_history.values())[-1].actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode.id, + operation_mode_factor=factor, + ) + ) + posts = [ + call + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] == POWER_SENSOR + ] + assert len(posts) == 1, posts + return posts[0].kwargs["values"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("listed", [(COLD, WARM), (WARM, COLD)]) +async def test_the_measured_power_is_that_of_the_element_at_the_present_fill_level( + listed, +): + """A warm buffer runs on the derated element, so full tilt there is 1200 W, not 2000 W.""" + mode = _mode(*listed) + frbc = _frbc(_system_description(mode)) + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=70.0 + ) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_without_a_fill_level_the_measured_power_is_a_guess_said_out_loud(caplog): + """Before any storage status, the bottom element answers, and the log says it was a guess.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any( + "no FRBC.StorageStatus" in record.getMessage() for record in caplog.records + ), caplog.records + + +@pytest.mark.parametrize("fill_level", [20.0, 40.0, 60.0]) +def test_a_schedule_is_translated_the_same_whichever_order_the_elements_come_in( + fill_level, +): + forward = power_to_fill_rate_with_metrics(_mode(COLD, WARM), 1000.0, fill_level) + backward = power_to_fill_rate_with_metrics(_mode(WARM, COLD), 1000.0, fill_level) + assert forward[:4] == pytest.approx(backward[:4]) + assert _range(forward[4]) == _range(backward[4]) + + +# --------------------------------------------------------------------------- +# Where one number covers a whole schedule: the charging and the storage efficiency +# --------------------------------------------------------------------------- + + +def _efficiency(mode: FRBCOperationMode) -> float | None: + return operation_mode_conversion_efficiency( + mode, fill_level_scale=1, energy_unit="kWh" + ) + + +def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(): + assert _efficiency(_mode(COLD, WARM)) == pytest.approx( + _efficiency(_mode(WARM, COLD)) + ) + + +def test_a_mode_whose_elements_convert_differently_states_the_lowest(): + """A COP of 4 when cold and 3 when warm plans as 3: a plan that converts less reaches its target early.""" + assert _efficiency(_mode(COLD, WARM)) == pytest.approx(3.0) + + +@pytest.mark.parametrize( + "heat_in_watts", + [0.0, -50.0], + ids=["standby, drawing power and delivering nothing", "circulation, drawing power and losing heat"], +) +def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_watts): + """Drawing power while the storage does not fill is not a conversion one ratio can state.""" + standby = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange( + start_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + ), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + assert _efficiency(_mode(standby)) is None + + +@pytest.mark.asyncio +async def test_a_standby_mode_does_not_drag_the_recorded_efficiency_to_zero(): + """Off, standby at 5 W, and on at a COP of 4.3: the scheduler is told 4.3, not 0. + + At 0, the stock could not rise at any power, so the cheapest plan would be never to run the heat pump, + and with the fill-level band priced rather than hard, that plan would be feasible rather than refused. + """ + off = _mode(_element(0, RANGE_TOP, power=0.0, heat_in_watts=0.0), label="off") + standby_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange(start_of_range=0, end_of_range=0), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + standby = _mode(standby_element, label="standby") + on = _mode(_element(0, RANGE_TOP, power=883.7, heat_in_watts=3800.0), label="on") + system_description = _system_description(off, standby, on) + frbc = _frbc( + system_description, charging_efficiency_sensor_id=CHARGING_EFFICIENCY_SENSOR + ) + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + assert await frbc._record_charging_efficiency(system_description, start) is True + recorded = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] + == CHARGING_EFFICIENCY_SENSOR + ] + assert recorded == [[pytest.approx(3800.0 / 883.7)]] + + +def _leakage(*elements: FRBCLeakageBehaviourElement) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=list(elements), + ) + + +LEAKS_LITTLE = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=3e-5 +) +LEAKS_MOST = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=60), leakage_rate=9e-5 +) +LEAKS_SOME = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=60, end_of_range=RANGE_TOP), + leakage_rate=6e-5, +) + + +# The storage efficiencies below are compared exactly, not approximately. +# They are the same arithmetic on the same numbers, so equal means equal, +# and a leakage rate per second becomes an efficiency so close to 100% that pytest.approx's default tolerance +# would call two different leakage rates the same, and pass with the defect these tests are about. + + +def test_the_storage_efficiency_does_not_depend_on_the_order_listed(): + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), resolution=timedelta(minutes=15) + ) + assert efficiency == leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_SOME, LEAKS_LITTLE, LEAKS_MOST), + resolution=timedelta(minutes=15), + ) + + +def test_the_storage_efficiency_states_the_leakiest_element(): + """The leakiest element is not always the top one, so it is looked for rather than assumed to come last.""" + leakiest = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_MOST), resolution=timedelta(minutes=15), fill_level_scale=1 + ) + # Alone, the leakiest element is measured against its own top; together, against the range's top. + together = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), + resolution=timedelta(minutes=15), + ) + top_only = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_MOST.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert together == top_only + assert leakiest != top_only From d0b1c1dc33612a8af650298900da1259af0050c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:44:03 +0100 Subject: [PATCH 08/59] docs(changelog): the element that applies, and what a standby mode states Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 1bcc43a4..4c06c478 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -19,7 +19,7 @@ Added * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] * ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged [`#253 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes, or the elements of one mode, state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] @@ -41,6 +41,7 @@ Changed Fixed ----- +* A leakage behaviour is recorded as a storage efficiency at the rate of its leakiest element, where it took the last one listed, which S2 does not require to be the leakiest or even the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] @@ -64,7 +65,7 @@ Fixed * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] * The instruction-status history is keyed by a string, as its type declares, instead of by the ``UUID`` that a no-op ``cast`` let through, which made every lookup miss [`#237 `__] -* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the mode's power range, and posts it to the power sensor [`#236 `__] +* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the power range of the element that applies at the latest fill level, and posts it to the power sensor [`#236 `__, `#256 `__] * A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] Added From b572a86f0c6a806a2c8661b1adee781c53d6ee0a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:07:49 +0100 Subject: [PATCH 09/59] fix(FRBC): plan with the element where the schedule starts, not the lowest An adversarial review showed that the lowest efficiency over a mode's elements is not the cautious choice the previous commit called it. In FlexMeasures' linear program a charging efficiency scales what the actuator adds to the stock, so understating it protects the lower band only: the plan overshoots the upper band, buys more than it uses, and against a usage forecast the plan cannot relax, it can turn a problem the actuator can meet into one the scheduler finds infeasible. A heat pump at a COP of 4 cold and 2 warm, planned at 2, empties its planned buffer in a few hours while the real one keeps up. So within a mode, the element that applies at the fill level the schedule starts from answers. That is also the part of a schedule that runs before the next storage status asks for a new one, and it is what the actuator really does there. The storage efficiency likewise takes the leakage element at the latest fill level, and the leakiest only where no fill level is known yet. Across modes, the lowest still goes out, as #253 decided; the same argument applies to it, and that is left as a question rather than changed here. Also from the review: - A fill level that is not a number, which s2python lets through, counts as no fill level, and the selector refuses one, where it used to pick the bottom element silently. - The selector's docstring now says what is true: s2python checks contiguity for an operation mode only, never for a leakage behaviour; a single-point range, which the standard rules out, applies only at the top; and of two elements with the same range, the first listed answers. - Tests now cover the latest of several storage statuses, a fill-level scale other than 1 where scaling would pick another element, no warning for a mode with one element, and a schedule starting in the upper element. Each was a production mutation that passed the suite before. Formatted with black as well, which the previous commit was not: the pre-commit hooks were never installed in this repository, so they did not run on commit, and CI's check caught it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- .../s2/control_types/FRBC/frbc_simple.py | 30 +++- .../s2/control_types/FRBC/utils.py | 68 +++++--- .../s2/control_types/translations.py | 18 +- tests/s2/test_frbc_charging_efficiency.py | 49 +++++- tests/s2/test_frbc_element_selection.py | 154 ++++++++++++++++-- 6 files changed, 262 insertions(+), 61 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 4c06c478..c90d1871 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -19,7 +19,7 @@ Added * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] * ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes, or the elements of one mode, state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] @@ -41,7 +41,7 @@ Changed Fixed ----- -* A leakage behaviour is recorded as a storage efficiency at the rate of its leakiest element, where it took the last one listed, which S2 does not require to be the leakiest or even the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] +* A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a62f3c8e..c29ad86e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -4,6 +4,7 @@ """ import asyncio +import math from datetime import datetime, timedelta from zoneinfo import ZoneInfo @@ -395,6 +396,13 @@ def _in_the_power_unit(self, power: float) -> float: return power return convert_units([power], S2_POWER_UNIT, self.power_unit)[0] + def _latest_fill_level(self) -> float | None: + """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" + if not self._storage_status_history: + return None + fill_level = list(self._storage_status_history.values())[-1].present_fill_level + return None if math.isnan(fill_level) else fill_level + async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor sd: FRBCSystemDescription = list(self._system_description_history.values())[-1] @@ -431,17 +439,15 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. - if self._storage_status_history: - fill_level = list(self._storage_status_history.values())[ - -1 - ].present_fill_level + fill_level = self._latest_fill_level() + if fill_level is not None: element = element_for_fill_level(operation_mode.elements, fill_level) else: # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. element = element_for_fill_level(operation_mode.elements, float("-inf")) if len(operation_mode.elements) > 1: self._logger.warning( - "FRBC.ActuatorStatus for operation mode %s arrived with no FRBC.StorageStatus yet, " + "FRBC.ActuatorStatus for operation mode %s arrived with no usable FRBC.StorageStatus yet, " "so its power is recorded as the element at the bottom of the fill-level range states it, " "which need not be the element the actuator runs on.", om_label(operation_mode), @@ -534,7 +540,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): ) async def _record_charging_efficiency( - self, system_description: FRBCSystemDescription, start: datetime + self, + system_description: FRBCSystemDescription, + start: datetime, + fill_level: float, ) -> bool: """Record what the actuator converts drawn power into stored energy at, over the window a schedule covers. @@ -544,6 +553,8 @@ async def _record_charging_efficiency( a schedule planned against a sensor holding nothing for its window is planned as if the actuator converted power one for one. :param start: where the schedule starts, which is where the value is recorded from. + :param fill_level: the fill level the schedule starts from, in the resource manager's own units, + which picks the element of each mode that answers. :returns: whether that schedule may be planned against the sensor, which it may not where nothing is recorded on it for this window. """ @@ -556,7 +567,7 @@ async def _record_charging_efficiency( try: for operation_mode in actuator.operation_modes: efficiency = operation_mode_conversion_efficiency( - operation_mode, self._fill_level_scale, self.energy_unit + operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) if efficiency is not None: efficiencies[operation_mode.id] = efficiency @@ -829,7 +840,9 @@ async def trigger_schedule( # What the operation modes say about converting drawn power into stored energy. # Recorded first, since FlexMeasures reads it from the sensor named here and not from the flex model, # and named only where it was recorded. - if await self._record_charging_efficiency(system_description, start): + if await self._record_charging_efficiency( + system_description, start, fill_level_at_start + ): flex_model["charging-efficiency"] = { "sensor": self._charging_efficiency_sensor_id } @@ -1007,6 +1020,7 @@ async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): message=leakage_behaviour, resolution=self._resolution, fill_level_scale=self._fill_level_scale, + fill_level=self._latest_fill_level(), ) now = self.now() diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index d6f78386..646b36bc 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,4 +1,5 @@ import logging +import math from datetime import timedelta from math import isclose from typing import List, Sequence, TypeVar @@ -318,9 +319,12 @@ def fill_rate_in_watts(fill_rate: float, energy_unit: str) -> float: def operation_mode_conversion_efficiency( - operation_mode: FRBCOperationMode, fill_level_scale: float, energy_unit: str + operation_mode: FRBCOperationMode, + fill_level_scale: float, + energy_unit: str, + fill_level: float, ) -> float | None: - """How much energy an operation mode puts into the storage for each unit of power it draws. + """How much energy an operation mode puts into the storage for each unit of power it draws, at a fill level. An operation mode states a fill rate and a power range, so the ratio between the two is what the mode converts drawn power into stored energy at: @@ -328,38 +332,38 @@ def operation_mode_conversion_efficiency( Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, and for a thermal actuator that ratio is well above one. - Each element's ratio is taken at the top of its range, rather than as the slope over that range. + A mode may convert differently at different fill levels, as a heat pump does when its buffer warms, + and FlexMeasures takes one charging efficiency for a whole schedule. + The element that applies at the fill level the schedule starts from answers, + since the start of a schedule is the part that runs before the next storage status asks for a new one. + Not the lowest element over the whole range: + that overstates what the actuator needs to fill the storage, so the plan overshoots its upper band and buys more than it uses, + and against a usage the plan cannot relax it can turn a problem the actuator can meet into one the scheduler finds infeasible. + + The ratio is taken at the top of that element's range, rather than as the slope over it. An actuator may well declare degenerate ranges, where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. - A mode whose elements convert differently, as a heat pump does when its buffer warms, states the lowest of them. - FlexMeasures takes one charging efficiency for a whole schedule, and does not know which fill levels it will pass through, - so the most cautious element answers, for the same reason the lowest mode answers across modes: - a plan that converts less than the actuator really does reaches its target early, where one that converts more reaches it late. - That also makes the answer independent of the order the elements were listed in, which S2 does not fix. - :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :returns: the efficiency, or None where no element of the mode converts drawn power into stored energy. - An element converts nothing where it draws no power, as an idle mode does, + :param fill_level: where the schedule starts, in the resource manager's own units, as the elements' ranges are. + :returns: the efficiency, or None where the element that applies converts no drawn power into stored energy. + It converts nothing where it draws no power, as an idle mode does, where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, or where it draws power while the storage does not fill, as a standby or circulation mode does: no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. """ - efficiencies = [] - # S2 gives every operation mode at least one element, and every element at least one power range. - for element in operation_mode.elements: - # todo: an element states one power range per commodity quantity, - # and no distinction is made between them here, - # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). - power = element.power_ranges[0].end_of_range - fill_rate = element.fill_rate.end_of_range * fill_level_scale - if power <= 0 or fill_rate <= 0: - continue - efficiencies.append(fill_rate_in_watts(fill_rate, energy_unit) / power) - return min(efficiencies) if efficiencies else None + element = element_for_fill_level(operation_mode.elements, fill_level) + # todo: an element states one power range per commodity quantity, + # and no distinction is made between them here, + # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). + power = element.power_ranges[0].end_of_range + fill_rate = element.fill_rate.end_of_range * fill_level_scale + if power <= 0 or fill_rate <= 0: + return None + return fill_rate_in_watts(fill_rate, energy_unit) / power def clip_fill_level_target_profile( @@ -472,15 +476,25 @@ def element_for_fill_level( ) -> ElementWithFillLevelRange: """The element that applies at a fill level, whatever order the elements were listed in. - S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, - and s2python checks that they are contiguous on a sorted copy, so the list itself may arrive in any order. + S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, in any order. + s2python checks that an operation mode's elements are contiguous, on a sorted copy, and leaves the list as it was sent; + it checks nothing of the kind for a leakage behaviour, whose elements may leave gaps or overlap. + Two neighbouring elements share a boundary, which belongs to the upper of the two: of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. - That includes an element whose range is a single point at the top, which no half-open rule could ever select. + A range that is a single point therefore applies only at the very top. + The standard says a range starts below where it ends, so such a range is outside it, though s2python lets it through. + Two elements with the same range are ambiguous, which overlapping elements always are, and the first listed answers. + A fill level outside the range the elements cover is answered as the nearest end of that range is, since a storage may be outside its declared range, and the answer should not jump as it crosses that end. - Elements with a gap between them, which S2 rules out, answer a fill level in the gap with the nearer, and on a tie the lower. + A fill level in a gap between elements is answered by the nearer, and on a tie the lower. + + :raises ValueError: for a fill level that is not a number, which no element applies at. """ + if math.isnan(fill_level): + raise ValueError("No element applies at a fill level that is not a number.") + def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: return e.fill_level_range.start_of_range, e.fill_level_range.end_of_range diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index 8b02b15e..a4cd5ecc 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -17,11 +17,14 @@ "Install it using `pip install flexmeasures-client[s2]`." ) +from flexmeasures_client.s2.control_types.FRBC.utils import element_for_fill_level + def leakage_behaviour_to_storage_efficiency( message: FRBCLeakageBehaviour, resolution=timedelta(minutes=15), fill_level_scale: float = 1, + fill_level: float | None = None, ) -> float: """ Convert a FRBC.LeakageBehaviour message into a FlexMeasures compatible storage efficiency. @@ -56,15 +59,18 @@ def leakage_behaviour_to_storage_efficiency( """ - # FlexMeasures takes one storage efficiency here, and does not know which fill levels a schedule will pass through, - # so the leakiest element answers, as the most cautious: a plan that expects to lose more reaches its target early. - # It is looked for rather than assumed to come last, since S2 does not fix the order the elements are listed in, - # and the leakiest is not always the top one. - leakiest = max(message.elements, key=lambda e: e.leakage_rate) + # FlexMeasures takes one storage efficiency here, for the whole window it covers. + # Where the fill level is known, the element that applies at it answers, as for a charging efficiency. + # Where it is not, the leakiest answers, looked for rather than assumed to come last, + # since S2 does not fix the order the elements are listed in, and the leakiest is not always the top one. + if fill_level is not None: + element = element_for_fill_level(message.elements, fill_level) + else: + element = max(message.elements, key=lambda e: e.leakage_rate) max_fill_level = max( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) - leakage = leakiest.leakage_rate * fill_level_scale + leakage = element.leakage_rate * fill_level_scale base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) # Discuss conversions diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 34b382c6..7217e9cd 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -326,8 +326,8 @@ def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): fill_rate_in_watts(1, "therm") -def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): - """A mode states an element per stretch of fill level, and the efficiency is the one at full tilt. +def test_an_efficiency_is_taken_at_the_top_of_the_element_that_applies(): + """A mode states an element per stretch of fill level, and the efficiency is that element's, at full tilt. The ratio, and not the slope over the element's range: a mode's ranges can be degenerate, where the actuator runs at one power and one fill rate only, @@ -353,6 +353,47 @@ def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): ) ) - efficiency = operation_mode_conversion_efficiency(mode, 1, "kWh") + below, above = ( + operation_mode_conversion_efficiency(mode, 1, "kWh", fill_level) + for fill_level in (RANGE_TOP / 2, 1.5 * RANGE_TOP) + ) + + assert below == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER) + assert above == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + + +@pytest.mark.asyncio +async def test_a_schedule_is_asked_for_against_the_efficiency_where_it_starts(): + """A heat pump that converts less once its buffer is warm is planned with the warm efficiency for a warm start.""" + cold = HEATER_HEAT_IN_WATTS / HEATER_POWER + warm = cold / 2 + mode = _operation_mode(HEATER_POWER, HEATER_FILL_RATE, "on") + mode.elements[0].fill_level_range.end_of_range = RANGE_TOP / 2 + mode.elements.append( + FRBCOperationModeElement( + fill_level_range=NumberRange( + start_of_range=RANGE_TOP / 2, end_of_range=RANGE_TOP + ), + fill_rate=NumberRange( + start_of_range=HEATER_FILL_RATE / 2, end_of_range=HEATER_FILL_RATE / 2 + ), + power_ranges=[ + PowerRange( + start_of_range=HEATER_POWER, + end_of_range=HEATER_POWER, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + ) + system_description = _system_description(mode) + frbc = _frbc(system_description) + # The latest status: a buffer warmed past the middle of its range since the one _frbc recorded. + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=0.75 * RANGE_TOP + ) + + await _asked_for(frbc, system_description) - assert efficiency == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + assert _recordings(frbc)[0]["values"] == [pytest.approx(warm)] + assert warm != pytest.approx(cold) diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 426b7d10..06ef021c 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -75,7 +75,9 @@ def _element( WARM = _element(40, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) -def _mode(*elements: FRBCOperationModeElement, label: str = "heat") -> FRBCOperationMode: +def _mode( + *elements: FRBCOperationModeElement, label: str = "heat" +) -> FRBCOperationMode: return FRBCOperationMode( id=get_unique_id(), diagnostic_label=label, @@ -164,7 +166,8 @@ def test_a_fill_level_in_a_gap_is_answered_by_the_nearer_element_and_on_a_tie_th def test_leakage_elements_are_selected_the_same_way(): low = FRBCLeakageBehaviourElement( - fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=1e-5 + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), + leakage_rate=1e-5, ) high = FRBCLeakageBehaviourElement( fill_level_range=NumberRange(start_of_range=40, end_of_range=RANGE_TOP), @@ -260,7 +263,8 @@ async def test_without_a_fill_level_the_measured_power_is_a_guess_said_out_loud( with caplog.at_level(logging.WARNING, logger="FRBCSimple"): assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) assert any( - "no FRBC.StorageStatus" in record.getMessage() for record in caplog.records + "no usable FRBC.StorageStatus" in record.getMessage() + for record in caplog.records ), caplog.records @@ -279,27 +283,48 @@ def test_a_schedule_is_translated_the_same_whichever_order_the_elements_come_in( # --------------------------------------------------------------------------- -def _efficiency(mode: FRBCOperationMode) -> float | None: +def _efficiency(mode: FRBCOperationMode, fill_level: float) -> float | None: return operation_mode_conversion_efficiency( - mode, fill_level_scale=1, energy_unit="kWh" + mode, fill_level_scale=1, energy_unit="kWh", fill_level=fill_level ) -def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(): - assert _efficiency(_mode(COLD, WARM)) == pytest.approx( - _efficiency(_mode(WARM, COLD)) +@pytest.mark.parametrize("fill_level", [-5.0, 20.0, 40.0, 60.0, 95.0]) +def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(fill_level): + assert _efficiency(_mode(COLD, WARM), fill_level) == _efficiency( + _mode(WARM, COLD), fill_level ) -def test_a_mode_whose_elements_convert_differently_states_the_lowest(): - """A COP of 4 when cold and 3 when warm plans as 3: a plan that converts less reaches its target early.""" - assert _efficiency(_mode(COLD, WARM)) == pytest.approx(3.0) +@pytest.mark.parametrize("fill_level, cop", [(20.0, 4.0), (60.0, 3.0)]) +def test_a_mode_states_the_efficiency_of_the_element_where_the_schedule_starts( + fill_level, cop +): + """A COP of 4 when cold and 3 when warm is 4 for a schedule starting cold, and 3 for one starting warm. + + Not the lowest of the two for both: that overstates what a cold buffer needs, + so the plan overshoots its upper band and buys more than it uses, + and against a usage it cannot relax it can find a problem the heat pump can meet infeasible. + The start is what matters most, since it runs before the next storage status asks for a new plan. + """ + assert _efficiency(_mode(COLD, WARM), fill_level) == pytest.approx(cop) + + +def test_an_element_that_tapers_off_applies_only_where_it_applies(): + """Near its highest flow temperature a heat pump may barely fill its buffer, and that does not hold for a cold one.""" + tapering = _element(50, 60, power=1200.0, heat_in_watts=10.0) + mode = _mode(_element(0, 50, power=1200.0, heat_in_watts=4800.0), tapering) + assert _efficiency(mode, 30.0) == pytest.approx(4.0) + assert _efficiency(mode, 55.0) == pytest.approx(10.0 / 1200.0) @pytest.mark.parametrize( "heat_in_watts", [0.0, -50.0], - ids=["standby, drawing power and delivering nothing", "circulation, drawing power and losing heat"], + ids=[ + "standby, drawing power and delivering nothing", + "circulation, drawing power and losing heat", + ], ) def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_watts): """Drawing power while the storage does not fill is not a conversion one ratio can state.""" @@ -317,7 +342,7 @@ def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_wa ) ], ) - assert _efficiency(_mode(standby)) is None + assert _efficiency(_mode(standby), 10.0) is None @pytest.mark.asyncio @@ -346,7 +371,9 @@ async def test_a_standby_mode_does_not_drag_the_recorded_efficiency_to_zero(): system_description, charging_efficiency_sensor_id=CHARGING_EFFICIENCY_SENSOR ) start = datetime(2024, 1, 2, tzinfo=timezone.utc) - assert await frbc._record_charging_efficiency(system_description, start) is True + assert ( + await frbc._record_charging_efficiency(system_description, start, 30.0) is True + ) recorded = [ call.kwargs["values"] for call in frbc._fm_client.post_sensor_data.await_args_list @@ -413,3 +440,102 @@ def test_the_storage_efficiency_states_the_leakiest_element(): ) assert together == top_only assert leakiest != top_only + + +# --------------------------------------------------------------------------- +# Which fill level: the latest, as the resource manager states it +# --------------------------------------------------------------------------- + + +def _report_fill_level(frbc: FRBCSimple, fill_level: float) -> None: + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=fill_level + ) + + +@pytest.mark.asyncio +async def test_the_measured_power_follows_the_latest_storage_status(): + """A buffer that has warmed since its first status runs on the derated element now.""" + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, 10.0) + _report_fill_level(frbc, 70.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_the_fill_level_is_compared_in_the_resource_managers_own_units(): + """The scale converts fill levels for FlexMeasures; the elements' ranges are still in the resource manager's units. + + At 7 in its own units the buffer is cold, though scaled by 10 it would read as 70, which is warm. + """ + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode), fill_level_scale=10) + _report_fill_level(frbc, 7.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + + +@pytest.mark.asyncio +async def test_with_one_element_there_is_nothing_to_guess(caplog): + mode = _mode(_element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0)) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + await _measured_power(frbc, mode, factor=1.0) + assert not caplog.records, caplog.records + + +@pytest.mark.asyncio +async def test_a_fill_level_that_is_not_a_number_counts_as_unknown(caplog): + """s2python lets a NaN fill level through, and no element applies at it, so it is treated as no fill level at all.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, float("nan")) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any("no usable FRBC.StorageStatus" in r.getMessage() for r in caplog.records) + + +def test_no_element_applies_at_a_fill_level_that_is_not_a_number(): + with pytest.raises(ValueError, match="not a number"): + element_for_fill_level([COLD, WARM], float("nan")) + + +def test_the_storage_efficiency_at_a_fill_level_is_that_elements(): + """Where the fill level is known, the leakage element that applies there answers, not the leakiest.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + at_20 = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=20.0 + ) + only_little = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_LITTLE.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert at_20 == only_little + assert at_20 != leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15) + ) + + +@pytest.mark.asyncio +async def test_the_handler_records_leakage_at_the_latest_fill_level(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + _report_fill_level(frbc, 20.0) + await frbc.send_leakage_behaviour(leakage) + posted = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == [ + [ + leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=20.0 + ) + ] + ] From 9587c94f213fe9574e82c27774518286a1569346 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:17 +0100 Subject: [PATCH 10/59] fix(TUNES): an operation mode without a label no longer stops planning S2 makes an operation mode's diagnostic label optional, and FillRateBasedControlTUNES tells its modes apart by looking for "THP" or "NES" inside it. On a mode without one that raised a TypeError, so a single unlabelled mode stopped both planning and the recording of every mode's conversion efficiency. A missing label now simply matches neither. Found by type checking against s2python's own types, which also asked for two variables that held a range and then a number, or a message and then a series, to be given a name for each. New tests cover the planning path, the efficiency path, and posting a usage forecast, none of which any test reached before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_tunes.py | 20 ++--- tests/s2/test_frbc_tunes.py | 75 +++++++++++++++++++ 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 4dae13ca..48ce8eb7 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -278,10 +278,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): active_operation_mode_fill_rate_sensor_id = self._nes_fill_rate_sensor_id # Operation Mode Factor to fill rate - fill_rate = operation_mode.elements[0].fill_rate + fill_rate_range = operation_mode.elements[0].fill_rate fill_rate = ( - fill_rate.start_of_range - + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + fill_rate_range.start_of_range + + (fill_rate_range.end_of_range - fill_rate_range.start_of_range) * factor ) * self._fill_level_scale # Send data to the sensor of the fill rate corresponding to the active operation mode @@ -331,10 +331,10 @@ async def trigger_schedule(self): # type: ignore[override] operation_mode = None efficiency_sensor_id = None for _operation_mode in actuator.operation_modes: - if "THP" in _operation_mode.diagnostic_label: + if "THP" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._thp_efficiency_sensor_id - elif "NES" in _operation_mode.diagnostic_label: + elif "NES" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._nes_efficiency_sensor_id else: @@ -607,9 +607,9 @@ async def send_conversion_efficiencies( ) for operation_mode in actuator.operation_modes: - if "THP" in operation_mode.diagnostic_label: + if "THP" in (operation_mode.diagnostic_label or ""): sensor_id = self._thp_efficiency_sensor_id - elif "NES" in operation_mode.diagnostic_label: + elif "NES" in (operation_mode.diagnostic_label or ""): sensor_id = self._nes_efficiency_sensor_id else: continue @@ -679,7 +679,7 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): minute=(start_time.minute // 15) * 15, second=0, microsecond=0 ) - usage_forecast = translate_usage_forecast_to_fm( + usage_forecast_series = translate_usage_forecast_to_fm( usage_forecast, RESOLUTION, strategy="mean", @@ -688,14 +688,14 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): # Scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) scale = timedelta(minutes=15) / timedelta(seconds=1) - scaled_usage_forecast = usage_forecast * scale + scaled_usage_forecast = usage_forecast_series * scale await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), unit=POWER_UNIT, # e.g. [0, 100] MW/(15 min) - duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), + duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) async def send_fill_level_target_profile( diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 4086b25b..0435ef37 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock import numpy as np +import pandas as pd import pytest import pytest_asyncio from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues @@ -342,3 +343,77 @@ async def test_a_leakage_behaviour_is_recorded_rather_than_refused( ) ] assert posts[0].kwargs["unit"] == "%" + + +@pytest.mark.asyncio +async def test_an_operation_mode_without_a_label_does_not_stop_tunes( + cem_in_frbc_control_type, +): + """S2 makes a diagnostic label optional, and TUNES tells its modes apart by label. + + It looked for "THP" and "NES" inside a label that may be None, which raised a TypeError, + so one unlabelled mode stopped the conversion efficiencies of every mode from being recorded. + """ + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + unlabelled.actuators[0].operation_modes[-1].diagnostic_label = None + fm_client.post_sensor_data.reset_mock() + + await frbc.send_conversion_efficiencies(unlabelled) + + assert fm_client.post_sensor_data.await_count > 0 + + +@pytest.mark.asyncio +async def test_planning_does_not_stop_on_an_unlabelled_operation_mode( + cem_in_frbc_control_type, +): + """The same optional label, where TUNES picks the mode to plan with.""" + from s2python.frbc import FRBCStorageStatus + + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + for operation_mode in unlabelled.actuators[0].operation_modes: + operation_mode.diagnostic_label = None + frbc._system_description_history[str(unlabelled.message_id)] = unlabelled + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + await frbc.trigger_schedule() + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_type): + """S2 states usage per second; TUNES posts it per quarter hour, on a quarter-hour grid, for as long as it was forecast.""" + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=2 * 3600 * 1000), + usage_rate_expected=0.001, + ) + ], + ) + ) + + posts = [ + call + for call in fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 12 + ] + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + values = posts[0].kwargs["values"] + assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) + assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) From 4585f2f60647b9f5f030fecf3b007da524924770 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:26 +0100 Subject: [PATCH 11/59] fix(client): drop a content-type check that could not fire, nor be printed request() ran check_content_type after its retry loop. It could never raise there: request_once parses every answer with aiohttp's json(), which refuses any content type but JSON, and the loop already reports that refusal as a ConnectionError, like every other failure to reach the API. Had it fired, it would have done harm. It built aiohttp's ContentTypeError from a message and a dict, where aiohttp expects the request's info and history, so anything that formatted the error, a log line included, raised an AttributeError of its own; and it read the body with response.text() without awaiting it. Its only test passed a mock whose text() was not a coroutine, and never formatted the error. The check is gone. The test that replaces it states where a non-JSON answer, such as a proxy's error page during a restart, is actually handled. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 11 +++---- src/flexmeasures_client/response_handling.py | 14 --------- tests/client/test_init.py | 33 ++++++++++++-------- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 14de2cc4..f8f3d232 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -34,11 +34,7 @@ WrongAPIVersionError, WrongHostError, ) -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - check_response, -) +from flexmeasures_client.response_handling import check_for_status, check_response from flexmeasures_client.utils import apply_deprecated_parameter_aliases, merge_updates LOGGER = logging.getLogger(__name__) @@ -463,8 +459,9 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - check_content_type(response) - + # A body that is not JSON never gets here: + # request_once parses every answer with aiohttp's json(), which refuses any other content type, + # and the loop above reports that as a ConnectionError like any other failure to reach the API. return await response.json(), response.status async def request_once( diff --git a/src/flexmeasures_client/response_handling.py b/src/flexmeasures_client/response_handling.py index 70d1c4d5..1f22ae3f 100644 --- a/src/flexmeasures_client/response_handling.py +++ b/src/flexmeasures_client/response_handling.py @@ -8,11 +8,8 @@ from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING, Callable -from aiohttp import ContentTypeError from yarl import URL -from flexmeasures_client.constants import CONTENT_TYPE - if TYPE_CHECKING: # Only imports the below statements during type checking from flexmeasures_client.client import FlexMeasuresClient @@ -163,17 +160,6 @@ async def check_response( return polling_step, reauth_once, URL(url) -def check_content_type(response): - """Check if response is in format application/json""" - content_type = response.headers.get("Content-Type", "") - if CONTENT_TYPE not in content_type: - text = response.text() - raise ContentTypeError( - "Unexpected content type response from the API", - {"Content-Type": content_type, "response": text}, - ) - - def check_for_status(status, expected_status: int | Iterable[int]): """Check if status is an acceptable 2xx status code. diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 52a02af7..73e795cb 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -21,11 +21,7 @@ _parse_sensor_json_fields, ) from flexmeasures_client.constants import CLIENT_VERSION_HEADER -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - parse_retry_after, -) +from flexmeasures_client.response_handling import check_for_status, parse_retry_after @pytest.mark.parametrize( @@ -837,15 +833,26 @@ def test_parse_sensor_json_fields(): assert sensor["attributes"]["key"] == "value" -def test_check_content_type_failure(): - """check_content_type raises on non-JSON content type.""" - from aiohttp import ContentTypeError as AiohttpContentTypeError +@pytest.mark.asyncio +async def test_an_answer_that_is_not_json_is_a_connection_error() -> None: + """A proxy's HTML error page, as during a server restart, is reported like every other failure to reach the API. - response = MagicMock() - response.headers = {"Content-Type": "text/html"} - response.text.return_value = "some html" - with pytest.raises(AiohttpContentTypeError): - check_content_type(response) + aiohttp's own json() refuses it inside the request loop, which is where this is handled, + so the separate content-type check that used to follow the loop could never fire, and was removed. + """ + with aioresponses() as m: + m.get( + "http://localhost:5000/api/", + status=200, + body="502 Bad Gateway", + content_type="text/html", + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ConnectionError, match="text/html"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() def test_check_for_status_failure(): From c341c4af45270f690e5c3dd40672f13f0dbc5a97 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 13:59:38 +0100 Subject: [PATCH 12/59] chore(types): type check against what the code imports, S2 messages included mypy ran with --follow-imports skip, so s2python, pydantic, pandas and aiohttp were Any to it, and a misspelt attribute on an S2 message passed. It now follows imports, with pydantic's plugin, since S2 messages are pydantic models, and the pre-commit hook installs the s2 extra so those types are there to check. The plugin is also the guard against sliding back: without the extra it fails to load, rather than leaving every S2 type silently Any. A misspelt message_id on a leakage behaviour is now an error naming it. What it found, besides the two fixes in the previous commits: - get_validation_error_summary iterated a pydantic ValidationError, which is not iterable, so it raised a TypeError on any error it was given. Nothing called it, so it is gone rather than repaired. - Revocations were recorded by the RevokeObject's raw id, while every other id a handler keeps is text, so a lookup by the text of the id would have missed. They are recorded as text now. Nothing calls is_revoked yet, which means a revocation is acknowledged and then never consulted; that is recorded as a finding for later rather than changed here. - The rest is annotation: a message that arrives as a model, a dict or text is normalised to a dict once and checked as one; get_message returns the dict it always returned rather than claiming a str; the message type of a registered schema is read through model_fields rather than the deprecated __fields__; an operation mode's id is text where a label is wanted; and the actuator's power ranges are pairs rather than tuples of any length. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .pre-commit-config.yaml | 3 +- pyproject.toml | 9 ++++-- src/flexmeasures_client/s2/__init__.py | 10 +++---- src/flexmeasures_client/s2/cem.py | 11 +++++--- .../s2/control_types/FRBC/frbc_simple.py | 7 +++-- .../s2/control_types/FRBC/utils.py | 2 +- .../s2/script/websockets_client.py | 6 ++-- src/flexmeasures_client/s2/utils.py | 19 ++++++------- tests/s2/test_frbc_simple_planning.py | 28 +++++++++++++++++++ 9 files changed, 66 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a53e82e7..99b3eef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,8 @@ repos: name: mypy (static typing) pass_filenames: false language: system - entry: uv run poe type-check + # With the s2 extra, so that the S2 message types are there to check against. + entry: uv run --extra s2 poe type-check verbose: true - repo: local diff --git a/pyproject.toml b/pyproject.toml index 8f1552e4..251b1d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,8 +89,13 @@ help = "Run S2 tests and generate coverage report." cmd = "pytest tests/s2 --cov --cov-report=" [tool.poe.tasks.type-check] -help = "Run mypy on every source file, including the bodies of functions without type hints." -cmd = "mypy --follow-imports skip --ignore-missing-imports --check-untyped-defs src" +help = "Run mypy on every source file, including the bodies of functions without type hints, against the types of what they import." +cmd = "mypy --ignore-missing-imports --check-untyped-defs src" + +[tool.mypy] +# The S2 messages are pydantic models, so without the plugin mypy misreads their constructors and fields. +# It also makes a missing s2 extra fail loudly, rather than leaving every S2 type silently Any. +plugins = ["pydantic.mypy"] [tool.poe.tasks.lint] help = "Run pre-commit hooks on all files." diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 981269e1..a1fcadc5 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -6,7 +6,7 @@ from collections import deque from dataclasses import dataclass from datetime import datetime -from typing import Callable, Coroutine, Dict, Type +from typing import Callable, Dict, Type from zoneinfo import ZoneInfo import pydantic @@ -75,7 +75,7 @@ async def wrap(*args, **kwargs): ) return outgoing_message - message_type = schema.__fields__.get("message_type").default + message_type = schema.model_fields["message_type"].default setattr(wrap, "_tag", Tag(message_type)) return wrap @@ -176,7 +176,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: message_type = "" if isinstance(message, pydantic.BaseModel): - message_type = message.message_type + message_type = message.model_dump().get("message_type", "") elif isinstance(message, dict): message_type = message.get("message_type", "") elif isinstance(message, str): @@ -186,7 +186,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: async def handle_message( self, message: pydantic.BaseModel | str | Dict - ) -> Coroutine: + ) -> pydantic.BaseModel | None: """ Calls the handler linked to the message_type and converts the output to a serialized dict, i.e. it converts all the inner objects to Python @@ -243,7 +243,7 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: Stores the revoked object ID into the objects_revoked list """ - self.objects_revoked.append(message.object_id) + self.objects_revoked.append(str(message.object_id)) return ReceptionStatus( subject_message_id=str(message.message_id), status=ReceptionStatusValues.OK diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 7e83cd8b..72ef9b3d 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -176,13 +176,16 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): the control type handler and in the CEM, then the one defined in the control type prevails. """ - response = None + response: pydantic.BaseModel | None = None if isinstance(message, pydantic.BaseModel): message = json.loads(message.json()) if isinstance(message, str): message = json.loads(message) + assert isinstance( + message, dict + ) # what a message is from here on, whatever form it came in self._logger.debug(f"Received: {message}") @@ -224,11 +227,11 @@ def update_control_type(self, control_type: ControlType): """ self._control_type = control_type - async def get_message(self) -> str: + async def get_message(self) -> dict: """Call this function to get the messages to be sent to the RM Returns: - str: message in JSON format + dict: the message, as JSON-serialisable Python objects """ message = await self._sending_queue.get() @@ -412,7 +415,7 @@ def handle_revoke_object(self, message: RevokeObject): for initials, control_type in control_types.items(): if initials in str(message.object_type): self._control_types_handlers[control_type].revoke_message( - message.object_id + str(message.object_id) ) return get_reception_status(message, ReceptionStatusValues.OK) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index c29ad86e..131a8753 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -570,7 +570,7 @@ async def _record_charging_efficiency( operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) if efficiency is not None: - efficiencies[operation_mode.id] = efficiency + efficiencies[str(operation_mode.id)] = efficiency except NotImplementedError: self._logger.exception( "Not asking for a schedule against a charging efficiency: a fill rate in %s cannot be compared against the watts S2 states a power range in.", @@ -773,7 +773,10 @@ async def trigger_schedule( # TODO: an element states one power range per commodity quantity, and no distinction is made between them here, # as the schedule translation does not make one either (see power_to_fill_rate_with_metrics). power_ranges = [ - tuple(sorted((power_range.start_of_range, power_range.end_of_range))) + ( + min(power_range.start_of_range, power_range.end_of_range), + max(power_range.start_of_range, power_range.end_of_range), + ) for operation_mode in actuator.operation_modes for element in operation_mode.elements for power_range in element.power_ranges diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 646b36bc..35e652a2 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -511,7 +511,7 @@ def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: def om_label(operation_mode: FRBCOperationMode) -> str: label = getattr(operation_mode, "diagnostic_label", None) - return label if isinstance(label, str) else operation_mode.id + return label if isinstance(label, str) else str(operation_mode.id) def explain_choice( diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 510062fe..942f1988 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -111,7 +111,7 @@ async def main_s2(): roles=[ Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) ], - instruction_processing_delay=Duration(1), + instruction_processing_delay=Duration(root=1), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, @@ -141,13 +141,13 @@ async def main_s2(): print("RECEIVING: ", control_type) - message = ReceptionStatus( + reception_status = ReceptionStatus( subject_message_id=control_type["message_id"], status=ReceptionStatusValues.OK, ) print("SENDING: ReceptionStatus") - await ws.send_str(message.model_dump_json()) + await ws.send_str(reception_status.model_dump_json()) electric_power = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 50eabaab..cfdcd55e 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import OrderedDict -from typing import Mapping, TypeVar +from typing import Any, Mapping, Protocol, TypeVar from uuid import uuid4 import pydantic @@ -49,15 +49,6 @@ def get_unique_id() -> str: return str(uuid4()) -def get_validation_error_summary(error: pydantic.ValidationError) -> str: - error_summary = "" - - for i, e in enumerate(error): - error_summary += f"\nValidationEror {i} -> \t {e['msg']}" - - return error_summary[1:] # skipping the first \n - - def get_message_id(message: pydantic.BaseModel) -> str | None: """ This function returns the message_id if it is found in the message, @@ -71,8 +62,14 @@ def get_message_id(message: pydantic.BaseModel) -> str | None: return None +class _HasMessageId(Protocol): + """Any S2 message: every one of them carries a message id.""" + + message_id: Any + + def get_reception_status( - subject_message: pydantic.BaseModel, + subject_message: _HasMessageId, status: ReceptionStatusValues = ReceptionStatusValues.OK, ): """ diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 5d5e5c77..c4c740d4 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -217,3 +217,31 @@ async def test_planning_without_a_storage_status_is_logged_not_printed(capsys, c record.levelname == "WARNING" and "FRBC.StorageStatus" in record.getMessage() for record in caplog.records ), caplog.records + + +@pytest.mark.asyncio +async def test_a_revoked_object_is_known_by_its_id_as_text(): + """Revocations are recorded by the id as text, which is how every other id in a handler is kept.""" + from s2python.common import RevokableObjects, RevokeObject + + frbc = frbc_simple() + object_id = get_unique_id() + await frbc.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_recognises_a_message_in_any_form_it_accepts(): + """supports_message takes a model, a dict or JSON text, and reads the type from each the same way.""" + frbc = frbc_simple() + description = system_description("2022-12-02T00:00:00+00:00") + as_dict = description.model_dump(mode="json") + assert frbc.supports_message(description) + assert frbc.supports_message(as_dict) + assert frbc.supports_message(description.model_dump_json()) + assert not frbc.supports_message(as_dict | {"message_type": "Handshake"}) From 1928d56749db3ad87751ffe7564c47c9352ef12d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:11:28 +0100 Subject: [PATCH 13/59] docs(changelog): an unlabelled TUNES mode, and two helpers that never worked Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ CHANGELOG.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c90d1871..51ef1771 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,6 +41,8 @@ Changed Fixed ----- +* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, and one unlabelled mode stopped both [`#257 `__] +* ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] * ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e030e46c..7b7fa787 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,11 @@ Changed * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] +Removed +------- + +* ``response_handling.check_content_type``, which could never fire where it was called, since every answer is already parsed as JSON before it, and which raised an ``AttributeError`` of its own when its error was printed. A non-JSON answer is reported as a ``ConnectionError``, as before [`#257 `__] + Fixed ----- From 2a644e6d85a9bec39d9a482c87ed0acf3b214159 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:37:09 +0100 Subject: [PATCH 14/59] fix: correct what this branch claimed, and test what review found untested An adversarial review ran every claim on this branch, and three were false: - The removed content-type check could fire. aiohttp's json() accepts JSON variants such as application/problem+json and application/vnd.api+json, and the check refused those, with an error that could not be printed. Removing it is right, but it changes behaviour: such an answer is now read like any other. The comment at the removed call, the test's docstring and the changelog entry now say so, instead of calling it dead code. - "A non-JSON answer is reported as a ConnectionError, as before" was only true of a wrong content type. A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError, as it was before. - One unlabelled TUNES mode stopped planning, and the recording of the efficiencies of the modes listed after it, not of every mode. Two more overstatements live only in pushed commit messages and the PR body: following imports leaves pandas Any, since it ships no type information and no stubs are installed, and the efficiency path was already reached by a test. Tests now cover what five mutations showed was unpinned: a revocation taken by the CEM itself, with no control type active; a model of a type a handler does not handle; a usage forecast starting off the quarter hour; and one averaged within a quarter hour rather than taking its highest value. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- CHANGELOG.rst | 2 +- src/flexmeasures_client/client.py | 8 ++-- tests/client/test_init.py | 5 ++- tests/s2/test_frbc_simple_planning.py | 32 ++++++++++++++ tests/s2/test_frbc_tunes.py | 63 +++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 51ef1771..a8cb28a7 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,7 +41,7 @@ Changed Fixed ----- -* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, and one unlabelled mode stopped both [`#257 `__] +* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7b7fa787..05d31188 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,7 +32,7 @@ Changed Removed ------- -* ``response_handling.check_content_type``, which could never fire where it was called, since every answer is already parsed as JSON before it, and which raised an ``AttributeError`` of its own when its error was printed. A non-JSON answer is reported as a ``ConnectionError``, as before [`#257 `__] +* ``response_handling.check_content_type``, which refused an answer in a JSON variant that aiohttp parses, such as ``application/problem+json``, with an error that raised an ``AttributeError`` of its own when printed. Such an answer is now read like any other. A content type that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 `__] Fixed ----- diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index f8f3d232..698f5bc6 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -459,9 +459,11 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - # A body that is not JSON never gets here: - # request_once parses every answer with aiohttp's json(), which refuses any other content type, - # and the loop above reports that as a ConnectionError like any other failure to reach the API. + # request_once has already parsed this answer with aiohttp's json(), + # which refuses a content type that is not JSON, and the loop above reports that as a ConnectionError. + # A check that used to follow here refused the JSON variants aiohttp accepts too, such as application/problem+json, + # with an error that could not even be printed, so it is gone. + # A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError. return await response.json(), response.status async def request_once( diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 73e795cb..a2b14436 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -837,8 +837,9 @@ def test_parse_sensor_json_fields(): async def test_an_answer_that_is_not_json_is_a_connection_error() -> None: """A proxy's HTML error page, as during a server restart, is reported like every other failure to reach the API. - aiohttp's own json() refuses it inside the request loop, which is where this is handled, - so the separate content-type check that used to follow the loop could never fire, and was removed. + aiohttp's own json() refuses it inside the request loop, which is where this is handled. + The separate content-type check that used to follow the loop is gone: + it could only fire on the JSON variants aiohttp accepts, such as application/problem+json, and it should not have refused those. """ with aioresponses() as m: m.get( diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index c4c740d4..be2da4e0 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -245,3 +245,35 @@ def test_a_handler_recognises_a_message_in_any_form_it_accepts(): assert frbc.supports_message(as_dict) assert frbc.supports_message(description.model_dump_json()) assert not frbc.supports_message(as_dict | {"message_type": "Handshake"}) + + +@pytest.mark.asyncio +async def test_the_cem_records_a_revocation_by_its_id_as_text(): + """With no control type active, the CEM itself takes the revocation and hands it to the handler it is for.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.cem import CEM + + cem = CEM(fm_client=frbc_simple()._fm_client) + frbc = frbc_simple() + cem.register_control_type(frbc) + object_id = get_unique_id() + await cem.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_turns_away_a_model_it_does_not_handle(): + from s2python.common import EnergyManagementRole, Handshake + + handshake = Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.CEM, + supported_protocol_versions=["0.0.2-beta"], + ) + assert not frbc_simple().supports_message(handshake) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 0435ef37..41b788c3 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -417,3 +417,66 @@ async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_t values = posts[0].kwargs["values"] assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_from_the_quarter_hour_it_starts_in( + cem_in_frbc_control_type, +): + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, 0, 7, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.001 + ) + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["start"] == datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( + cem_in_frbc_control_type, +): + """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths, per quarter hour.""" + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + half = Duration(root=int(7.5 * 60 * 1000)) + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.001), + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.003), + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) From 4d5435eda369891f8446ac3a81bcb1e6a8a7cb85 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:40:29 +0100 Subject: [PATCH 15/59] fix(FRBC): every rate per second, as S2 states it, under the instructions The schedule translation keeps a running fill level, so that each step is answered with the operation mode for the fill level the storage is then at. It advanced that level by a fill rate per second times a step in hours, so it moved 3600 times too little, and every step of a twelve-hour schedule was answered at the fill level the schedule started from (finding C1). Fixing that alone would have been worse: the level would then rise at the real rate and never fall, and pin every step at the top. So the integrator gets one written contract, the one S2 states for all three rates: a change in fill level per second, in the resource manager's own units, with usage and leakage lowering it where positive. Its step is a duration rather than a number of some unit. - Leakage now counts (C2). It read leakage_rate_per_second, which S2 does not have, and an actuator's leakage_behaviour, which cannot exist, both through getattr with a default, so the branch never ran. The handler now passes the latest FRBC.LeakageBehaviour, and the element that applies at the fill level. - Usage now counts. The handler passes the latest FRBC.UsageForecast, read per step by usage_rate_at; a frame may still carry a usage_forecast column instead, per second, which is what TUNES does, now dividing by the quarter hour it posts in. - The step is the schedule's own (B5): its duration over its number of values, which is the resolution of the sensor FlexMeasures answered at. It was taken to be a quarter hour whatever it was, so an hourly sensor's twelve values were laid on forty-eight quarter hours, pandas refused, and no instruction went out. - An instruction goes out wherever the mode or its factor changes, not only where the power does: at a constant power the fill level can pass from one element into another, which needs another factor. - The factor is the one that draws the power asked for (C4), taken from the power range, rather than worked back out of the fill rate, which failed wherever the fill rate was one value and the power a range: asked for 1200 W from a mode running 800 to 1200 W, that asked for 800 W. Where the power is one value and the fill rate a range, every factor draws that power, so the one that fills most is asked for. op_mode_compute_factor is gone. - The storage efficiency compounds a leakage rate per second (B10), where it compounded it per hour, declaring a buffer that loses about 7% over twelve hours as holding its heat to within 0.002%. - The handler's power and fill-rate measurements cover its own resolution (D9), rather than a quarter hour whatever it planned on. - translations.py is linted again: it opened with `# flake8: noqa`, which hid nothing, as it turns out. TUNES shares the translation, so its instructions change with it, for the better: it carried the same per-hour integration. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 13 +- .../s2/control_types/FRBC/frbc_tunes.py | 4 +- .../s2/control_types/FRBC/utils.py | 331 +++++++-------- .../s2/control_types/translations.py | 17 +- tests/s2/test_frbc_time_base.py | 391 ++++++++++++++++++ tests/s2/test_frbc_tunes.py | 40 ++ tests/s2/test_frbc_utils.py | 45 +- 7 files changed, 642 insertions(+), 199 deletions(-) create mode 100644 tests/s2/test_frbc_time_base.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 131a8753..36cbdc18 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -396,6 +396,11 @@ def _in_the_power_unit(self, power: float) -> float: return power return convert_units([power], S2_POWER_UNIT, self.power_unit)[0] + @staticmethod + def _latest(history: dict): + """The message a history holds that arrived last, or None while it holds none.""" + return list(history.values())[-1] if history else None + def _latest_fill_level(self) -> float | None: """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" if not self._storage_status_history: @@ -507,7 +512,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=recorded_at, values=[value], unit=unit, - duration=timedelta(minutes=15), + duration=self._resolution, ) except (JobFailedError, ValueError, ConnectionError, NotImplementedError): # This runs in a task nobody awaits, @@ -536,7 +541,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=self.now(), values=[rate], unit=self.fill_rate_unit(), - duration=timedelta(minutes=15), + duration=self._resolution, ) async def _record_charging_efficiency( @@ -914,6 +919,10 @@ async def trigger_schedule( soc_min / self._fill_level_scale, soc_max / self._fill_level_scale, ), + # What the storage loses by itself and what is drawn from it, as the resource manager last said, + # so that the fill level each step is answered at falls as well as rises. + leakage_behaviour=self._latest(self._leakage_behaviour_history), + usage_forecast=self._latest(self._usage_forecast_history), ) await self.send_instruction_batch( diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 48ce8eb7..7eef5496 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,9 +480,11 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) + # Posted per quarter hour by send_usage_forecast, and read back as such, + # while the schedule translation takes a usage rate per second, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) + ) / (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 35e652a2..587698ea 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -4,7 +4,6 @@ from math import isclose from typing import List, Sequence, TypeVar -import numpy as np import pandas as pd try: @@ -15,6 +14,7 @@ FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCSystemDescription, + FRBCUsageForecast, ) from s2python.frbc.frbc_operation_mode_element import FRBCOperationModeElement except ImportError: @@ -29,99 +29,49 @@ LOGGER = logging.getLogger(__name__) -def op_mode_compute_factor( - op_mode_elem: FRBCOperationModeElement, - fill_rate: float, - logger: logging.Logger = LOGGER, -) -> float: - """Compute the operation mode factor for a given fill rate.""" - start_fill_rate = op_mode_elem.fill_rate.start_of_range - end_fill_rate = op_mode_elem.fill_rate.end_of_range - delta_fill_rate = end_fill_rate - start_fill_rate - - fill_rate = max(fill_rate, start_fill_rate) - fill_rate = min(fill_rate, end_fill_rate) - - # Case that start_fill_rate == end_fill_rate - if np.isclose(delta_fill_rate, 0): - return 0 - - omf = (fill_rate - start_fill_rate) / delta_fill_rate - if omf < 0 or omf > 1: - logger.error( - f"Invalid operation mode factor {omf} computed from fill_rate {fill_rate}" - ) - return omf - - def compute_next_fill_level( fill_level: float, fill_rate: float, - leakage_behaviour: FRBCLeakageBehaviour | None, - deltaT: float, - usage: float = 0.0, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_rate: float = 0.0, ) -> float: - """ - Compute the next fill level of a storage system over a time step. - - Parameters - ---------- - fill_level - Current fill level in system units (e.g., kWh, °C, height) - fill_rate - Fill rate chosen from the operation mode (units per hour) - leakage_behaviour - FRBC LeakageBehaviour; may be None - deltaT - Time step in hours - usage - Consumption/usage over the time step (in system units per hour) - - Returns - ------- - next_fill_level : float - Fill level after applying fill rate, leakage, and usage - """ + """The fill level one step later. - # --- Apply fill rate over deltaT --- - delta_fill = fill_rate * deltaT - - # --- Apply leakage over deltaT --- - total_leakage = 0.0 - if leakage_behaviour and leakage_behaviour.elements: - # Sum contributions from all relevant elements (or pick one based on fill level) - element = next( - ( - e - for e in leakage_behaviour.elements - if e.fill_level_range.start_of_range - <= fill_level - <= e.fill_level_range.end_of_range - ), - None, - ) - if element is None: - # Take closest element - element = min( - leakage_behaviour.elements, - key=lambda e: max( - 0.0, - e.fill_level_range.start_of_range - fill_level, - fill_level - e.fill_level_range.end_of_range, - ), - ) + This is the unit contract for every rate it takes, and it is the one S2 states: + each is a change in fill level per second, in the resource manager's own fill-level units. + A fill rate raises the fill level, while a usage rate and a leakage rate both lower it where they are positive. + The step is a duration, rather than a number of some unit, so that it cannot be taken in the wrong one. + + :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; + none where the resource manager has not said. + :param usage_rate: what is drawn from the storage, as the usage forecast expects it over this step. + """ + leakage_rate = 0.0 + if leakage_behaviour is not None: + leakage_rate = element_for_fill_level( + leakage_behaviour.elements, fill_level + ).leakage_rate + return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() - # leakage_rate_per_second is in system units lost per second - total_leakage = ( - getattr(element, "leakage_rate_per_second", 0.0) * deltaT * 3600 - ) # convert hours → seconds - # --- Apply usage --- - total_usage = usage * deltaT +def usage_rate_at( + usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp +) -> float: + """The usage rate a forecast expects at a moment, per second, as S2 states it. - # --- Next fill level --- - next_fill_level = fill_level + delta_fill - total_leakage - total_usage - return next_fill_level + A forecast is a run of elements, each lasting its own duration from where the one before it ended. + Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. + """ + if usage_forecast is None: + return 0.0 + element_start = pd.Timestamp(usage_forecast.start_time) + for element in usage_forecast.elements: + element_end = element_start + timedelta(milliseconds=element.duration.root) + if element_start <= moment < element_end: + return element.usage_rate_expected + element_start = element_end + return 0.0 def fm_schedule_to_instructions( @@ -130,15 +80,30 @@ def fm_schedule_to_instructions( initial_fill_level: float, logger: logging.Logger = LOGGER, fill_level_bounds: tuple[float, float] | None = None, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_forecast: FRBCUsageForecast | None = None, ) -> List[FRBCInstruction]: """Turn a FlexMeasures schedule into the instructions that follow it. + Each step is answered with the operation mode, and the factor within it, that draws the power the schedule asks for + at the fill level the storage is then at, so a running fill level is kept alongside, under the contract of compute_next_fill_level. + An instruction is sent where that answer changes, which can happen at a constant power too, + where the fill level passes from one element of a mode into another. + + :param schedule: a schedule as FlexMeasures returns it, or a frame indexed by time with a "schedule" column. + A frame may also carry a "usage_forecast" column, per second as S2 states it, + which then takes the place of usage_forecast. :param fill_level_bounds: the fill levels this conversion holds its own running fill level between, in the resource manager's units. These have to be the hard bounds the schedule was planned against, or a step is answered with the operation mode for a fill level the scheduler never planned. Without them the declared fill-level range is used, which is right only for a caller that asked for a schedule against that same range. + :param leakage_behaviour: what the storage loses by itself, as the resource manager last said. + :param usage_forecast: what is drawn from the storage, as the resource manager last forecast it. + + The step is the schedule's own: a schedule from FlexMeasures states it as its duration over its number of values, + which is the resolution of the sensor it was asked for, whatever resolution the caller plans on. """ if fill_level_bounds is not None and fill_level_bounds[0] > fill_level_bounds[1]: @@ -153,85 +118,99 @@ def fm_schedule_to_instructions( return [] if isinstance(schedule, dict): + values = schedule["values"] + if len(values) == 0: + return [] start = pd.Timestamp(schedule["start"]) - schedule_duration = pd.Timedelta(schedule["duration"]) - idx = pd.DatetimeIndex( - pd.date_range( - start=start, - end=start + schedule_duration - timedelta(minutes=15), - freq="15min", - ) + step = pd.Timedelta(schedule["duration"]) / len(values) + idx = pd.date_range(start=start, periods=len(values), freq=step) + schedule = pd.Series(values, index=idx, name="schedule").to_frame() + else: + # A frame of one row has no later step for its fill level to matter to. + step = ( + pd.Timedelta(schedule.index[1] - schedule.index[0]) + if len(schedule) > 1 + else pd.Timedelta(0) ) - schedule = pd.Series(schedule["values"], index=idx, name="schedule").to_frame() - - previous_power = None - instructions = [] - - actuator = system_description.actuators[0] if len(system_description.actuators) != 1: raise NotImplementedError( f"This CEM only supports 1 actuator but" f"{len(system_description.actuators)} were provided" ) - + actuator = system_description.actuators[0] operation_modes: list[FRBCOperationMode] = actuator.operation_modes - fill_level = initial_fill_level + # Held between the hard bounds the schedule was planned against, + # so that each step is answered with the operation mode for the fill level the scheduler planned. + # Falling back to the declared range keeps every other caller as it was. + if fill_level_bounds is None: + fill_level_range: NumberRange = system_description.storage.fill_level_range + fill_level_min = fill_level_range.start_of_range + fill_level_max = fill_level_range.end_of_range + else: + fill_level_min, fill_level_max = fill_level_bounds - deltaT = timedelta(minutes=15) / timedelta(hours=1) + instructions: list[FRBCInstruction] = [] + last_instructed: tuple[str, float] | None = None + fill_level = initial_fill_level + has_usage_column = "usage_forecast" in schedule.columns for timestamp, row in schedule.iterrows(): power = row["schedule"] - usage = row.get("usage_forecast", 0) - - if previous_power is None or not isclose(previous_power, power): - # Convert from power to fill rate - results = [ - (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) - for om in operation_modes - ] - - # Step 1: minimize fill-level penalty (primary) - min_fill_penalty = min(r[2] for r in results) - fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] - - # Step 2: among those, minimize power penalty (secondary) - min_power_penalty = min(r[3] for r in fill_level_candidates) - power_candidates = [ - r for r in fill_level_candidates if r[3] == min_power_penalty - ] - - # Step 3: among those, pick the highest efficiency - best = max(power_candidates, key=lambda r: r[4]) - - ( - best_operation_mode, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - best_element, - ) = best - - logger.debug( - "Selected FRBC operation mode '%s' (element='%s') for power=%.3f, fill_level=%.3f: " - "fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", - om_label(best_operation_mode), - getattr(best_element, "id", "unknown"), - power, - fill_level, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - explain_choice(results, best), - ) - operation_mode_factor = op_mode_compute_factor( - best_element, fill_rate=best_fill_rate, logger=logger - ) + # The mode and factor for this power at this fill level, every step, + # since the element that applies can change while the power does not. + results = [ + (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) + for om in operation_modes + ] + + # Step 1: minimize fill-level penalty (primary) + min_fill_penalty = min(r[2] for r in results) + fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] + # Step 2: among those, minimize power penalty (secondary) + min_power_penalty = min(r[3] for r in fill_level_candidates) + power_candidates = [ + r for r in fill_level_candidates if r[3] == min_power_penalty + ] + + # Step 3: among those, pick the highest efficiency + best = max(power_candidates, key=lambda r: r[4]) + + ( + best_operation_mode, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + best_element, + operation_mode_factor, + ) = best + + logger.debug( + "Selected FRBC operation mode '%s' for power=%.3f, fill_level=%.3f: " + "factor=%.6f, fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", + om_label(best_operation_mode), + power, + fill_level, + operation_mode_factor, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + explain_choice(results, best), + ) + + # The factor is the one that draws the power asked for, taken from the power range itself. + # It used to be worked back out of the fill rate instead, + # which fails wherever the fill-rate range is a single value and the power range is not: + # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. + answer = (str(best_operation_mode.id), operation_mode_factor) + if last_instructed is None or not ( + answer[0] == last_instructed[0] and isclose(answer[1], last_instructed[1]) + ): instruction = FRBCInstruction( message_id=get_unique_id(), id=get_unique_id(), @@ -245,29 +224,22 @@ def fm_schedule_to_instructions( f"Instruction created: at {timestamp} set {actuator.diagnostic_label if isinstance(actuator.diagnostic_label, str) else actuator} to {best_operation_mode.diagnostic_label if isinstance(best_operation_mode.diagnostic_label, str) else best_operation_mode} with factor {operation_mode_factor}" ) instructions.append(instruction) + last_instructed = answer - # Update fill level + usage_rate = ( + float(row["usage_forecast"]) + if has_usage_column + else usage_rate_at(usage_forecast, pd.Timestamp(timestamp)) + ) fill_level = compute_next_fill_level( fill_level=fill_level, - fill_rate=best_fill_rate, # S2-derived fill rate - leakage_behaviour=getattr(actuator, "leakage_behaviour", None), - deltaT=deltaT, - usage=usage, + fill_rate=best_fill_rate, + step=step, + leakage_behaviour=leakage_behaviour, + usage_rate=usage_rate, ) - - # Hold the running fill level between the hard bounds the schedule was planned against, - # so that each step is answered with the operation mode for the fill level the scheduler planned. - # Falling back to the declared range keeps every other caller as it was. - if fill_level_bounds is None: - fill_level_range: NumberRange = system_description.storage.fill_level_range - fill_level_min = fill_level_range.start_of_range - fill_level_max = fill_level_range.end_of_range - else: - fill_level_min, fill_level_max = fill_level_bounds fill_level = max(fill_level_min, min(fill_level_max, fill_level)) - previous_power = power - return instructions @@ -408,7 +380,7 @@ def power_to_fill_rate_with_metrics( operation_mode: FRBCOperationMode, power: float, fill_level: float, -) -> tuple[float, float, float, float, FRBCOperationModeElement]: +) -> tuple[float, float, float, float, FRBCOperationModeElement, float]: """ Evaluate an operation mode for given power and fill level. @@ -418,6 +390,7 @@ def power_to_fill_rate_with_metrics( power_penalty: float efficiency: float element: FRBCOperationModeElement that produced fill_rate + factor: the operation mode factor that draws this power, which is what an instruction asks for """ # --- 1. Select the element that applies at this fill level --- best_element = element_for_fill_level(operation_mode.elements, fill_level) @@ -441,7 +414,12 @@ def power_to_fill_rate_with_metrics( pr_end = power_range.end_of_range if pr_end == pr_start: - factor = 0.0 + # Every factor draws this one power, so the power cannot say which factor is meant. + # The one that fills the storage most for it is asked for, which is either end of the fill-rate range, + # and where that range is a single value too, the factor makes no difference and stays at zero. + fr_start = best_element.fill_rate.start_of_range + fr_end = best_element.fill_rate.end_of_range + factor = 1.0 if fr_end > fr_start else 0.0 efficiency = 0.0 else: factor = (power - pr_start) / (pr_end - pr_start) @@ -455,7 +433,7 @@ def power_to_fill_rate_with_metrics( fr_end = best_element.fill_rate.end_of_range fill_rate = fr_start + factor * (fr_end - fr_start) - return fill_rate, fill_penalty, power_penalty, efficiency, best_element + return fill_rate, fill_penalty, power_penalty, efficiency, best_element, factor def clamp_distance(value: float, start: float, end: float) -> float: @@ -516,10 +494,18 @@ def om_label(operation_mode: FRBCOperationMode) -> str: def explain_choice( results: list[ - tuple[FRBCOperationMode, float, float, float, float, FRBCOperationModeElement] + tuple[ + FRBCOperationMode, + float, + float, + float, + float, + FRBCOperationModeElement, + float, + ] ], best: tuple[ - FRBCOperationMode, float, float, float, float, FRBCOperationModeElement + FRBCOperationMode, float, float, float, float, FRBCOperationModeElement, float ], ) -> str: """ @@ -532,11 +518,12 @@ def explain_choice( best_power_penalty, best_efficiency, best_element, + _best_factor, ) = best lines: list[str] = [] - for om, fr, fill_pen, power_pen, eff, element in results: + for om, fr, fill_pen, power_pen, eff, element, _factor in results: label = om_label(om) element_label = getattr(element, "id", "unknown") diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a4cd5ecc..f12b2d01 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -1,5 +1,3 @@ -# flake8: noqa - from datetime import timedelta import numpy as np @@ -35,7 +33,7 @@ def leakage_behaviour_to_storage_efficiency( due to leakage within the given range of the fill level. This is defined as a function of the fill level. - Storage Efficiency: percentage of the storage that remains after one time period. + Storage Efficiency: percentage of the storage that remains after one time step of the given resolution. Example: @@ -71,12 +69,13 @@ def leakage_behaviour_to_storage_efficiency( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) leakage = element.leakage_rate * fill_level_scale - base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) - - # Discuss conversions - storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** ( - resolution / base_resolution - ) + # S2 states a leakage rate per second, so what remains after one second is compounded over the seconds in a step. + # This used to compound it over the hours in a step instead, + # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, + # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. + # The loss is taken as a share of the top of the range, as before, which understates it at lower fill levels. + seconds_per_step = pd.Timedelta(resolution) / pd.Timedelta(seconds=1) + storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** seconds_per_step return storage_efficiency diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py new file mode 100644 index 00000000..d1e91623 --- /dev/null +++ b/tests/s2/test_frbc_time_base.py @@ -0,0 +1,391 @@ +"""The time base under the instructions: every rate per second, as S2 states it. + +S2 states a fill rate, a usage rate and a leakage rate each as a change in fill level per second. +The schedule translation keeps a running fill level, to answer each step with the operation mode for the fill level +the storage is then at, and it used to advance that level by a rate per second times a step in hours, +so it moved 3600 times too little, and every step of a twelve-hour schedule was answered at the fill level it started from. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import pandas as pd +import pytest +from s2python.common import ( + Commodity, + CommodityQuantity, + Duration, + NumberRange, + PowerRange, +) +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, +) + +from flexmeasures_client.client import FlexMeasuresClient, _server_version_at_least +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + compute_next_fill_level, + fm_schedule_to_instructions, + usage_rate_at, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +START = datetime(2024, 1, 1, tzinfo=timezone.utc) +QUARTER_HOUR = timedelta(minutes=15) + + +def _leakage(*elements: tuple[float, float, float]) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=START, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + leakage_rate=rate, + ) + for lo, hi, rate in elements + ], + ) + + +def _usage(*elements: tuple[timedelta, float]) -> FRBCUsageForecast: + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=int(duration / timedelta(milliseconds=1))), + usage_rate_expected=rate, + ) + for duration, rate in elements + ], + ) + + +# --------------------------------------------------------------------------- +# The integrator's contract +# --------------------------------------------------------------------------- + + +def test_a_fill_rate_is_integrated_per_second(): + """0.001 per second for a quarter hour is 0.9, which the old integrator made 0.00025.""" + assert compute_next_fill_level(1.0, 0.001, QUARTER_HOUR) == pytest.approx(1.9) + + +def test_usage_lowers_the_fill_level_per_second(): + assert compute_next_fill_level( + 10.0, 0.0, QUARTER_HOUR, usage_rate=0.0005 + ) == pytest.approx(10.0 - 0.45) + + +def test_leakage_lowers_the_fill_level_at_the_rate_of_the_element_that_applies(): + """At 50, in the leakier upper element, an hour loses 3.6; the lower element would lose 0.36.""" + leakage = _leakage((50, 100, 0.001), (0, 50, 0.0001)) + assert compute_next_fill_level( + 50.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(46.4) + assert compute_next_fill_level( + 20.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(19.64) + + +@pytest.mark.parametrize( + "moment, expected", + [ + (START - timedelta(minutes=1), 0.0), + (START + timedelta(minutes=30), 0.001), + (START + timedelta(minutes=90), 0.002), + # Where one element ends, the next begins, and its rate applies. + (START + timedelta(minutes=60), 0.002), + (START + timedelta(minutes=150), 0.0), + ], +) +def test_the_usage_rate_a_forecast_expects_at_a_moment(moment, expected): + """Elements follow one another for their own durations; outside them nothing is forecast.""" + forecast = _usage((timedelta(hours=1), 0.001), (timedelta(hours=1), 0.002)) + assert usage_rate_at(forecast, pd.Timestamp(moment)) == pytest.approx(expected) + + +def test_without_a_forecast_nothing_is_used(): + assert usage_rate_at(None, pd.Timestamp(START)) == 0.0 + + +# --------------------------------------------------------------------------- +# The translation +# --------------------------------------------------------------------------- + + +def _element(lo, hi, power_top, fill_rate_top, power_bottom=0.0, fill_rate_bottom=0.0): + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + fill_rate=NumberRange( + start_of_range=fill_rate_bottom, end_of_range=fill_rate_top + ), + power_ranges=[ + PowerRange( + start_of_range=power_bottom, + end_of_range=power_top, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +def _system_description(*elements) -> FRBCSystemDescription: + mode = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heat", + elements=list(elements), + abnormal_condition_only=False, + ) + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=START, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=[mode], + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=True, + provides_fill_level_target_profile=False, + provides_usage_forecast=True, + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + ), + ) + + +#: Below 50 a mode draws up to 1000 W, above it up to 3000 W, filling at most 1/150 per second either way. +#: At a constant 500 W it fills 3 per quarter hour below 50, and 1 per quarter hour above it. +LOW = _element(0, 50, power_top=1000.0, fill_rate_top=1 / 150) +HIGH = _element(50, 100, power_top=3000.0, fill_rate_top=1 / 150) + + +def _schedule(values, step=QUARTER_HOUR) -> dict: + return { + "start": START.isoformat(), + "duration": pd.Timedelta(step * len(values)).isoformat(), + "values": values, + } + + +def _factors(instructions) -> list[tuple[datetime, float]]: + return [(i.execution_time, i.operation_mode_factor) for i in instructions] + + +def test_a_constant_power_is_instructed_anew_where_the_fill_level_crosses_into_another_element(): + """500 W is half of what the lower element draws at full tilt, and a sixth of the upper one's. + + From 45, three quarter hours of filling at 3 each take the buffer past 50, + where the same 500 W needs another factor, which is a new instruction although the power did not change. + """ + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + 2 * QUARTER_HOUR, pytest.approx(1 / 6)), + ] + + +def test_usage_brings_the_running_fill_level_down_across_a_boundary(): + """Filling 1 per quarter hour above 50 while 4 is used leaves 3 less each step, so from 55 the buffer drops below 50.""" + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_leakage_brings_the_running_fill_level_down_across_a_boundary(): + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + leakage_behaviour=_leakage((0, 100, 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_a_usage_column_takes_the_place_of_a_forecast(): + """A caller holding its usage per step in the frame, as TUNES does, passes it per second.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=55.0 + ) + assert _factors(instructions)[1] == ( + START + 2 * QUARTER_HOUR, + pytest.approx(0.5), + ) + + +@pytest.mark.parametrize( + "step", [timedelta(minutes=5), QUARTER_HOUR, timedelta(hours=1)] +) +def test_a_schedule_is_read_at_the_resolution_it_comes_in(step): + """FlexMeasures answers at the power sensor's resolution, which was taken to be a quarter hour whatever it was. + + Twelve hourly values were laid on forty-eight quarter hours, and pandas refused, so no instruction went out. + """ + instructions = fm_schedule_to_instructions( + _schedule([0.0] * 6 + [1000.0] * 6, step=step), + _system_description(_element(0, 100, power_top=1000.0, fill_rate_top=1e-9)), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.0)), + (START + 6 * step, pytest.approx(1.0)), + ] + + +def test_the_instruction_asks_for_the_factor_that_draws_the_power(): + """A mode running 800 to 1200 W at one fill rate, asked for 1200 W, is asked for factor 1, which draws 1200 W.""" + instructions = fm_schedule_to_instructions( + _schedule([1200.0]), + _system_description( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=2e-3, + ) + ), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [(START, pytest.approx(1.0))] + + +# --------------------------------------------------------------------------- +# The storage efficiency +# --------------------------------------------------------------------------- + + +def test_the_storage_efficiency_compounds_a_leakage_rate_per_second(): + """A 10 kWh buffer losing 1.5 kWh a day keeps 99.844% per quarter hour, about 92.8% over twelve hours.""" + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage((0, 10, 1.5 / 86400)), resolution=QUARTER_HOUR + ) + assert efficiency == pytest.approx(100 * (1 - 1.5 / 86400 / 10) ** 900, rel=1e-12) + assert 100 * (efficiency / 100) ** 48 == pytest.approx(92.774, abs=1e-3) + + +# --------------------------------------------------------------------------- +# The handler +# --------------------------------------------------------------------------- + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._fm_client.server_version_at_least = AsyncMock( + side_effect=lambda minimum: _server_version_at_least("1.1.0", minimum) + ) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=45.0 + ) + frbc._fm_client.trigger_and_get_schedule = AsyncMock( + return_value=_schedule([500.0] * 4) + ) + frbc.send_instruction_batch = AsyncMock(return_value=0) + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_leakage_and_usage(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description, resolution=timedelta(minutes=5)) + older, latest = _leakage((0, 100, 1e-4)), _leakage((0, 100, 2e-4)) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(latest.message_id)] = latest + forecast = _usage((timedelta(hours=1), 1e-3)) + frbc._usage_forecast_history[str(forecast.message_id)] = forecast + + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + + kwargs = translate.call_args.kwargs + assert kwargs["leakage_behaviour"] is latest + assert kwargs["usage_forecast"] is forecast + + +@pytest.mark.asyncio +async def test_a_measurement_covers_the_handlers_own_resolution(): + """A power measurement for one step of a five-minute grid is not posted as covering a quarter hour.""" + system_description = _system_description(LOW, HIGH) + frbc = _frbc( + system_description, resolution=timedelta(minutes=5), fill_rate_sensor_id=6 + ) + actuator = system_description.actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=0.5, + ) + ) + durations = { + (call.args[:1] or (call.kwargs.get("sensor_id"),))[0]: call.kwargs["duration"] + for call in frbc._fm_client.post_sensor_data.await_args_list + } + assert durations == {1: timedelta(minutes=5), 6: timedelta(minutes=5)} + + +def test_a_schedule_without_values_asks_for_nothing(): + """Its step would be its duration over no values, so there is none, and no instruction either.""" + assert ( + fm_schedule_to_instructions( + {"start": START.isoformat(), "duration": "PT0H", "values": []}, + _system_description(LOW, HIGH), + initial_fill_level=45.0, + ) + == [] + ) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 41b788c3..ef02599f 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -480,3 +480,43 @@ async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( if c.kwargs.get("sensor_id") == 12 ] assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) + + +@pytest.mark.asyncio +async def test_planning_reads_its_usage_forecast_back_per_second( + cem_in_frbc_control_type, +): + """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" + from unittest.mock import patch + + from s2python.frbc import FRBCStorageStatus + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + def sensor_data(sensor_id, start=None, duration=None, **kwargs): + # As many quarter hours as were asked for: the fixture's fixed day of values fits no other window. + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + # The usage forecast as posted, 900 per quarter hour, and anything else as a plain efficiency. + value = 900.0 if sensor_id == 12 else 1.0 + return { + "start": str(start), + "duration": str(duration), + "values": [value] * steps, + } + + fm_client.get_sensor_data.side_effect = sensor_data + # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. + fm_client.get_schedule.return_value = {"values": [0.0] * 24} + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_tunes.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule() + + frame = translate.call_args.args[0] + usage = list(frame["usage_forecast"].dropna()) + assert usage and usage == [pytest.approx(1.0)] * len(usage) diff --git a/tests/s2/test_frbc_utils.py b/tests/s2/test_frbc_utils.py index e9461821..13d29c4e 100644 --- a/tests/s2/test_frbc_utils.py +++ b/tests/s2/test_frbc_utils.py @@ -12,7 +12,6 @@ clamp_distance, fm_schedule_to_instructions, get_unique_id, - op_mode_compute_factor, power_to_fill_rate_with_metrics, ) @@ -39,28 +38,44 @@ def example_op_mode_elem(default_power_range): @pytest.mark.parametrize( - "fill_rate_range, input_fill_rate, expected_factor", + "power_range, fill_rate_range, power, expected_factor", [ - ((0.0, 1.0), 0.0, 0.0), - ((0.0, 1.0), 0.5, 0.5), - ((0.0, 1.0), 1.0, 1.0), - ((2.0, 4.0), 3.0, 0.5), - ((1.0, 1.0), 1.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 100.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 150.0, 0.5), + ((100.0, 200.0), (0.0, 1.0), 200.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 250.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 50.0, 0.0), + # One fill rate, a range of power: the factor still follows from the power. + # Worked back out of the fill rate instead, this asked for the bottom of the power range. + ((800.0, 1200.0), (2.0, 2.0), 1200.0, 1.0), + ((800.0, 1200.0), (2.0, 2.0), 1000.0, 0.5), + # One power, a range of fill rate: every factor draws this power, so the one that fills most is asked for. + ((1000.0, 1000.0), (1.0, 2.0), 1000.0, 1.0), + ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 0.0), + # One power and one fill rate, as an on-off actuator states them: the factor makes no difference. + ((1000.0, 1000.0), (2.0, 2.0), 1000.0, 0.0), ], ) -def test_op_mode_compute_factor( - fill_rate_range, input_fill_rate, expected_factor, default_power_range +def test_the_factor_is_the_one_that_draws_the_power_asked_for( + power_range, fill_rate_range, power, expected_factor ): - op_mode_elem = FRBCOperationModeElement( + element = FRBCOperationModeElement( fill_level_range=NumberRange(start_of_range=0.0, end_of_range=1.0), fill_rate=NumberRange( start_of_range=fill_rate_range[0], end_of_range=fill_rate_range[1] ), - power_ranges=default_power_range, - running_costs=NumberRange(start_of_range=1.0, end_of_range=1.0), + power_ranges=[ + PowerRange( + start_of_range=power_range[0], + end_of_range=power_range[1], + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], ) - - factor = op_mode_compute_factor(op_mode_elem, fill_rate=input_fill_rate) + operation_mode = FRBCOperationMode( + id=get_unique_id(), elements=[element], abnormal_condition_only=False + ) + *_, factor = power_to_fill_rate_with_metrics(operation_mode, power, 0.5) assert np.isclose(factor, expected_factor) @@ -150,7 +165,7 @@ def test_power_to_fill_rate_with_metrics_comprehensive( ) # --- Call the method under test --- - fill_rate, fill_penalty, power_penalty, efficiency, element = ( + fill_rate, fill_penalty, power_penalty, efficiency, element, _factor = ( power_to_fill_rate_with_metrics(op_mode, test_power, test_fill_level) ) From d0ae8ce6aed82716a4f819c40f9b25f289f7c65b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:45:49 +0100 Subject: [PATCH 16/59] docs(changelog): every rate per second under the instructions Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index a8cb28a7..f92543e1 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -38,9 +38,20 @@ Changed * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] +Removed +------- + +* ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] + Fixed ----- +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast`` [`#258 `__] +* A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes [`#258 `__] +* An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] +* A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] +* An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] * ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] From 23872889dc4171f4fccb5e6226fe50a0c0cb4d07 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:07:49 +0100 Subject: [PATCH 17/59] fix(FRBC): follow each step as a resource manager does, which review showed it did not An adversarial review built its own integrator from S2's definitions, a second at a time, and found the previous commit agreed with it only while nothing changed within a step. Where something did, it did not: - A step that crossed from one element into another was integrated as if it had not: from 46, filling at 0.02 per second below 50 and 0.01 above, a quarter hour at factor 1 reaches 57, where holding the first element said 64, and at an hour's step the client thought a buffer full that was at 84. The same held for leakage. - Usage was sampled at the instant a step began, where FlexMeasures planned with the mean over the step, so a five-minute draw in each quarter hour counted fully or not at all, and a forecast starting off the step counted as nothing. follow_operation_mode now follows a step a minute at a time, as a resource manager follows an instruction: the mode and the factor are what the instruction fixes, while the element of the mode and of the leakage behaviour, and the usage a forecast expects, are each taken where and when they apply. The fill level is held between its bounds throughout, rather than only at the end of a step. Also from the review: - An instruction no longer goes out for a solver's own rounding: near factor 0, math.isclose without an absolute tolerance is exact equality, so 800 W give or take a ten-millionth from a mode running 800 to 1200 W sent 30 instructions in twelve hours instead of one. - A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, rather than a NaN that pinned the running fill level at the top of its bounds for the rest of the schedule. - FillRateBasedControlTUNES reads its usage forecast back divided by the fill-level scale it posted it with, which left it ten times too small at TUNES' default scale, and plans with its latest leakage behaviour. The previous commit message called TUNES' change "for the better"; without these it was not. - One power and a fill-rate range listed downwards asks for the top of the range, where operation_mode_conversion_efficiency declares the efficiency FlexMeasures plans with, so the instruction and the plan agree. - FRBCSimple refuses a schedule_duration that is not a whole number of resolution steps: FlexMeasures answers with whole steps only, so the step, read as the duration over the number of values, would come out wrong. - The handler's storage efficiency is what remains after one step of its own resolution, and FlexMeasures reads it per step of the leakage sensor's resolution, so the two have to agree. That is now said where the sensor is taken and where the value is posted. Tests cover each of these, and each of the four mutations the review found surviving, among them an on/off heat pump told to switch off: both modes ask for factor 0, and only the mode tells them apart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 6 +- .../s2/control_types/FRBC/frbc_simple.py | 12 ++ .../s2/control_types/FRBC/frbc_tunes.py | 15 +- .../s2/control_types/FRBC/utils.py | 86 +++++++-- tests/s2/test_frbc_time_base.py | 179 ++++++++++++++++++ tests/s2/test_frbc_tunes.py | 24 ++- tests/s2/test_frbc_utils.py | 5 +- 7 files changed, 303 insertions(+), 24 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index f92543e1..260e969a 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,9 +46,11 @@ Removed Fixed ----- -* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast`` [`#258 `__] +* ``FillRateBasedControlTUNES`` reads back the usage forecast it posted in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FRBCSimple`` refuses a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed a minute at a time, as a resource manager follows an instruction, so that a fill level crossing into another element, or a change in usage within a step, is taken when it happens [`#258 `__] * A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is a solver's own rounding, is not a new instruction [`#258 `__] * An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] * A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] * An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 36cbdc18..187981e6 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -165,6 +165,9 @@ def __init__( soc_minima_sensor_id: int | None = None, soc_maxima_sensor_id: int | None = None, usage_forecast_sensor_id: int | None = None, + # Its resolution has to be the handler's resolution: a storage efficiency is what remains after one step, + # computed here for a step of the handler's resolution, + # and FlexMeasures reads it as one step of the sensor's own resolution. leakage_behaviour_sensor_id: int | None = None, # Likewise optional: without a sensor to record it on, no charging efficiency is declared, # and the scheduler plans the storage as if the actuator converted power into stored energy one for one. @@ -211,6 +214,13 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + if schedule_duration % resolution: + # FlexMeasures answers with whole steps of the power sensor's resolution only, + # so a duration between two whole numbers of steps comes back with fewer values than it spans, + # and the schedule's step, read as its duration over its number of values, would come out wrong. + raise ValueError( + f"schedule_duration has to be a whole number of resolution steps, but {schedule_duration} is not a multiple of {resolution}." + ) if schedule_attempts < 1: # A schedule is asked for at least once, or no schedule is asked for at all. raise ValueError( @@ -1028,6 +1038,8 @@ async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): ) return + # What remains after one step of this handler's resolution, + # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. storage_efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 7eef5496..f6b29c6b 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,11 +480,14 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) - # Posted per quarter hour by send_usage_forecast, and read back as such, - # while the schedule translation takes a usage rate per second, as S2 states it. + # Posted per quarter hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, + # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) / (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) + ) / ( + (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) + * self._fill_level_scale + ) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") @@ -507,6 +510,12 @@ async def trigger_schedule(self): # type: ignore[override] system_description, initial_fill_level=soc_at_start / self._fill_level_scale, logger=self._logger, + # What the storage loses by itself, so that the running fill level falls as well as rises. + leakage_behaviour=( + list(self._leakage_behaviour_history.values())[-1] + if self._leakage_behaviour_history + else None + ), ) # Revoke all previous instructions diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 587698ea..a513d9eb 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -45,7 +45,7 @@ def compute_next_fill_level( :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; none where the resource manager has not said. - :param usage_rate: what is drawn from the storage, as the usage forecast expects it over this step. + :param usage_rate: what is drawn from the storage, held over this step. """ leakage_rate = 0.0 if leakage_behaviour is not None: @@ -55,6 +55,55 @@ def compute_next_fill_level( return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() +#: How finely a step is followed, so that an element boundary, or a change in usage, is taken when it happens. +FOLLOWED_EVERY = timedelta(minutes=1) +#: How close two factors are to count as the same instruction: a solver's own rounding is not a new instruction. +FACTOR_TOLERANCE = 1e-6 + + +def follow_operation_mode( + fill_level: float, + operation_mode: FRBCOperationMode, + factor: float, + start: pd.Timestamp, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_forecast: FRBCUsageForecast | None = None, + usage_rate: float | None = None, + bounds: tuple[float, float] | None = None, +) -> float: + """The fill level after following one operation mode at one factor for a step, as a resource manager does. + + An instruction fixes the mode and the factor, not the element: + within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, + and the usage a forecast expects can change. Each is taken where and when it applies, to within FOLLOWED_EVERY, + rather than held at what it was when the step began. + + :param usage_rate: one usage rate for the whole step, which takes the place of the forecast. + :param bounds: the fill levels the storage is held between, at every moment rather than only at the end. + """ + elapsed = timedelta(0) + while elapsed < step: + dt = min(FOLLOWED_EVERY, step - elapsed) + element = element_for_fill_level(operation_mode.elements, fill_level) + low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range + fill_level = compute_next_fill_level( + fill_level, + low + (high - low) * factor, + dt, + leakage_behaviour=leakage_behaviour, + usage_rate=( + usage_rate + if usage_rate is not None + else usage_rate_at(usage_forecast, start + elapsed) + ), + ) + if bounds is not None: + fill_level = max(bounds[0], min(bounds[1], fill_level)) + elapsed += dt + return fill_level + + def usage_rate_at( usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp ) -> float: @@ -209,7 +258,8 @@ def fm_schedule_to_instructions( # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. answer = (str(best_operation_mode.id), operation_mode_factor) if last_instructed is None or not ( - answer[0] == last_instructed[0] and isclose(answer[1], last_instructed[1]) + answer[0] == last_instructed[0] + and isclose(answer[1], last_instructed[1], abs_tol=FACTOR_TOLERANCE) ): instruction = FRBCInstruction( message_id=get_unique_id(), @@ -226,19 +276,24 @@ def fm_schedule_to_instructions( instructions.append(instruction) last_instructed = answer - usage_rate = ( - float(row["usage_forecast"]) - if has_usage_column - else usage_rate_at(usage_forecast, pd.Timestamp(timestamp)) - ) - fill_level = compute_next_fill_level( - fill_level=fill_level, - fill_rate=best_fill_rate, - step=step, + # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: + # a NaN fill level would be held at the top of the bounds, and answered there from then on. + usage_rate = None + if has_usage_column: + usage_rate = ( + 0.0 if pd.isna(row["usage_forecast"]) else float(row["usage_forecast"]) + ) + fill_level = follow_operation_mode( + fill_level, + best_operation_mode, + operation_mode_factor, + pd.Timestamp(timestamp), + step, leakage_behaviour=leakage_behaviour, + usage_forecast=usage_forecast, usage_rate=usage_rate, + bounds=(fill_level_min, fill_level_max), ) - fill_level = max(fill_level_min, min(fill_level_max, fill_level)) return instructions @@ -415,11 +470,12 @@ def power_to_fill_rate_with_metrics( if pr_end == pr_start: # Every factor draws this one power, so the power cannot say which factor is meant. - # The one that fills the storage most for it is asked for, which is either end of the fill-rate range, - # and where that range is a single value too, the factor makes no difference and stays at zero. + # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency + # declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. + # Where the fill-rate range is a single value too, the factor makes no difference and stays at zero. fr_start = best_element.fill_rate.start_of_range fr_end = best_element.fill_rate.end_of_range - factor = 1.0 if fr_end > fr_start else 0.0 + factor = 0.0 if fr_end == fr_start else 1.0 efficiency = 0.0 else: factor = (power - pr_start) / (pr_end - pr_start) diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py index d1e91623..257903a1 100644 --- a/tests/s2/test_frbc_time_base.py +++ b/tests/s2/test_frbc_time_base.py @@ -389,3 +389,182 @@ def test_a_schedule_without_values_asks_for_nothing(): ) == [] ) + + +# --------------------------------------------------------------------------- +# Following a step as a resource manager does (from the review of #258) +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + FOLLOWED_EVERY, + follow_operation_mode, +) + + +def _mode(*elements, label="heat") -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _system_description_of(*modes) -> FRBCSystemDescription: + description = _system_description(LOW) + description.actuators[0].operation_modes = list(modes) + return description + + +def test_an_on_off_heat_pump_is_told_to_switch_off(): + """On and off each run at one power, so both ask for factor 0: only the mode tells them apart.""" + on = _mode( + _element( + 0, + 100, + power_top=883.7, + power_bottom=883.7, + fill_rate_top=1e-3, + fill_rate_bottom=1e-3, + ), + label="on", + ) + off = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0), label="off") + instructions = fm_schedule_to_instructions( + _schedule([883.7, 883.7, 0.0, 0.0]), + _system_description_of(on, off), + initial_fill_level=10.0, + ) + assert [(i.execution_time, str(i.operation_mode)) for i in instructions] == [ + (START, str(on.id)), + (START + 2 * QUARTER_HOUR, str(off.id)), + ] + + +def test_a_solvers_rounding_is_not_a_new_instruction(): + """At the bottom of a modulating range, 800 W give or take a ten-millionth is one instruction, not one per step.""" + modulating = _mode( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=1e-3, + ) + ) + noisy = [800.0 + (1e-7 if i % 2 else -1e-7) for i in range(48)] + instructions = fm_schedule_to_instructions( + _schedule(noisy), _system_description_of(modulating), initial_fill_level=10.0 + ) + assert len(instructions) == 1 + + +def test_usage_is_followed_through_the_step_not_sampled_at_its_start(): + """A five-minute draw in each quarter hour takes five minutes' worth, not none and not fifteen.""" + burst = _usage(*[(timedelta(minutes=5), 0.03), (timedelta(minutes=10), 0.0)] * 4) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=burst + ) == pytest.approx(50.0 - 0.03 * 300) + + +def test_usage_counts_from_where_the_forecast_starts(): + """A forecast from 7 minutes past counts from then: 8 minutes of usage in a step that began on the hour.""" + late = FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START + timedelta(minutes=7), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.01 + ) + ], + ) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=late + ) == pytest.approx(50.0 - 0.01 * 480) + + +#: Fills 0.02 per second at full tilt below 50, half that above, as a heat pump does when its buffer warms. +STEEP = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) +SHALLOW = _element(50, 100, power_top=2000.0, fill_rate_top=0.01) + + +def test_a_step_that_crosses_into_another_element_is_followed_across_it(): + """From 46, factor 1 reaches 50 after 200 s, then fills at half the rate: 57, not the 64 of holding the first element.""" + level = follow_operation_mode( + 46.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), QUARTER_HOUR + ) + assert level == pytest.approx(57.0, abs=0.02 * FOLLOWED_EVERY.total_seconds()) + + +def test_leakage_is_followed_across_an_element_boundary_too(): + """Leaking 0.01 per second above 50 and nothing below, a buffer at 54 settles at 50 rather than draining to 45.""" + leakage = _leakage((0, 50, 0.0), (50, 100, 0.01)) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + level = follow_operation_mode( + 54.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, leakage_behaviour=leakage + ) + assert level == pytest.approx(50.0, abs=0.01 * FOLLOWED_EVERY.total_seconds()) + + +def test_a_gap_in_a_usage_column_is_no_usage_rather_than_a_buffer_full_to_the_top(): + index = pd.date_range(START, periods=4, freq=QUARTER_HOUR) + with_gap = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [float("nan")] * 4}, index=index + ) + without = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [0.0] * 4}, index=index + ) + description = _system_description(LOW, HIGH) + assert _factors( + fm_schedule_to_instructions(with_gap, description, initial_fill_level=45.0) + ) == _factors( + fm_schedule_to_instructions(without, description, initial_fill_level=45.0) + ) + + +def test_a_frame_is_read_at_its_own_step(): + """An hourly frame: an hour at 500 W fills 12, so the buffer is past 50 by the second row.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 3}, + index=pd.date_range(START, periods=3, freq=timedelta(hours=1)), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + timedelta(hours=1), pytest.approx(1 / 6)), + ] + + +def test_a_schedule_duration_between_two_whole_steps_is_refused(): + """FlexMeasures answers with whole steps only, so such a schedule would be read at the wrong step.""" + with pytest.raises(ValueError, match="whole number of resolution steps"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + schedule_duration=timedelta(hours=2, minutes=30), + resolution=timedelta(hours=1), + ) + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_usage_forecast(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description) + older, latest = _usage((timedelta(hours=1), 1e-4)), _usage( + (timedelta(hours=1), 2e-4) + ) + frbc._usage_forecast_history[str(older.message_id)] = older + frbc._usage_forecast_history[str(latest.message_id)] = latest + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["usage_forecast"] is latest diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index ef02599f..ffbe7098 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -489,7 +489,12 @@ async def test_planning_reads_its_usage_forecast_back_per_second( """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" from unittest.mock import patch - from s2python.frbc import FRBCStorageStatus + from s2python.common import NumberRange + from s2python.frbc import ( + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCStorageStatus, + ) cem, fm_client, _ = cem_in_frbc_control_type frbc = cem._control_types_handlers[cem.control_type] @@ -509,6 +514,19 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): } fm_client.get_sensor_data.side_effect = sensor_data + # Not 1, which would hide a missing division: TUNES posts usage scaled into FlexMeasures' units. + frbc._fill_level_scale = 0.1 + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. fm_client.get_schedule.return_value = {"values": [0.0] * 24} with patch( @@ -519,4 +537,6 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): frame = translate.call_args.args[0] usage = list(frame["usage_forecast"].dropna()) - assert usage and usage == [pytest.approx(1.0)] * len(usage) + # 900 per quarter hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. + assert usage and usage == [pytest.approx(10.0)] * len(usage) + assert translate.call_args.kwargs["leakage_behaviour"] is leakage diff --git a/tests/s2/test_frbc_utils.py b/tests/s2/test_frbc_utils.py index 13d29c4e..26061aa8 100644 --- a/tests/s2/test_frbc_utils.py +++ b/tests/s2/test_frbc_utils.py @@ -49,9 +49,10 @@ def example_op_mode_elem(default_power_range): # Worked back out of the fill rate instead, this asked for the bottom of the power range. ((800.0, 1200.0), (2.0, 2.0), 1200.0, 1.0), ((800.0, 1200.0), (2.0, 2.0), 1000.0, 0.5), - # One power, a range of fill rate: every factor draws this power, so the one that fills most is asked for. + # One power, a range of fill rate: every factor draws this power, + # so the top of the fill-rate range is asked for, which is where the efficiency FlexMeasures plans with is declared. ((1000.0, 1000.0), (1.0, 2.0), 1000.0, 1.0), - ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 0.0), + ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 1.0), # One power and one fill rate, as an on-off actuator states them: the factor makes no difference. ((1000.0, 1000.0), (2.0, 2.0), 1000.0, 0.0), ], From 2f8902cb3d5df1b165fddee0b659c7095dfeb0b8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 14:59:08 +0100 Subject: [PATCH 18/59] fix(example): a heat pump with a buffer that is planned for, instructed, and simulated The shipped example sent no instruction at all, for two reasons that only show with both halves running (findings A1, A2). Its resource manager sent its storage status before its system description, which planned nothing, since planning hangs off a storage status and needs a description to turn a schedule into instructions; and its description was dated 2026-01-15, while the server seeds prices from the day it starts, so a schedule's window had no prices. It also could not have shown steering if it had: one operation mode, with a power range of -0.5 to 0.5 W against a fill rate of -0.5 to 0.5 per second, which is 1.8 MW, over a declared range of 0.4. The resource manager is now a heat pump keeping a hot-water buffer warm. It is idle, or heats at anything from 400 to 1500 W, at a COP of 4 while the buffer is below 6 kWh and 3 above it. It declares its leakage, warmer losing more; a day of hot-water use with showers in the morning and washing up in the evening; and a comfort profile that pre-heats before the showers. It sends its description first, dated now, and then simulates its buffer from S2's own definitions, written separately from the CEM's code: it follows each instruction from the moment it is due, and reports its fill level every so often, which asks for a new schedule. That simulation is a second opinion on the CEM's arithmetic, and a test holds the two to agreement across an element boundary. The server gives the handler a sensor for every declaration (fill rate, band, usage, leakage, charging efficiency), so each can be seen next to the schedule planned against it, and plans from now rather than a day after the description. Its connection handling is split out of the aiohttp handler, so a test can serve the shipped resource manager with the shipped CEM, over an in-memory websocket, with only FlexMeasures stood in for. Two library changes the example needed: - A system description that arrives after the latest storage status, or that changes what the device can do, is planned for against that status (finding A3), rather than waiting for a status that a resource manager whose fill level holds steady may never send. It plans without re-posting the old status as a new measurement. - A schedule never starts before now. Planning happens on every storage status while a description can be days old, so a schedule anchored to the description alone slid into the past, and its instructions all arrived as one overdue burst. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/__init__.py | 20 +- .../s2/control_types/FRBC/frbc_simple.py | 16 +- .../s2/script/websockets_client.py | 449 +++++++++++++----- .../s2/script/websockets_server.py | 121 +++-- tests/s2/test_example_end_to_end.py | 342 +++++++++++++ tests/s2/test_frbc_simple_planning.py | 57 +++ 6 files changed, 807 insertions(+), 198 deletions(-) create mode 100644 tests/s2/test_example_end_to_end.py diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 830f3b0d..b4c695e3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -230,11 +230,21 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message - # A description says what the device can do, not what it is doing, so it is stored and - # no schedule is asked for here. - # Planning happens on a storage status, which is what carries a new state of charge, - # and which is the only one of the two a resource manager is required to keep sending - # (see handle_storage_status). + # A description says what the device can do, and a storage status what it holds now, + # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), + # but a description that arrives after the latest status, or that changes what the device can do, + # is answered with a schedule against that latest status. + # Otherwise a resource manager that sends its status first, or whose fill level holds steady + # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. + if self._storage_status_history: + request_number = self._count_request(STORAGE_STATUS_REQUESTS) + task = asyncio.create_task( + self.trigger_schedule( + system_description_id, request_number=request_number + ) + ) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCUsageForecast) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 187981e6..a8655a77 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -291,8 +291,15 @@ def align_to_resolution(self, moment: datetime) -> datetime: return moment.replace(microsecond=0) - timedelta(seconds=seconds_into_the_day) def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary.""" - return self.align_to_resolution(valid_from + self._valid_from_shift) + """Where a schedule asked for at valid_from starts: shifted, never before now, then floored to a boundary. + + Never before now, since planning happens on every storage status while a description may be days old: + anchored to that description alone, every status in a long session was answered with the same window, + which slid into the past, so its instructions were all due already and arrived as one burst. + """ + return self.align_to_resolution( + max(valid_from + self._valid_from_shift, self.now()) + ) async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None @@ -309,9 +316,8 @@ async def send_storage_status( # The state of charge this status reports is what a schedule starts from, so this is # where a schedule is asked for. - # A system description is needed to turn that schedule into instructions, and a resource - # manager sends one before its first status, so its absence here means the two have gone - # out of step rather than that nothing has arrived yet. + # A system description is needed to turn that schedule into instructions. + # Without one yet, nothing is planned here, and the description plans against this status when it arrives. if not self._system_description_history: self._logger.warning( "Cannot ask for a schedule: no FRBC.SystemDescription has arrived yet." diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 942f1988..3413e4b4 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -1,6 +1,20 @@ +"""An example resource manager: a heat pump keeping a hot-water buffer warm, speaking S2's FRBC to a CEM. + +Run it by path, next to the example CEM server, as the CEM documentation shows. +It declares what the heat pump can do and what its buffer needs, then simulates that buffer: +it follows each instruction from the moment it is due, and reports its fill level every so often, +which is what asks the CEM for a new schedule. + +The buffer is simulated from S2's own definitions, and not with the CEM's code, +so that it is a second opinion on where the instructions take the storage rather than a copy of the first. +""" + +from __future__ import annotations + import asyncio -from datetime import datetime, timezone -from zoneinfo import ZoneInfo +import os +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone import aiohttp from s2python.common import ( @@ -21,19 +35,241 @@ from s2python.frbc import ( FRBCActuatorDescription, FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, FRBCInstruction, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCOperationModeElement, FRBCStorageDescription, FRBCStorageStatus, FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, ) from flexmeasures_client.s2.utils import get_unique_id +#: The range the buffer's fill level should stay in, in kilowatt hours of stored heat. +FILL_LEVEL_RANGE = (2.0, 10.0) +#: Below this fill level the water is cold enough for the heat pump to run at a COP of 4; above it, 3. +COLD_BELOW = 6.0 +#: What the buffer holds when the simulation starts, and what it can physically hold at most. +STARTING_FILL_LEVEL = 5.0 +PHYSICAL_CAPACITY = 12.0 +#: How often the simulated buffer reports its fill level, in seconds of wall-clock time. +STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) +ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC + + +def heat(watts: float) -> float: + """A rate of heat in watts as a fill rate in kilowatt hours per second, which is how S2 states a fill rate.""" + return watts / 3.6e6 + + +def per_second(kwh_per_hour: float) -> float: + return kwh_per_hour / 3600 + + +def _element( + lowest: float, + highest: float, + power: tuple[float, float], + heat_in_watts: tuple[float, float], +) -> FRBCOperationModeElement: + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=heat(heat_in_watts[0]), end_of_range=heat(heat_in_watts[1]) + ), + power_ranges=[ + PowerRange( + start_of_range=power[0], + end_of_range=power[1], + commodity_quantity=ELECTRIC_POWER, + ) + ], + ) + + +def operation_modes() -> list[FRBCOperationMode]: + """Idle, or heating at anything from 400 to 1500 W, at a COP of 4 while the buffer is cold and 3 once it is warm.""" + idle = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="idle", + elements=[_element(0, PHYSICAL_CAPACITY, (0, 0), (0, 0))], + abnormal_condition_only=False, + ) + heating = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heating", + elements=[ + _element(0, COLD_BELOW, (400, 1500), (4 * 400, 4 * 1500)), + _element(COLD_BELOW, PHYSICAL_CAPACITY, (400, 1500), (3 * 400, 3 * 1500)), + ], + abnormal_condition_only=False, + ) + return [idle, heating] + + +def system_description( + modes: list[FRBCOperationMode], valid_from: datetime +) -> FRBCSystemDescription: + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=valid_from, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + diagnostic_label="heat pump", + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=modes, + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + diagnostic_label="hot-water buffer", + fill_level_label="kWh of stored heat", + provides_leakage_behaviour=True, + provides_fill_level_target_profile=True, + provides_usage_forecast=True, + fill_level_range=NumberRange( + start_of_range=FILL_LEVEL_RANGE[0], end_of_range=FILL_LEVEL_RANGE[1] + ), + ), + ) + + +def leakage_behaviour(valid_from: datetime) -> FRBCLeakageBehaviour: + """A warmer buffer loses more: 0.8 kWh a day while cold, 1.5 kWh a day once warm.""" + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=valid_from, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=COLD_BELOW), + leakage_rate=per_second(0.8 / 24), + ), + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=COLD_BELOW, end_of_range=PHYSICAL_CAPACITY + ), + leakage_rate=per_second(1.5 / 24), + ), + ], + ) + + +def expected_usage(hour: int) -> float: + """Hot water drawn in a given hour of the day, in kWh: showers in the morning, washing up in the evening.""" + return {7: 3.0, 19: 1.0, 20: 1.0}.get(hour, 0.1) + + +def usage_forecast(start: datetime) -> FRBCUsageForecast: + """A day of hot-water use, hour by hour, from the hour start falls in.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), + usage_rate_expected=per_second(expected_usage((start.hour + h) % 24)), + ) + for h in range(24) + ], + ) + + +def fill_level_target_profile(start: datetime) -> FRBCFillLevelTargetProfile: + """Kept within the declared range, and at least 6 kWh in the hour before the morning showers.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration(root=3600 * 1000), + fill_level_range=NumberRange( + start_of_range=6.0 if (start.hour + h) % 24 == 6 else 3.0, + end_of_range=FILL_LEVEL_RANGE[1], + ), + ) + for h in range(24) + ], + ) + + +@dataclass +class Buffer: + """The simulated buffer: where its fill level goes, following S2's definitions. + + S2 states every rate per second, in the fill level's own units: + a fill rate raises the fill level, and usage and leakage lower it. + An operation mode's fill rate is interpolated between the ends of its element's range by the factor, + from the element whose fill-level range holds the fill level. + """ + + modes: dict[str, FRBCOperationMode] + leakage: FRBCLeakageBehaviour + fill_level: float = STARTING_FILL_LEVEL + mode_id: str | None = None + factor: float = 0.0 + at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @staticmethod + def _holding(elements, fill_level: float): + """The element whose range holds the fill level, the upper one at a shared boundary, the nearest outside.""" + lowest = min(e.fill_level_range.start_of_range for e in elements) + highest = max(e.fill_level_range.end_of_range for e in elements) + level = min(max(fill_level, lowest), highest) + return max( + ( + e + for e in elements + if e.fill_level_range.start_of_range + <= level + <= e.fill_level_range.end_of_range + ), + key=lambda e: ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ), + ) + + def rates(self, moment: datetime) -> tuple[float, float, float]: + """Fill, usage and leakage, each per second, at this fill level and moment.""" + fill = 0.0 + if self.mode_id is not None: + element = self._holding(self.modes[self.mode_id].elements, self.fill_level) + low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range + fill = low + (high - low) * self.factor + usage = per_second(expected_usage(moment.hour)) + leakage = self._holding(self.leakage.elements, self.fill_level).leakage_rate + return fill, usage, leakage + + def advance(self, until: datetime, step: timedelta = timedelta(minutes=1)) -> None: + """Move the fill level forward to a moment, a minute at a time, so that an element boundary is crossed in time.""" + while self.at < until: + dt = min(step, until - self.at) + fill, usage, leakage = self.rates(self.at) + self.fill_level += (fill - usage - leakage) * dt.total_seconds() + self.fill_level = min(max(self.fill_level, 0.0), PHYSICAL_CAPACITY) + self.at += dt + + def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: + self.advance(moment) + self.mode_id = str(instruction.operation_mode) + self.factor = instruction.operation_mode_factor + async def apply_at_execution_time( - ws, instruction: FRBCInstruction, previous_operation_mode_id=None + ws, + instruction: FRBCInstruction, + previous_operation_mode_id=None, + buffer: Buffer | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -49,6 +285,9 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) + now = datetime.now(timezone.utc) + if buffer is not None: + buffer.follow(instruction, now) status = FRBCActuatorStatus( message_id=get_unique_id(), actuator_id=instruction.actuator_id, @@ -57,13 +296,13 @@ async def apply_at_execution_time( previous_operation_mode_id=previous_operation_mode_id, # The moment the transition happened, so that what is recorded about it is placed there, # rather than wherever the message is read. - transition_timestamp=datetime.now(timezone.utc), + transition_timestamp=now, ) print("SENDING: FRBC.ActuatorStatus") await ws.send_str(status.model_dump_json()) -async def answer_instructions(ws) -> None: +async def answer_instructions(ws, buffer: Buffer | None = None) -> None: """Acknowledge every instruction, and report the transition each one asks for when it is due.""" active_operation_mode_id = None @@ -81,138 +320,100 @@ async def answer_instructions(ws) -> None: ) # In a task of its own, so that an instruction due later does not hold up the ones behind it. asyncio.create_task( - apply_at_execution_time(ws, instruction, active_operation_mode_id) + apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) ) active_operation_mode_id = instruction.operation_mode -async def main_s2(): - async with aiohttp.ClientSession() as session: - async with session.ws_connect("http://localhost:8080/ws") as ws: - message = Handshake( - message_id=get_unique_id(), - role=EnergyManagementRole.RM, - supported_protocol_versions=["0.1.0"], - ) - - print("SENDING: HANDSHAKE") - - await ws.send_str(message.model_dump_json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # send resource manager details - - resource_manager_details_message = ResourceManagerDetails( - message_id=get_unique_id(), - resource_id=get_unique_id(), - roles=[ - Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) - ], - instruction_processing_delay=Duration(root=1), - available_control_types=[ - ControlType.FILL_RATE_BASED_CONTROL, - ControlType.NO_SELECTION, - ], - provides_forecast=True, - provides_power_measurement_types=[ - CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - ], +async def report_fill_level( + ws, buffer: Buffer, interval: float = STATUS_INTERVAL +) -> None: + """Report where the buffer is every interval, which is what asks the CEM for a new schedule.""" + while True: + await asyncio.sleep(interval) + buffer.advance(datetime.now(timezone.utc)) + print(f"SENDING: FRBC.StorageStatus at {buffer.fill_level:.2f} kWh") + await ws.send_str( + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level ).model_dump_json() + ) - print("SENDING: ResourceManagerDetails") - - await ws.send_str(resource_manager_details_message) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # Let the server activate the control type - control_type = await ws.receive() - - control_type = control_type.json() - - print("RECEIVING: ", control_type) - - reception_status = ReceptionStatus( - subject_message_id=control_type["message_id"], - status=ReceptionStatusValues.OK, - ) - - print("SENDING: ReceptionStatus") - await ws.send_str(reception_status.model_dump_json()) - - electric_power = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - - # send storage status - storage_status = FRBCStorageStatus( - message_id=get_unique_id(), present_fill_level=0.4 - ) - - print("SENDING: FRBC.StorageStatus") - - await ws.send_str(storage_status.model_dump_json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - # send system description - operation_mode_element = FRBCOperationModeElement( - fill_level_range=NumberRange(start_of_range=0.0, end_of_range=0.5), - fill_rate=NumberRange(start_of_range=-0.5, end_of_range=0.5), - power_ranges=[ - PowerRange( - start_of_range=-0.5, - end_of_range=0.5, - commodity_quantity=electric_power, - ) - ], - ) +async def receive_until(ws, message_type: str) -> dict: + """The next message of a type, past the reception statuses that arrive in between.""" + while True: + message = (await ws.receive()).json() + print("RECEIVED: ", message) + if message.get("message_type") == message_type: + return message - operation_mode = FRBCOperationMode( - id=get_unique_id(), - elements=[operation_mode_element], - abnormal_condition_only=False, - diagnostic_label="full operating range of a perfectly efficiency battery", - ) - actuator = FRBCActuatorDescription( - id=get_unique_id(), - supported_commodities=[Commodity.ELECTRICITY], - operation_modes=[operation_mode], - transitions=[], - timers=[], - ) +async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: + """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. - storage = FRBCStorageDescription( - provides_leakage_behaviour=False, - provides_fill_level_target_profile=False, - provides_usage_forecast=False, - fill_level_range=NumberRange(start_of_range=0.05, end_of_range=0.45), - ) - - valid_from = datetime(2026, 1, 15, 20, tzinfo=ZoneInfo("Europe/Amsterdam")) - - system_description_message = FRBCSystemDescription( - message_id=get_unique_id(), - valid_from=valid_from, - actuators=[actuator], - storage=storage, - ).model_dump_json() + In the order S2 suggests: the system description first, then the storage's status. + Returns the simulated buffer, for whoever wants to look at where the instructions took it. + """ + await ws.send_str( + Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.RM, + supported_protocol_versions=["0.1.0"], + ).model_dump_json() + ) + await receive_until(ws, "HandshakeResponse") + + await ws.send_str( + ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + name="heat pump with a hot-water buffer", + roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + instruction_processing_delay=Duration(root=1000), + available_control_types=[ + ControlType.FILL_RATE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + provides_forecast=True, + provides_power_measurement_types=[ELECTRIC_POWER], + ).model_dump_json() + ) + select = await receive_until(ws, "SelectControlType") + await ws.send_str( + ReceptionStatus( + subject_message_id=select["message_id"], status=ReceptionStatusValues.OK + ).model_dump_json() + ) - print("SENDING: FRBC.SystemDescription") + now = datetime.now(timezone.utc) + modes = operation_modes() + leakage = leakage_behaviour(now) + buffer = Buffer(modes={str(m.id): m for m in modes}, leakage=leakage, at=now) + for message in ( + system_description(modes, now), + leakage, + usage_forecast(now), + fill_level_target_profile(now), + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level + ), + ): + print(f"SENDING: {message.message_type}") + await ws.send_str(message.model_dump_json()) + + reporting = asyncio.create_task(report_fill_level(ws, buffer, interval)) + try: + await answer_instructions(ws, buffer) + finally: + reporting.cancel() + return buffer - await ws.send_str(system_description_message) - await answer_instructions(ws) +async def main_s2(): + async with aiohttp.ClientSession() as session: + async with session.ws_connect("http://localhost:8080/ws") as ws: + await run_resource_manager(ws) async def main_websocket(): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 108a475b..175cc7d7 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -28,9 +28,28 @@ # system description that asked for it and runs for twelve hours, so two days would just do. SEED_PRICE_DAYS = 3 +#: Each sensor the site needs: its unit, its resolution, and any attributes. +#: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. +SITE_SENSORS: dict[str, tuple[str, str, dict]] = { + "price": ("EUR/kWh", "PT15M", {}), + # S2 states a power as positive when it is consumed, + # and this sensor holds both an actuator's power and the schedule planned for it. + # Left unsaid, FlexMeasures reads a power sensor the other way round, + # and flips a schedule's sign on the way in, which puts the two in opposite directions. + "power": ("kW", "PT15M", {"consumption_is_positive": True}), + "state of charge": ("kWh", "PT0M", {}), + "RM discharge": ("dimensionless", "PT15M", {}), + "fill rate": ("kWh/s", "PT15M", {}), + "soc minima": ("kWh", "PT15M", {}), + "soc maxima": ("kWh", "PT15M", {}), + "usage forecast": ("kWh/s", "PT15M", {}), + "storage efficiency": ("%", "PT15M", {}), + "charging efficiency": ("dimensionless", "PT15M", {}), +} + # Keys under which the configured site is kept on the aiohttp application. FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) -SENSORS = web.AppKey("sensors", tuple) +SENSORS = web.AppKey("sensors", dict) async def rm_details_watchdog(ws, cem: CEM): @@ -54,7 +73,8 @@ async def rm_details_watchdog(ws, cem: CEM): cem._logger.debug(f"CONTROL TYPE: {cem._control_type}") - # after this, schedule will be triggered on reception of a new system description + # From here on, a schedule is asked for on each storage status, + # and on a system description that arrives after one. async def websocket_producer(ws, cem: CEM): @@ -103,19 +123,34 @@ async def websocket_handler(request): # calls, which takes about ten seconds against a local server, while s2-python's # reception-status timeout is five: a resource manager gave up before the CEM had read its # first message. - fm_client = request.app[FM_CLIENT] - price_sensor, power_sensor, soc_sensor, rm_discharge_sensor = request.app[SENSORS] + await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + return ws + +async def serve_resource_manager( + ws, fm_client: FlexMeasuresClient, sensors: dict[str, dict] +) -> None: + """Act as the CEM for one resource manager connected over ws, until it disconnects.""" cem = CEM( - sensor_id=power_sensor["id"], + sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, ) frbc = FRBCSimple( - power_sensor_id=power_sensor["id"], - price_sensor_id=price_sensor["id"], - soc_sensor_id=soc_sensor["id"], - rm_discharge_sensor_id=rm_discharge_sensor["id"], + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + # Where the resource manager's declarations are recorded, so that each can be seen next to the schedule planned against it. + fill_rate_sensor_id=sensors["fill rate"]["id"], + soc_minima_sensor_id=sensors["soc minima"]["id"], + soc_maxima_sensor_id=sensors["soc maxima"]["id"], + usage_forecast_sensor_id=sensors["usage forecast"]["id"], + leakage_behaviour_sensor_id=sensors["storage efficiency"]["id"], + charging_efficiency_sensor_id=sensors["charging efficiency"]["id"], + timezone=SITE_TIMEZONE, + # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. + valid_from_shift=timedelta(0), ) cem.register_control_type(frbc) @@ -126,12 +161,10 @@ async def websocket_handler(request): rm_details_watchdog(ws, cem), ) - return ws - async def configure_site( site_name: str, fm_client: FlexMeasuresClient -) -> tuple[dict, dict, dict, dict]: +) -> dict[str, dict]: """Create or update the site asset and its sensors, and seed it with demo prices.""" account = await fm_client.get_account() if account is None: @@ -164,32 +197,21 @@ async def configure_site( site_asset["id"], site_asset_specs, parse_json_fields=True ) - sensors = site_asset.get("sensors", []) - price_sensor = None - power_sensor = None - soc_sensor = None - rm_discharge_sensor = None - for sensor in sensors: - if sensor["name"] == "price": - price_sensor = sensor - elif sensor["name"] == "power": - power_sensor = sensor - elif sensor["name"] == "state of charge": - soc_sensor = sensor - elif sensor["name"] == "RM discharge": - rm_discharge_sensor = sensor - - if price_sensor is None: - price_sensor = await fm_client.add_sensor( - name="price", - event_resolution="PT15M", - unit="EUR/kWh", + existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} + sensors: dict[str, dict] = {} + for name, (unit, resolution, attributes) in SITE_SENSORS.items(): + sensors[name] = existing.get(name) or await fm_client.add_sensor( + name=name, + event_resolution=resolution, + unit=unit, generic_asset_id=site_asset["id"], timezone=SITE_TIMEZONE, + **({"attributes": attributes} if attributes else {}), ) + # Demo prices, so that a freshly configured site has something to schedule against. # One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - # plans a day and a half ahead of the moment a resource manager sends its system description: + # plans ahead of the moment a resource manager reports its storage: # a fixed date here goes stale the day after it is written, and the scheduler then fails the # job with UnknownPricesException rather than returning a schedule. one_day = [ @@ -201,42 +223,13 @@ async def configure_site( hour=0, minute=0, second=0, microsecond=0 ) await fm_client.post_sensor_data( - sensor_id=price_sensor["id"], + sensor_id=sensors["price"]["id"], start=today, duration=timedelta(days=SEED_PRICE_DAYS), values=one_day * SEED_PRICE_DAYS, unit="EUR/kWh", ) - if power_sensor is None: - power_sensor = await fm_client.add_sensor( - name="power", - event_resolution="PT15M", - unit="kW", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. - attributes={"consumption_is_positive": True}, - ) - if soc_sensor is None: - soc_sensor = await fm_client.add_sensor( - name="state of charge", - event_resolution="PT0M", - unit="kWh", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - if rm_discharge_sensor is None: - rm_discharge_sensor = await fm_client.add_sensor( - name="RM discharge", - event_resolution="PT15M", - unit="dimensionless", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - return price_sensor, power_sensor, soc_sensor, rm_discharge_sensor + return sensors async def configure_site_once(app: web.Application) -> None: diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py new file mode 100644 index 00000000..9b97b64e --- /dev/null +++ b/tests/s2/test_example_end_to_end.py @@ -0,0 +1,342 @@ +"""The shipped example, both halves, end to end: the resource manager it ships, served by the CEM it ships. + +Only FlexMeasures is stood in for, and the websocket between the two is held in memory. +The example used to send no instruction at all, for two reasons the halves only show together: +the resource manager sent its storage status before its system description, which planned nothing, +and its description was dated ten months before the prices the server seeds. +""" + +from __future__ import annotations + +import asyncio +import gc +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import aiohttp +import pandas as pd +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.script import websockets_server as server +from flexmeasures_client.s2.script.websockets_client import run_resource_manager + + +class _Text: + type = aiohttp.WSMsgType.TEXT + + def __init__(self, data: str): + self.data = data + + def json(self) -> dict: + return json.loads(self.data) + + +#: Put on an end's inbox, it ends that end's stream of messages, as a websocket closed by its peer does. +_CLOSED = object() + + +class _End: + """One end of an in-memory websocket: what it sends arrives at the other end.""" + + def __init__(self, inbox: asyncio.Queue, outbox: asyncio.Queue): + self._inbox, self._outbox = inbox, outbox + self.received: list[dict] = [] + + async def send_str(self, data: str) -> None: + await self._outbox.put(data) + + async def send_json(self, message) -> None: + await self._outbox.put(json.dumps(message)) + + async def receive(self) -> _Text: + data = await self._inbox.get() + self.received.append(json.loads(data)) + return _Text(data) + + def __aiter__(self): + return self + + async def __anext__(self) -> _Text: + data = await self._inbox.get() + if data is _CLOSED: + raise StopAsyncIteration + self.received.append(json.loads(data)) + return _Text(data) + + async def close(self) -> None: + pass + + def of_type(self, message_type: str) -> list[dict]: + return [m for m in self.received if m.get("message_type") == message_type] + + +def _connected() -> tuple[_End, _End]: + to_resource_manager, to_cem = asyncio.Queue(), asyncio.Queue() + return _End(to_resource_manager, to_cem), _End(to_cem, to_resource_manager) + + +#: What the stand-in FlexMeasures plans, per quarter hour: idle, then full heat, then minimum heat. +SHAPE = [0.0, 0.0, 1500.0, 1500.0, 400.0, 400.0, 0.0, 0.0] + + +def _flexmeasures() -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.server_version_at_least = AsyncMock(return_value=True) + fm.get_sensor = AsyncMock( + return_value={"attributes": {"consumption_is_positive": True}} + ) + + def schedule(start, duration, **kwargs) -> dict: + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + return { + "start": pd.Timestamp(start).isoformat(), + "duration": pd.Timedelta(duration).isoformat(), + "values": [SHAPE[i % len(SHAPE)] for i in range(steps)], + "scheduler_info": {"scheduler": "StorageScheduler"}, + } + + fm.trigger_and_get_schedule = AsyncMock(side_effect=lambda **kw: schedule(**kw)) + return fm + + +async def _until(condition, timeout: float = 15.0) -> None: + async def wait(): + while not condition(): + await asyncio.sleep(0.05) + + await asyncio.wait_for(wait(), timeout) + + +@pytest.mark.asyncio +async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + unhandled: list[dict] = [] + loop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: unhandled.append(context)) + began = datetime.now(timezone.utc) + + tasks = [ + asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), + asyncio.create_task(run_resource_manager(resource_manager, interval=0.3)), + ] + try: + # Two storage statuses past the first, each a new plan, so that replanning is part of what is checked. + await _until(lambda: fm.trigger_and_get_schedule.await_count >= 3) + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 3) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + gc.collect() + loop.set_exception_handler(None) + + instructions = resource_manager.of_type("FRBC.Instruction") + distinct = { + (i["operation_mode"], round(i["operation_mode_factor"], 6)) + for i in instructions + } + assert len(distinct) >= 2, instructions + + refused = [ + m for m in resource_manager.of_type("ReceptionStatus") if m["status"] != "OK" + ] + assert refused == [] + + start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] + assert began - timedelta(minutes=15) < start <= began + timedelta(seconds=5) + assert unhandled == [] + + # Every declaration reached the sensor the server made for it, to be seen next to the schedule planned against it. + posted_to = { + call.args[0] if call.args else call.kwargs.get("sensor_id") + for call in fm.post_sensor_data.await_args_list + + fm.post_measurements.await_args_list + } + for name in ( + "power", + "state of charge", + "fill rate", + "soc minima", + "soc maxima", + "usage forecast", + "storage efficiency", + "charging efficiency", + ): + assert sensors[name]["id"] in posted_to, name + + +# --------------------------------------------------------------------------- +# The example's own buffer, as a second opinion on the CEM's arithmetic +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + compute_next_fill_level, + usage_rate_at, +) +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + Buffer, + heat, + leakage_behaviour, + operation_modes, + per_second, + usage_forecast, +) + +NOON = datetime(2024, 1, 1, 12, tzinfo=timezone.utc) + + +def _heating_buffer(fill_level: float) -> Buffer: + modes = operation_modes() + heating = next(m for m in modes if m.diagnostic_label == "heating") + return Buffer( + modes={str(m.id): m for m in modes}, + leakage=leakage_behaviour(NOON), + fill_level=fill_level, + mode_id=str(heating.id), + factor=1.0, + at=NOON, + ) + + +def test_the_example_buffer_follows_s2s_definitions(): + """A quarter hour at full heat from a cold 2 kWh: 6 kW of heat in, a tenth of a kWh an hour drawn, and a cold buffer's leakage.""" + buffer = _heating_buffer(2.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 2.0 + (heat(6000) - per_second(0.1) - per_second(0.8 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_the_example_and_the_cem_agree_on_where_the_buffer_goes(): + """Two integrators written separately from S2's definitions, one in the CEM and one in the example's buffer. + + While no boundary between elements is crossed they hold every rate constant over a step, and so must agree. + """ + buffer = _heating_buffer(2.0) + element_fill_rate = heat(6000) + cem_fill_level = 2.0 + for step in range(2): + moment = pd.Timestamp(NOON + step * timedelta(minutes=15)) + cem_fill_level = compute_next_fill_level( + cem_fill_level, + element_fill_rate, + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage_rate=usage_rate_at(usage_forecast(NOON), moment), + ) + buffer.advance(NOON + timedelta(minutes=30)) + assert cem_fill_level < 6.0 # no element boundary crossed + assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) + + +def test_a_warm_buffer_heats_at_the_warm_coefficient(): + """At 7 kWh the buffer is past the cold element, so full heat delivers 4.5 kW, and it leaks as a warm buffer does.""" + buffer = _heating_buffer(7.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 7.0 + (heat(4500) - per_second(0.1) - per_second(1.5 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_an_instruction_takes_over_from_where_the_last_one_left_the_buffer(): + """Idle for a quarter hour, losing only what is drawn and what leaks, then a quarter hour at full heat.""" + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + buffer = _heating_buffer(4.0) + buffer.mode_id, buffer.factor = None, 0.0 # idle until told otherwise + heating = next(m for m in buffer.modes.values() if m.diagnostic_label == "heating") + buffer.follow( + FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=heating.id, + operation_mode_factor=1.0, + execution_time=NOON + timedelta(minutes=15), + abnormal_condition=False, + ), + NOON + timedelta(minutes=15), + ) + buffer.advance(NOON + timedelta(minutes=30)) + losses = per_second(0.1) + per_second(0.8 / 24) + expected = 4.0 - losses * 900 + (heat(6000) - losses) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +@pytest.mark.asyncio +async def test_the_resource_manager_hands_back_its_buffer_when_the_cem_goes_away(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + serving = asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)) + managing = asyncio.create_task(run_resource_manager(resource_manager, interval=60)) + try: + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 1) + await resource_manager._inbox.put(_CLOSED) + buffer = await asyncio.wait_for(managing, timeout=5) + finally: + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + assert 0 <= buffer.fill_level <= 12 + + +@pytest.mark.asyncio +async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_today(): + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [{"id": 50, "name": "price"}], + } + ] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + + sensors = await server.configure_site(server.SITE_NAME, fm) + + assert set(sensors) == set(server.SITE_SENSORS) + assert sensors["price"]["id"] == 50 # kept, not made again + made = {call.kwargs["name"]: call.kwargs for call in fm.add_sensor.await_args_list} + assert "price" not in made + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["fill rate"]["unit"] == "kWh/s" + assert made["soc minima"]["unit"] == "kWh" + prices = fm.post_sensor_data.await_args.kwargs + assert prices["sensor_id"] == 50 + assert prices["start"].date() == datetime.now(prices["start"].tzinfo).date() + + +def test_the_example_and_the_cem_agree_across_an_element_boundary(): + """Both follow a step a minute at a time, so from 5 kWh, through the cold element and into the warm one, they agree.""" + from flexmeasures_client.s2.control_types.FRBC.utils import follow_operation_mode + + buffer = _heating_buffer(5.0) + heating = buffer.modes[buffer.mode_id] + cem_fill_level = 5.0 + for step in range(4): + cem_fill_level = follow_operation_mode( + cem_fill_level, + heating, + 1.0, + pd.Timestamp(NOON + step * timedelta(minutes=15)), + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage_forecast=usage_forecast(NOON), + ) + buffer.advance(NOON + timedelta(hours=1)) + assert ( + cem_fill_level > 6.0 + ) # the boundary between the cold and the warm element was crossed + assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index be2da4e0..e3c0a745 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -31,6 +31,8 @@ from flexmeasures_client.s2.utils import get_unique_id POWER_SENSOR_ID = 1 +#: A clock set before the descriptions below are valid, so that their own times decide where a schedule starts. +BEFORE_THE_EXAMPLES = datetime(2022, 12, 1, tzinfo=timezone.utc) SOC_SENSOR_ID = 3 @@ -175,6 +177,7 @@ async def test_a_storage_status_before_any_description_still_records_the_measure def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): """An unaligned start leaves FlexMeasures with no prices for the window, and the job fails.""" frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES start = frbc.schedule_start(datetime.fromisoformat(valid_from)) @@ -187,6 +190,7 @@ def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): @pytest.mark.asyncio async def test_the_schedule_is_asked_for_from_an_aligned_start(): frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES await frbc.handle_message(system_description("2022-12-02T09:07:33.512+00:00")) await settle(frbc) @@ -277,3 +281,56 @@ def test_a_handler_turns_away_a_model_it_does_not_handle(): supported_protocol_versions=["0.0.2-beta"], ) assert not frbc_simple().supports_message(handshake) + + +def test_a_schedule_never_starts_before_now(): + """A description days old still plans from now, on a boundary, rather than a window that has already passed.""" + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_status_before_its_description_is_planned_for_when_the_description_arrives(): + """The order the example resource manager used to send them in, which left it without a single instruction.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + frbc._fm_client.trigger_and_get_schedule.assert_awaited_once() + assert not frbc._sending_queue.empty() + + +@pytest.mark.asyncio +async def test_a_changed_description_is_planned_for_against_the_latest_status(): + """What the device can do changed, so the plan made for what it could do before no longer answers.""" + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 1 + + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_the_soc_is_not_reposted_when_a_description_asks_for_a_schedule(): + """The latest status is planned from, not recorded again as if it had been measured now.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + posts_after_the_status = frbc._fm_client.post_measurements.await_count + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.post_measurements.await_count == posts_after_the_status From fa3dc5925fe0efc178e8a48152b6d7a80ca62cf2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:16:13 +0100 Subject: [PATCH 19/59] docs(changelog): the example, and the planning triggers it needed Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 260e969a..95b9c2ae 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,6 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, and plans from now [`#259 `__] Removed ------- @@ -73,9 +74,9 @@ Fixed * An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks, and recorded none of what they declared [`#244 `__] * A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] -* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, so that the window it schedules for has prices [`#241 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, instead of when a system description arrives, so a resource manager that sends its description before its first status is planned for at all [`#240 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window [`#240 `__] +* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] +* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status or changes what the device can do, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from before now, which a description days old made it slide into [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] From 10f1cc724d3a4c3b13e94297544a98add8d68499 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:20:40 +0100 Subject: [PATCH 20/59] fix(FRBC): a state of charge in any supported energy unit reaches FlexMeasures FRBCSimple passed its energy_unit straight through as the flex model's soc-unit, while FlexMeasures validates soc-unit with OneOf(["kWh", "MWh"]). So four of the six units fill_rate_in_watts documents, J, kJ, MJ and Wh, made every trigger a 422, and the storage got no schedule at all (finding B2). The plan for this was to refuse those units when the handler is made. The carve source answers it better, by converting joules at the edge, and that is what happens now, for every supported unit: the state of charge and its two hard bounds go to FlexMeasures in kWh, converted from the recorded unit with the table fill_rate_in_watts already keeps, while kWh and MWh pass through as they are. The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself, and the bounds the schedule is read back against stay in the recorded unit, since the operation modes are stated in it. An energy unit outside that table is now refused when the handler is made, where it used to surface only at planning time, as a logged failure to record a charging efficiency. The handler for that failure is gone, since nothing can now reach it. Not changed, and now pinned by a test: the measured state-of-charge sensor is never named in the flex model (finding B13 stands). Named there, FlexMeasures would save the state of charge it plans onto that sensor, beside the measurements. The carve source does name it, and the carve will have to leave that line behind. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 48 +++++++---- tests/s2/test_frbc_charging_efficiency.py | 79 +++++++++++++++---- 2 files changed, 96 insertions(+), 31 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a8655a77..110b3f26 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -30,6 +30,7 @@ from flexmeasures_client.exceptions import JobFailedError from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( + WATTS_PER_ENERGY_UNIT_PER_SECOND, clip_fill_level_target_profile, element_for_fill_level, fm_schedule_to_instructions, @@ -248,6 +249,12 @@ def __init__( ) self._hard_bound_factor = hard_bound_factor self.power_unit = power_unit + if energy_unit not in WATTS_PER_ENERGY_UNIT_PER_SECOND: + # A fill rate is per second of this unit, and has to be compared with the watts S2 states a power in, + # so a unit that cannot be converted to watts cannot be planned with. + raise ValueError( + f"energy_unit has to be one of {', '.join(WATTS_PER_ENERGY_UNIT_PER_SECOND)}, but {energy_unit!r} was given." + ) self.energy_unit = energy_unit # delay the start of the schedule from the time `valid_from` @@ -417,6 +424,15 @@ def _latest(history: dict): """The message a history holds that arrived last, or None while it holds none.""" return list(history.values())[-1] if history else None + def _soc_unit(self) -> tuple[str, float]: + """The unit a state of charge goes to FlexMeasures in, and what a fill level is multiplied by to be in it.""" + if self.energy_unit in ("kWh", "MWh"): + return self.energy_unit, 1.0 + return "kWh", ( + WATTS_PER_ENERGY_UNIT_PER_SECOND[self.energy_unit] + / WATTS_PER_ENERGY_UNIT_PER_SECOND["kWh"] + ) + def _latest_fill_level(self) -> float | None: """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" if not self._storage_status_history: @@ -585,19 +601,13 @@ async def _record_charging_efficiency( # One actuator, as everywhere else in this handler. actuator = system_description.actuators[0] efficiencies: dict[str, float] = {} - try: - for operation_mode in actuator.operation_modes: - efficiency = operation_mode_conversion_efficiency( - operation_mode, self._fill_level_scale, self.energy_unit, fill_level - ) - if efficiency is not None: - efficiencies[str(operation_mode.id)] = efficiency - except NotImplementedError: - self._logger.exception( - "Not asking for a schedule against a charging efficiency: a fill rate in %s cannot be compared against the watts S2 states a power range in.", - self.fill_rate_unit(), + # The energy unit is one fill_rate_in_watts knows, which the constructor saw to. + for operation_mode in actuator.operation_modes: + efficiency = operation_mode_conversion_efficiency( + operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) - return False + if efficiency is not None: + efficiencies[str(operation_mode.id)] = efficiency if not efficiencies: # Every mode is idle, or every mode only feeds back, and neither converts drawn power into anything. @@ -828,12 +838,16 @@ async def trigger_schedule( # and the window a charging efficiency has to be recorded over to be planned against. start = self.schedule_start(system_description.valid_from) + # FlexMeasures takes a state of charge in kWh or MWh only, so a fill level recorded in another energy unit + # is converted here, at the edge, rather than refused, and the bounds read back below stay in the recorded unit. + # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. + soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { - # The same unit the fill level is recorded in, or the scheduler would plan against bounds a thousandfold off the measurements it plans from. - "soc-unit": self.energy_unit, - "soc-at-start": soc_at_start, # TODO: use forecast of the SOC instead - "soc-min": soc_min, - "soc-max": soc_max, + "soc-unit": soc_unit, + "soc-at-start": soc_at_start + * to_soc_unit, # TODO: use forecast of the SOC instead + "soc-min": soc_min * to_soc_unit, + "soc-max": soc_max * to_soc_unit, } # What the resource manager declared about its storage, each entry only where a sensor holds it: # an entry naming a sensor that was never configured has the scheduler plan against nothing at all. diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 7217e9cd..6baea1dd 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -301,20 +301,16 @@ async def test_a_fill_level_scale_is_applied_to_the_efficiency(): ] -@pytest.mark.asyncio -async def test_an_energy_unit_that_cannot_be_compared_to_watts_is_not_guessed_at( - caplog, -): - """A fill rate in a unit nothing converts is refused, rather than recorded thousandfold off.""" - system_description = _heater() - frbc = _frbc(system_description, energy_unit="therm") - - with caplog.at_level(logging.ERROR): - flex_model = await _asked_for(frbc, system_description) - - assert "charging-efficiency" not in flex_model - assert _recordings(frbc) == [] - assert "cannot be compared" in caplog.text +def test_an_energy_unit_that_cannot_be_compared_to_watts_is_refused_at_once(): + """A fill rate in a unit nothing converts would be recorded thousandfold off, so the handler is not made at all.""" + with pytest.raises(ValueError, match="energy_unit has to be one of"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + energy_unit="therm", + ) def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): @@ -397,3 +393,58 @@ async def test_a_schedule_is_asked_for_against_the_efficiency_where_it_starts(): assert _recordings(frbc)[0]["values"] == [pytest.approx(warm)] assert warm != pytest.approx(cold) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "energy_unit, soc_unit, per_unit", + [ + ("MJ", "kWh", 1 / 3.6), + ("Wh", "kWh", 1 / 1000), + ("J", "kWh", 1 / 3.6e6), + ("kWh", "kWh", 1.0), + ("MWh", "MWh", 1.0), + ], +) +async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( + energy_unit, soc_unit, per_unit +): + """FlexMeasures takes kWh or MWh only, so any other energy unit is converted at the edge, not refused by the server.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit=energy_unit) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-unit"] == soc_unit + assert flex_model["soc-at-start"] == pytest.approx(FILL_LEVEL_AT_START * per_unit) + assert flex_model["soc-max"] == pytest.approx(1.5 * RANGE_TOP * per_unit) + + +@pytest.mark.asyncio +async def test_the_bounds_read_back_stay_in_the_recorded_unit(): + """Only what goes to FlexMeasures is converted: the schedule is matched against the operation modes in their own units.""" + from unittest.mock import patch + + system_description = _heater() + frbc = _frbc(system_description, energy_unit="MJ") + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["fill_level_bounds"] == ( + pytest.approx(0.0), + pytest.approx(1.5 * RANGE_TOP), + ) + + +@pytest.mark.asyncio +async def test_the_measured_state_of_charge_is_not_named_in_the_flex_model(): + """The state of charge goes out as a number, never as the sensor it is measured on. + + Named there, FlexMeasures would save the state of charge it plans onto that sensor, + beside the measurements, where a later schedule would read a plan as if it had been measured. + """ + system_description = _heater() + frbc = _frbc(system_description) + flex_model = await _asked_for(frbc, system_description) + assert "state-of-charge" not in flex_model + assert isinstance(flex_model["soc-at-start"], float) From 5ba8ee240b9ec111bd1a6d03f53acc0f507d43bf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:21:06 +0100 Subject: [PATCH 21/59] docs(changelog): which energy units a handler takes, and what FlexMeasures is sent Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 95b9c2ae..ee497dba 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -17,7 +17,7 @@ Added ----- * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] -* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] +* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] * ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] From 324e7f26d6c2359b8247f4919a9c0ed9cf17d713 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 15:43:41 +0100 Subject: [PATCH 22/59] fix(FRBC): follow each step exactly, between the moments anything changes A second round of review showed that following a step a minute at a time traded one class of error for four others, and cost ten times the time: - Usage was sampled at the start of each minute. Half-minute draws starting on the half minute were never seen at all, and the same draws starting on the minute were counted twice. - A level settling on an element boundary, where filling stops above it, flipped across that boundary every step: 127 instructions in 144 steps, against one. - An element narrower than a minute's travel was skipped: a level that stops at 50 was carried to 80. - A 48-hour schedule against the largest forecast S2 allows took 1.7 s, on the event loop, scanning the forecast once a minute. Every rate is constant between the moments the usage changes and the moments the level reaches an element boundary, of the mode or of the leakage behaviour, or one of its bounds. So follow_operation_mode now steps from one such moment to the next and is exact. A level that reaches a boundary whose far side would push it back stays at that boundary, as a real storage does, and so does a level at one of its bounds, which rises again from there once filling wins. The usage forecast is read through a UsageProfile, built once per schedule and looked up by bisection. Also from the second round: - FillRateBasedControlTUNES posted its usage forecast as an amount per quarter hour labelled MW. FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour, so it planned a quarter of the real draw. It now posts per hour, and reads back accordingly. - FRBCSimple refuses a resolution or schedule duration that is not positive, where a zero resolution raised a ZeroDivisionError. - The factor tolerance's justification was wrong: FlexMeasures rounds a schedule to six decimals of MW, so what it absorbs is the noise of converting to watts, not a solver's rounding. The tolerance stays; the comment and the changelog now say what it is for. - One of the first round's mutations had stopped applying after black split its line, so the gate's count overstated what was checked. The harness now rejects a mutant that cannot run, and every mutation is anchored to the formatted code. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 8 +- .../s2/control_types/FRBC/frbc_simple.py | 4 + .../s2/control_types/FRBC/frbc_tunes.py | 15 +- .../s2/control_types/FRBC/utils.py | 192 +++++++++++++----- tests/s2/test_frbc_time_base.py | 165 ++++++++++++++- tests/s2/test_frbc_tunes.py | 25 ++- 6 files changed, 329 insertions(+), 80 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 260e969a..cce80717 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -46,11 +46,11 @@ Removed Fixed ----- -* ``FillRateBasedControlTUNES`` reads back the usage forecast it posted in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] -* ``FRBCSimple`` refuses a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] -* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed a minute at a time, as a resource manager follows an instruction, so that a fill level crossing into another element, or a change in usage within a step, is taken when it happens [`#258 `__] +* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] * A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] -* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is a solver's own rounding, is not a new instruction [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#258 `__] * An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] * A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] * An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 187981e6..efcb97e5 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -214,6 +214,10 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + if resolution <= timedelta(0) or schedule_duration <= timedelta(0): + raise ValueError( + f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." + ) if schedule_duration % resolution: # FlexMeasures answers with whole steps of the power sensor's resolution only, # so a duration between two whole numbers of steps comes back with fewer values than it spans, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index f6b29c6b..d13e5708 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -480,14 +480,11 @@ async def trigger_schedule(self): # type: ignore[override] except Exception as exc: self._logger.error(str(exc)) - # Posted per quarter hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, + # Posted per hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) / ( - (pd.Timedelta(RESOLUTION) / pd.Timedelta(seconds=1)) - * self._fill_level_scale - ) + ) / ((pd.Timedelta(hours=1) / pd.Timedelta(seconds=1)) * self._fill_level_scale) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") @@ -697,15 +694,17 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): fill_level_scale=self._fill_level_scale, ) - # Scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) - scale = timedelta(minutes=15) / timedelta(seconds=1) + # From per second, as S2 states a usage rate, to per hour, which is what a value labelled MW means: + # FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour. + # Posted per quarter hour instead, it was planned as a quarter of the real draw. + scale = timedelta(hours=1) / timedelta(seconds=1) scaled_usage_forecast = usage_forecast_series * scale await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), - unit=POWER_UNIT, # e.g. [0, 100] MW/(15 min) + unit=POWER_UNIT, # a rate per hour, as MW is duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index a513d9eb..49f07c2a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,3 +1,4 @@ +import bisect import logging import math from datetime import timedelta @@ -55,12 +56,73 @@ def compute_next_fill_level( return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() -#: How finely a step is followed, so that an element boundary, or a change in usage, is taken when it happens. -FOLLOWED_EVERY = timedelta(minutes=1) -#: How close two factors are to count as the same instruction: a solver's own rounding is not a new instruction. +#: How close two factors are to count as the same instruction. +#: It absorbs the float noise of converting a schedule from MW to W, which is not a new instruction. FACTOR_TOLERANCE = 1e-6 +class UsageProfile: + """What a usage forecast expects, per second, at any moment: piecewise constant, looked up by bisection. + + A forecast is a run of elements, each lasting its own duration from where the one before it ended. + Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. + Built once per schedule, since a forecast may hold hundreds of elements and a schedule asks about it often. + """ + + def __init__( + self, usage_forecast: FRBCUsageForecast | None = None, rate: float | None = None + ): + #: The moments the rate changes at, and the rate from each of them on; one constant rate has none. + self._changes: list[pd.Timestamp] = [] + self._rates: list[float] = [] + self._before = 0.0 if rate is None else rate + if usage_forecast is None: + return + moment = pd.Timestamp(usage_forecast.start_time) + for element in usage_forecast.elements: + self._changes.append(moment) + self._rates.append(element.usage_rate_expected) + moment = moment + timedelta(milliseconds=element.duration.root) + self._changes.append(moment) + self._rates.append(0.0) + + def rate_at(self, moment: pd.Timestamp) -> float: + index = bisect.bisect_right(self._changes, moment) - 1 + return self._before if index < 0 else self._rates[index] + + def next_change_after(self, moment: pd.Timestamp) -> pd.Timestamp | None: + index = bisect.bisect_right(self._changes, moment) + return self._changes[index] if index < len(self._changes) else None + + +def usage_rate_at( + usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp +) -> float: + """The usage rate a forecast expects at a moment, per second, as S2 states it (see UsageProfile).""" + return UsageProfile(usage_forecast).rate_at(moment) + + +def _sides(elements, fill_level: float): + """The element that applies just below a fill level, and the one just above it. + + Inside an element both are that element; at a boundary two elements share, they are the two neighbours. + """ + above = element_for_fill_level(elements, fill_level) + below_candidates = [ + e + for e in elements + if e.fill_level_range.start_of_range + < fill_level + <= e.fill_level_range.end_of_range + ] + below = ( + max(below_candidates, key=lambda e: e.fill_level_range.start_of_range) + if below_candidates + else above + ) + return below, above + + def follow_operation_mode( fill_level: float, operation_mode: FRBCOperationMode, @@ -68,59 +130,79 @@ def follow_operation_mode( start: pd.Timestamp, step: timedelta, leakage_behaviour: FRBCLeakageBehaviour | None = None, - usage_forecast: FRBCUsageForecast | None = None, - usage_rate: float | None = None, + usage: UsageProfile | None = None, bounds: tuple[float, float] | None = None, ) -> float: """The fill level after following one operation mode at one factor for a step, as a resource manager does. An instruction fixes the mode and the factor, not the element: within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, - and the usage a forecast expects can change. Each is taken where and when it applies, to within FOLLOWED_EVERY, - rather than held at what it was when the step began. - - :param usage_rate: one usage rate for the whole step, which takes the place of the forecast. - :param bounds: the fill levels the storage is held between, at every moment rather than only at the end. + and the usage a forecast expects can change. + Every rate is constant between such moments, so the step is followed exactly, from one of them to the next. + A level that reaches a boundary its far side would push it back from, as where a heat pump cannot heat above it, + stays at that boundary, as a real storage does, rather than flipping across it. + So does a level that reaches one of the bounds. + + :param usage: what is drawn from the storage, per second; none where nothing is known. + :param bounds: the fill levels the storage is held between. """ - elapsed = timedelta(0) - while elapsed < step: - dt = min(FOLLOWED_EVERY, step - elapsed) - element = element_for_fill_level(operation_mode.elements, fill_level) - low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range - fill_level = compute_next_fill_level( - fill_level, - low + (high - low) * factor, - dt, - leakage_behaviour=leakage_behaviour, - usage_rate=( - usage_rate - if usage_rate is not None - else usage_rate_at(usage_forecast, start + elapsed) - ), - ) - if bounds is not None: - fill_level = max(bounds[0], min(bounds[1], fill_level)) - elapsed += dt - return fill_level - - -def usage_rate_at( - usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp -) -> float: - """The usage rate a forecast expects at a moment, per second, as S2 states it. + usage = usage or UsageProfile() + leakage_elements = leakage_behaviour.elements if leakage_behaviour else [] + lowest, highest = bounds if bounds is not None else (-math.inf, math.inf) + boundaries = sorted( + { + edge + for e in list(operation_mode.elements) + list(leakage_elements) + for edge in ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ) + } + | ({lowest, highest} - {-math.inf, math.inf}) + ) - A forecast is a run of elements, each lasting its own duration from where the one before it ended. - Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. - """ - if usage_forecast is None: - return 0.0 - element_start = pd.Timestamp(usage_forecast.start_time) - for element in usage_forecast.elements: - element_end = element_start + timedelta(milliseconds=element.duration.root) - if element_start <= moment < element_end: - return element.usage_rate_expected - element_start = element_end - return 0.0 + def net_rate(mode_element, leakage_element, moment) -> float: + low, high = ( + mode_element.fill_rate.start_of_range, + mode_element.fill_rate.end_of_range, + ) + leakage = leakage_element.leakage_rate if leakage_element is not None else 0.0 + return low + (high - low) * factor - usage.rate_at(moment) - leakage + + moment, end = pd.Timestamp(start), pd.Timestamp(start) + step + fill_level = min(max(fill_level, lowest), highest) + while moment < end: + mode_below, mode_above = _sides(operation_mode.elements, fill_level) + leak_below, leak_above = ( + _sides(leakage_elements, fill_level) if leakage_elements else (None, None) + ) + rising = net_rate(mode_above, leak_above, moment) + falling = net_rate(mode_below, leak_below, moment) + if rising > 0 and fill_level < highest: + rate = rising + elif falling < 0 and fill_level > lowest: + rate = falling + else: + rate = 0.0 # held where it is, at a boundary or a bound, until something changes + horizon = end + change = usage.next_change_after(moment) + if change is not None and change < horizon: + horizon = change + target = None + if rate > 0: + target = next((b for b in boundaries if b > fill_level), None) + elif rate < 0: + target = next((b for b in reversed(boundaries) if b < fill_level), None) + if target is not None: + # Compared in seconds first: a slow rate can put a boundary further off than a timedelta can say. + seconds_to_target = (target - fill_level) / rate + if seconds_to_target < (horizon - moment).total_seconds(): + fill_level = target + moment = moment + pd.Timedelta(seconds=seconds_to_target) + continue + fill_level += rate * (horizon - moment).total_seconds() + moment = horizon + return min(max(fill_level, lowest), highest) def fm_schedule_to_instructions( @@ -202,6 +284,7 @@ def fm_schedule_to_instructions( instructions: list[FRBCInstruction] = [] last_instructed: tuple[str, float] | None = None + forecast_usage = UsageProfile(usage_forecast) fill_level = initial_fill_level has_usage_column = "usage_forecast" in schedule.columns @@ -278,10 +361,14 @@ def fm_schedule_to_instructions( # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: # a NaN fill level would be held at the top of the bounds, and answered there from then on. - usage_rate = None + usage = forecast_usage if has_usage_column: - usage_rate = ( - 0.0 if pd.isna(row["usage_forecast"]) else float(row["usage_forecast"]) + usage = UsageProfile( + rate=( + 0.0 + if pd.isna(row["usage_forecast"]) + else float(row["usage_forecast"]) + ) ) fill_level = follow_operation_mode( fill_level, @@ -290,8 +377,7 @@ def fm_schedule_to_instructions( pd.Timestamp(timestamp), step, leakage_behaviour=leakage_behaviour, - usage_forecast=usage_forecast, - usage_rate=usage_rate, + usage=usage, bounds=(fill_level_min, fill_level_max), ) diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py index 257903a1..eb940673 100644 --- a/tests/s2/test_frbc_time_base.py +++ b/tests/s2/test_frbc_time_base.py @@ -396,7 +396,7 @@ def test_a_schedule_without_values_asks_for_nothing(): # --------------------------------------------------------------------------- from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 - FOLLOWED_EVERY, + UsageProfile, follow_operation_mode, ) @@ -465,7 +465,7 @@ def test_usage_is_followed_through_the_step_not_sampled_at_its_start(): burst = _usage(*[(timedelta(minutes=5), 0.03), (timedelta(minutes=10), 0.0)] * 4) idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) assert follow_operation_mode( - 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=burst + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(burst) ) == pytest.approx(50.0 - 0.03 * 300) @@ -482,7 +482,7 @@ def test_usage_counts_from_where_the_forecast_starts(): ) idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) assert follow_operation_mode( - 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage_forecast=late + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(late) ) == pytest.approx(50.0 - 0.01 * 480) @@ -496,7 +496,7 @@ def test_a_step_that_crosses_into_another_element_is_followed_across_it(): level = follow_operation_mode( 46.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), QUARTER_HOUR ) - assert level == pytest.approx(57.0, abs=0.02 * FOLLOWED_EVERY.total_seconds()) + assert level == pytest.approx(57.0) def test_leakage_is_followed_across_an_element_boundary_too(): @@ -506,7 +506,7 @@ def test_leakage_is_followed_across_an_element_boundary_too(): level = follow_operation_mode( 54.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, leakage_behaviour=leakage ) - assert level == pytest.approx(50.0, abs=0.01 * FOLLOWED_EVERY.total_seconds()) + assert level == pytest.approx(50.0) def test_a_gap_in_a_usage_column_is_no_usage_rather_than_a_buffer_full_to_the_top(): @@ -568,3 +568,158 @@ async def test_the_handler_translates_with_the_latest_usage_forecast(): ) as translate: await frbc.trigger_schedule(system_description.message_id) assert translate.call_args.kwargs["usage_forecast"] is latest + + +# --------------------------------------------------------------------------- +# Followed exactly, between the moments anything changes (second round of review) +# --------------------------------------------------------------------------- + +IDLE = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + + +@pytest.mark.parametrize("offset", [timedelta(0), timedelta(seconds=30)]) +def test_a_draw_shorter_than_a_minute_counts_exactly_once(offset): + """Half-minute draws every two minutes, starting on the minute or off it: a quarter hour holds 7 or 8 of them, exactly.""" + elements = [] + if offset: + elements.append((offset, 0.0)) + while sum((d for d, _ in elements), timedelta(0)) < QUARTER_HOUR: + elements += [(timedelta(seconds=30), 0.1), (timedelta(seconds=90), 0.0)] + pulses = _usage(*elements) + drawn_for = sum( + ( + min(QUARTER_HOUR, start + d) - start + for start, (d, rate) in zip( + [ + sum((e[0] for e in elements[:k]), timedelta(0)) + for k in range(len(elements)) + ], + elements, + ) + if rate and start < QUARTER_HOUR + ), + timedelta(0), + ) + level = follow_operation_mode( + 50.0, IDLE, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(pulses) + ) + assert level == pytest.approx(50.0 - 0.1 * drawn_for.total_seconds()) + + +def test_a_level_held_at_a_boundary_stays_there_rather_than_flipping_across(): + """Filling below 50 and not above, while leaking everywhere: the level settles at 50, and one instruction serves 12 hours.""" + below = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) + above = _element(50, 100, power_top=1000.0, power_bottom=200.0, fill_rate_top=0.0) + leaking = _leakage((0, 100, 0.01)) + instructions = fm_schedule_to_instructions( + _schedule([600.0] * 48), + _system_description_of(_mode(below, above)), + initial_fill_level=40.0, + leakage_behaviour=leaking, + ) + assert len(instructions) <= 2, _factors(instructions) + + +def test_an_element_narrower_than_a_minutes_travel_is_not_skipped(): + """Filling at 1 per second below 50 and not at all above: from 20, the level stops at 50, not 30 past it.""" + stop = _mode( + _element(0, 50, power_top=1000.0, fill_rate_top=1.0), + _element(50, 100, power_top=1000.0, fill_rate_top=0.0), + ) + assert follow_operation_mode( + 20.0, stop, 1.0, pd.Timestamp(START), QUARTER_HOUR + ) == pytest.approx(50.0) + + +def test_a_step_that_is_not_a_whole_number_of_minutes_is_followed_to_its_end(): + level = follow_operation_mode( + 10.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), timedelta(seconds=90) + ) + assert level == pytest.approx(10.0 + 0.02 * 90) + + +def test_a_level_draining_to_the_lower_bound_is_held_there(): + drained = follow_operation_mode( + 5.0, + IDLE, + 0.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(rate=0.01), + bounds=(0.0, 100.0), + ) + assert drained == 0.0 + + +def test_a_small_but_real_change_of_factor_is_instructed(): + """The tolerance absorbs conversion noise, not a change the schedule asks for: 600 W then 605 W is two instructions.""" + modulating = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=1e-6)) + instructions = fm_schedule_to_instructions( + _schedule([600.0, 605.0]), + _system_description_of(modulating), + initial_fill_level=10.0, + ) + assert [round(f, 3) for _, f in _factors(instructions)] == [0.6, 0.605] + + +def test_a_usage_column_takes_precedence_over_a_forecast(): + """A caller that gives both means its column: 4 per quarter hour drawn, not the forecast's nothing.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + with_both = fm_schedule_to_instructions( + frame, + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 0.0)), + ) + assert _factors(with_both)[1] == (START + 2 * QUARTER_HOUR, pytest.approx(0.5)) + + +@pytest.mark.parametrize( + "resolution, duration", + [(timedelta(0), timedelta(hours=12)), (QUARTER_HOUR, timedelta(0))], +) +def test_a_handler_needs_a_positive_resolution_and_duration(resolution, duration): + with pytest.raises(ValueError, match="have to be positive"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + resolution=resolution, + schedule_duration=duration, + ) + + +def test_a_long_schedule_against_a_detailed_forecast_is_translated_quickly(): + """48 hours, at quarter hours, against the largest forecast S2 allows: well under the 1.7 s that stepping each minute took.""" + import time + + detailed = _usage(*[(timedelta(minutes=10), 1e-4 * (i % 3)) for i in range(288)]) + started = time.perf_counter() + fm_schedule_to_instructions( + _schedule([500.0, 0.0] * 96), + _system_description(LOW, HIGH), + initial_fill_level=45.0, + usage_forecast=detailed, + leakage_behaviour=_leakage((0, 50, 1e-4), (50, 100, 2e-4)), + ) + assert time.perf_counter() - started < 0.5 + + +def test_a_level_held_at_its_lower_bound_rises_from_there_once_filling_wins(): + """Drained to 0 in the first 5 minutes, held there while the draw lasts, then filling from 0, not from below it.""" + draw_then_none = _usage((timedelta(minutes=10), 0.02), (timedelta(minutes=5), 0.0)) + filling = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=0.01)) + level = follow_operation_mode( + 3.0, + filling, + 1.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(draw_then_none), + bounds=(0.0, 100.0), + ) + assert level == pytest.approx(0.01 * 300) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index ffbe7098..8363afb0 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -386,8 +386,11 @@ async def test_planning_does_not_stop_on_an_unlabelled_operation_mode( @pytest.mark.asyncio -async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_type): - """S2 states usage per second; TUNES posts it per quarter hour, on a quarter-hour grid, for as long as it was forecast.""" +async def test_a_usage_forecast_is_posted_per_hour(cem_in_frbc_control_type): + """S2 states usage per second; TUNES posts it per hour, labelled MW, on a quarter-hour grid, for as long as it was forecast. + + FlexMeasures reads soc-usage as a power, a rate per hour, so a value per quarter hour would be planned as a quarter of the draw. + """ from s2python.common import Duration from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement @@ -415,7 +418,7 @@ async def test_a_usage_forecast_is_posted_per_quarter_hour(cem_in_frbc_control_t ] assert len(posts) == 1, fm_client.post_sensor_data.await_args_list values = posts[0].kwargs["values"] - assert values == [pytest.approx(0.001 * 15 * 60)] * len(values) + assert values == [pytest.approx(0.001 * 3600)] * len(values) assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) @@ -454,7 +457,7 @@ async def test_a_usage_forecast_is_posted_from_the_quarter_hour_it_starts_in( async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( cem_in_frbc_control_type, ): - """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths, per quarter hour.""" + """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths per second, per hour.""" from s2python.common import Duration from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement @@ -479,14 +482,14 @@ async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( for c in fm_client.post_sensor_data.await_args_list if c.kwargs.get("sensor_id") == 12 ] - assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 15 * 60) + assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 3600) @pytest.mark.asyncio async def test_planning_reads_its_usage_forecast_back_per_second( cem_in_frbc_control_type, ): - """TUNES posts a usage forecast per quarter hour; the schedule translation takes a rate per second, as S2 states it.""" + """TUNES posts a usage forecast per hour; the schedule translation takes a rate per second, as S2 states it.""" from unittest.mock import patch from s2python.common import NumberRange @@ -505,8 +508,8 @@ async def test_planning_reads_its_usage_forecast_back_per_second( def sensor_data(sensor_id, start=None, duration=None, **kwargs): # As many quarter hours as were asked for: the fixture's fixed day of values fits no other window. steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) - # The usage forecast as posted, 900 per quarter hour, and anything else as a plain efficiency. - value = 900.0 if sensor_id == 12 else 1.0 + # The usage forecast as posted, 3600 per hour, and anything else as a plain efficiency. + value = 3600.0 if sensor_id == 12 else 1.0 return { "start": str(start), "duration": str(duration), @@ -526,7 +529,9 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): ) ], ) - frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + older = leakage.model_copy(update={"message_id": get_unique_id()}) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage # the latest # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. fm_client.get_schedule.return_value = {"values": [0.0] * 24} with patch( @@ -537,6 +542,6 @@ def sensor_data(sensor_id, start=None, duration=None, **kwargs): frame = translate.call_args.args[0] usage = list(frame["usage_forecast"].dropna()) - # 900 per quarter hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. + # 3600 per hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. assert usage and usage == [pytest.approx(10.0)] * len(usage) assert translate.call_args.kwargs["leakage_behaviour"] is leakage From 392ba5008d5d4da60a57ca6150e463d28713301c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:05:45 +0100 Subject: [PATCH 23/59] fix(example): follow only the latest plan, keep the forecasts current, and what review found An adversarial review of the example, and of the two library changes it needed, found the example resource manager following every plan it had ever been sent. Each instruction ran in a task of its own, nothing cancelled the tasks of a replaced batch, and the server sent no revocations. In the live demo against a sandboxed FlexMeasures, thirty instructions for the same quarter hour fired at once when it came, one from each replan. The server now sets send_revocations, so a newer plan revokes what is left of the one before, and the resource manager cancels a revoked instruction it has not yet followed, and acknowledges the revocation. Also from the review: - The leakage, the usage forecast and the comfort profile were sent once, at connect, and the storage efficiency was recorded once, for twelve hours from the message, so planning went without leakage after twelve hours and without usage or a band after a day. The resource manager now sends all three again every S2_EXAMPLE_FORECAST_INTERVAL seconds, from then on, and the server seeds its demo prices again every twelve hours. - In the library, a storage efficiency is now recorded each time a schedule is asked for, over the window it covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. On arrival, it covered the wrong hours, at a fill level nobody had reported yet, and as a share of the top of the range it understated the loss at lower fill levels. - align_to_resolution floored the local clock, which in the repeated hour of an autumn DST change took the first pass of that hour, so a schedule could start 67 minutes in the past. It now floors the instant, in UTC. - A state of charge is posted as a reading at an instant, where it was posted over the minute after it, which FlexMeasures stored as a forecast. - The example declares what it does: transitions between its two modes, which S2 lists as the ones possible; no PowerForecast, since it sends none; and a consumer of electricity rather than a storage of it. - Claims corrected: a description that differs from the one before it, its validity included, asks for a schedule, not only one that "changes what the device can do"; and a schedule starts in the current step, which has begun, not "never before now". - The test that held the example's buffer to the CEM's integrator compared two copies of one discretisation. The CEM is now exact and the buffer steps a minute at a time, so the test bounds their difference by a minute of the rate difference at the crossing, which is what two correct discretisations owe each other. Tests cover everything the review's thirteen mutations found unpinned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 9 +- .../s2/control_types/FRBC/__init__.py | 4 +- .../s2/control_types/FRBC/frbc_simple.py | 76 ++++-- .../s2/control_types/translations.py | 12 +- .../s2/script/websockets_client.py | 94 ++++++- .../s2/script/websockets_server.py | 46 +++- tests/s2/test_example_end_to_end.py | 247 +++++++++++++++++- tests/s2/test_frbc_declared_profiles.py | 73 +++++- tests/s2/test_frbc_element_selection.py | 11 +- tests/s2/test_frbc_simple_planning.py | 61 +++++ tests/s2/test_frbc_soft_band.py | 23 ++ 11 files changed, 599 insertions(+), 57 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b9a0de76..7aec9f68 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,7 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, and plans from now [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] Removed ------- @@ -47,6 +47,9 @@ Removed Fixed ----- +* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels [`#259 `__] +* A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] +* A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] * Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] @@ -75,8 +78,8 @@ Fixed * A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] * The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status or changes what the device can do, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from before now, which a description days old made it slide into [`#240 `__, `#259 `__] +* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from a step before the current one, which a description days old made it slide into [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index b4c695e3..39ef719e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -232,8 +232,10 @@ def handle_system_description( # A description says what the device can do, and a storage status what it holds now, # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), - # but a description that arrives after the latest status, or that changes what the device can do, + # but a description that arrives after the latest status, and differs from the description before it, # is answered with a schedule against that latest status. + # Anything but the message id counts as a difference, its validity included, + # so a resource manager that re-dates its description on every resend asks for a schedule every time. # Otherwise a resource manager that sends its status first, or whose fill level holds steady # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. if self._storage_status_history: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 2eb6d535..d6331642 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -214,6 +214,7 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + self._storage_efficiency_recorded: tuple[datetime, float] | None = None if resolution <= timedelta(0) or schedule_duration <= timedelta(0): raise ValueError( f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." @@ -289,17 +290,22 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - seconds_into_the_day = ( - moment.hour * 3600 + moment.minute * 60 + moment.second - ) % resolution_seconds - return moment.replace(microsecond=0) - timedelta(seconds=seconds_into_the_day) + # Floored as an instant, in UTC, and handed back in the moment's own time zone. + # Flooring the local clock instead took the repeated hour of an autumn DST change for its first pass, + # an hour earlier than meant. For Europe/Amsterdam and a resolution dividing an hour the two agree everywhere else. + seconds = int(moment.timestamp()) + floored = datetime.fromtimestamp( + seconds - seconds % resolution_seconds, tz=ZoneInfo("UTC") + ) + return floored.astimezone(moment.tzinfo) def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, never before now, then floored to a boundary. + """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. - Never before now, since planning happens on every storage status while a description may be days old: + Not into an earlier step, since planning happens on every storage status while a description may be days old: anchored to that description alone, every status in a long session was answered with the same window, which slid into the past, so its instructions were all due already and arrived as one burst. + The step now falls in has begun already, so its first instruction is due at once. """ return self.align_to_resolution( max(valid_from + self._valid_from_shift, self.now()) @@ -315,7 +321,8 @@ async def send_storage_status( # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, - duration=timedelta(minutes=1), + # A reading at an instant, not over the minute after it, which FlexMeasures took for a forecast. + duration=timedelta(0), ) # The state of charge this status reports is what a schedule starts from, so this is @@ -861,7 +868,7 @@ async def trigger_schedule( ) if self._usage_forecast_sensor_id is not None: flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] - if self._leakage_behaviour_sensor_id is not None: + if await self._record_storage_efficiency(start, fill_level_at_start): flex_model["storage-efficiency"] = { "sensor": self._leakage_behaviour_sensor_id } @@ -1041,28 +1048,53 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): ) async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): - """Record what a resource manager's storage loses on its own, as a storage efficiency.""" + """Keep a resource manager's leakage behaviour for planning, which records it when it asks for a schedule. + + Recorded on arrival instead, it covered the hours after the message rather than the window being planned, + at a fill level nobody had reported yet, and planning went on without it once those hours had passed. + """ if self._leakage_behaviour_sensor_id is None: self._logger.info( "Not recording an FRBC.LeakageBehaviour: no leakage-behaviour sensor is configured." ) - return + async def _record_storage_efficiency( + self, start: datetime, fill_level: float + ) -> bool: + """Record what the storage keeps of itself per step, over the window a schedule covers, from where it starts. + + :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. + """ + leakage_behaviour = self._latest(self._leakage_behaviour_history) + if self._leakage_behaviour_sensor_id is None or leakage_behaviour is None: + return False # What remains after one step of this handler's resolution, # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. - storage_efficiency = leakage_behaviour_to_storage_efficiency( + efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, fill_level_scale=self._fill_level_scale, - fill_level=self._latest_fill_level(), - ) - - now = self.now() - await self._fm_client.post_sensor_data( - sensor_id=self._leakage_behaviour_sensor_id, - start=self.align_to_resolution(leakage_behaviour.valid_from or now), - prior=now, - values=[storage_efficiency], - unit="%", - duration=self._schedule_duration, + fill_level=fill_level, + share_of=fill_level, ) + if self._storage_efficiency_recorded == (start, efficiency): + return True + try: + await self._fm_client.post_sensor_data( + sensor_id=self._leakage_behaviour_sensor_id, + start=start, + prior=self.now(), + values=[efficiency], + unit="%", + duration=self._schedule_duration, + ) + except (JobFailedError, ValueError, ConnectionError): + self._logger.exception( + "Not asking for a schedule against a storage efficiency: could not record %s %% on sensor %s for %s.", + efficiency, + self._leakage_behaviour_sensor_id, + start, + ) + return False + self._storage_efficiency_recorded = (start, efficiency) + return True diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index f12b2d01..e61744db 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -23,6 +23,7 @@ def leakage_behaviour_to_storage_efficiency( resolution=timedelta(minutes=15), fill_level_scale: float = 1, fill_level: float | None = None, + share_of: float | None = None, ) -> float: """ Convert a FRBC.LeakageBehaviour message into a FlexMeasures compatible storage efficiency. @@ -73,9 +74,16 @@ def leakage_behaviour_to_storage_efficiency( # This used to compound it over the hours in a step instead, # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. - # The loss is taken as a share of the top of the range, as before, which understates it at lower fill levels. + # FlexMeasures applies a storage efficiency to the stock as it stands, so the loss is a share of a fill level: + # of share_of where given, as the fill level a schedule starts from is, + # and otherwise of the top of the range, which understates it at lower fill levels. + reference = ( + share_of * fill_level_scale + if share_of is not None and share_of > 0 + else max_fill_level + ) seconds_per_step = pd.Timedelta(resolution) / pd.Timedelta(seconds=1) - storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** seconds_per_step + storage_efficiency = 100 * max(0.0, 1 - leakage / reference) ** seconds_per_step return storage_efficiency diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 3413e4b4..e17b7e37 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -29,8 +29,10 @@ ReceptionStatus, ReceptionStatusValues, ResourceManagerDetails, + RevokeObject, Role, RoleType, + Transition, ) from s2python.frbc import ( FRBCActuatorDescription, @@ -60,6 +62,8 @@ PHYSICAL_CAPACITY = 12.0 #: How often the simulated buffer reports its fill level, in seconds of wall-clock time. STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) +#: How often the forecasts are sent again, each covering the day from then on, so that planning never runs out of them. +FORECAST_INTERVAL = float(os.getenv("S2_EXAMPLE_FORECAST_INTERVAL", "3600")) ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC @@ -116,6 +120,21 @@ def operation_modes() -> list[FRBCOperationMode]: def system_description( modes: list[FRBCOperationMode], valid_from: datetime ) -> FRBCSystemDescription: + # S2 lists the transitions an actuator can make, so the heat pump says it can go from either mode to the other, + # at once and at no cost. + transitions = [ + Transition( + id=get_unique_id(), + from_=a.id, + to=b.id, + start_timers=[], + blocking_timers=[], + abnormal_condition_only=False, + ) + for a in modes + for b in modes + if a.id != b.id + ] return FRBCSystemDescription( message_id=get_unique_id(), valid_from=valid_from, @@ -125,7 +144,7 @@ def system_description( diagnostic_label="heat pump", supported_commodities=[Commodity.ELECTRICITY], operation_modes=modes, - transitions=[], + transitions=transitions, timers=[], ) ], @@ -303,13 +322,31 @@ async def apply_at_execution_time( async def answer_instructions(ws, buffer: Buffer | None = None) -> None: - """Acknowledge every instruction, and report the transition each one asks for when it is due.""" + """Acknowledge every instruction, and report the transition each one asks for when it is due. + + An instruction the CEM revokes, because a newer plan replaces the one it belonged to, is not followed: + without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. + """ active_operation_mode_id = None + pending: dict[str, asyncio.Task] = {} async for msg in ws: payload = msg.json() print("RECEIVED: ", payload) - if payload.get("message_type") != "FRBC.Instruction": + message_type = payload.get("message_type") + if message_type == "RevokeObject": + revocation = RevokeObject.model_validate(payload) + task = pending.pop(str(revocation.object_id), None) + if task is not None: + task.cancel() + await ws.send_str( + ReceptionStatus( + subject_message_id=revocation.message_id, + status=ReceptionStatusValues.OK, + ).model_dump_json() + ) + continue + if message_type != "FRBC.Instruction": continue instruction = FRBCInstruction.model_validate(payload) await ws.send_str( @@ -319,9 +356,16 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: ).model_dump_json() ) # In a task of its own, so that an instruction due later does not hold up the ones behind it. - asyncio.create_task( + task = asyncio.create_task( apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) ) + key = str(instruction.id) + pending[key] = task + + def forget(done: asyncio.Task, key: str = key) -> None: + pending.pop(key, None) + + task.add_done_callback(forget) active_operation_mode_id = instruction.operation_mode @@ -340,6 +384,27 @@ async def report_fill_level( ) +async def refresh_forecasts( + ws, leakage: FRBCLeakageBehaviour, interval: float = FORECAST_INTERVAL +) -> None: + """Send the leakage, the usage forecast and the comfort profile again every interval, each from now on. + + Sent only once, they would cover the first day, and the CEM would plan without them from then on. + """ + while True: + await asyncio.sleep(interval) + now = datetime.now(timezone.utc) + for message in ( + leakage.model_copy( + update={"message_id": get_unique_id(), "valid_from": now} + ), + usage_forecast(now), + fill_level_target_profile(now), + ): + print(f"SENDING: {message.message_type}") + await ws.send_str(message.model_dump_json()) + + async def receive_until(ws, message_type: str) -> dict: """The next message of a type, past the reception statuses that arrive in between.""" while True: @@ -349,7 +414,9 @@ async def receive_until(ws, message_type: str) -> dict: return message -async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: +async def run_resource_manager( + ws, interval: float = STATUS_INTERVAL, forecast_interval: float = FORECAST_INTERVAL +) -> Buffer: """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. In the order S2 suggests: the system description first, then the storage's status. @@ -369,13 +436,18 @@ async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: message_id=get_unique_id(), resource_id=get_unique_id(), name="heat pump with a hot-water buffer", - roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + # It draws electricity and stores heat, so it is a consumer of the commodity it is planned in. + roles=[ + Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY) + ], instruction_processing_delay=Duration(root=1000), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, ], - provides_forecast=True, + # It sends no PowerForecast. It could measure its power, which is what the next field says, + # and S2 requires at least one type there. + provides_forecast=False, provides_power_measurement_types=[ELECTRIC_POWER], ).model_dump_json() ) @@ -402,11 +474,15 @@ async def run_resource_manager(ws, interval: float = STATUS_INTERVAL) -> Buffer: print(f"SENDING: {message.message_type}") await ws.send_str(message.model_dump_json()) - reporting = asyncio.create_task(report_fill_level(ws, buffer, interval)) + background = [ + asyncio.create_task(report_fill_level(ws, buffer, interval)), + asyncio.create_task(refresh_forecasts(ws, leakage, forecast_interval)), + ] try: await answer_instructions(ws, buffer) finally: - reporting.cancel() + for task in background: + task.cancel() return buffer diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 175cc7d7..9b1a8b5f 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -24,9 +24,10 @@ SITE_NAME = "My CEM" SITE_TIMEZONE = "Europe/Amsterdam" -# Enough days of demo prices to cover what the CEM plans for: a schedule starts a day after the -# system description that asked for it and runs for twelve hours, so two days would just do. +# Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, +# with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 +SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. #: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. @@ -152,6 +153,8 @@ async def serve_resource_manager( # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. valid_from_shift=timedelta(0), ) + # A newer plan revokes what is left of the one before, so the resource manager does not follow both. + frbc.send_revocations = True cem.register_control_type(frbc) # create "parallel" tasks for the message producer and consumer @@ -209,11 +212,18 @@ async def configure_site( **({"attributes": attributes} if attributes else {}), ) - # Demo prices, so that a freshly configured site has something to schedule against. - # One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - # plans ahead of the moment a resource manager reports its storage: - # a fixed date here goes stale the day after it is written, and the scheduler then fails the - # job with UnknownPricesException rather than returning a schedule. + await seed_prices(fm_client, sensors["price"]["id"]) + return sensors + + +async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: + """Demo prices, so that a site has something to schedule against. + + One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM + plans ahead of the moment a resource manager reports its storage: + a fixed date here goes stale the day after it is written, and the scheduler then fails the + job with UnknownPricesException rather than returning a schedule. + """ one_day = [ 0.10, 0.11, 0.12, 0.15, 0.18, 0.17, 0.11, 0.09, 0.10, 0.09, 0.09, 0.10, 0.08, 0.05, 0.04, 0.04, @@ -223,13 +233,30 @@ async def configure_site( hour=0, minute=0, second=0, microsecond=0 ) await fm_client.post_sensor_data( - sensor_id=sensors["price"]["id"], + sensor_id=price_sensor_id, start=today, duration=timedelta(days=SEED_PRICE_DAYS), values=one_day * SEED_PRICE_DAYS, unit="EUR/kWh", ) - return sensors + + +async def keep_prices_seeded(app: web.Application): + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them.""" + + async def reseed(): + while True: + await asyncio.sleep(SEED_PRICES_EVERY.total_seconds()) + try: + await seed_prices(app[FM_CLIENT], app[SENSORS]["price"]["id"]) + except (ConnectionError, ValueError): + LOGGER.exception( + "Could not seed the demo prices again; trying at the next interval." + ) + + task = asyncio.create_task(reseed()) + yield + task.cancel() async def configure_site_once(app: web.Application) -> None: @@ -258,6 +285,7 @@ async def close_fm_client(app: web.Application) -> None: app = web.Application() app.on_startup.append(configure_site_once) +app.cleanup_ctx.append(keep_prices_seeded) app.on_cleanup.append(close_fm_client) app.add_routes([web.get("/ws", websocket_handler)]) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 9b97b64e..bc4741e6 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -121,7 +121,9 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): tasks = [ asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), - asyncio.create_task(run_resource_manager(resource_manager, interval=0.3)), + asyncio.create_task( + run_resource_manager(resource_manager, interval=0.3, forecast_interval=0.3) + ), ] try: # Two storage statuses past the first, each a new plan, so that replanning is part of what is checked. @@ -144,6 +146,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): refused = [ m for m in resource_manager.of_type("ReceptionStatus") if m["status"] != "OK" ] + assert len(cem.of_type("FRBC.UsageForecast")) >= 2 # sent again, not only once assert refused == [] start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] @@ -319,8 +322,15 @@ async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_toda def test_the_example_and_the_cem_agree_across_an_element_boundary(): - """Both follow a step a minute at a time, so from 5 kWh, through the cold element and into the warm one, they agree.""" - from flexmeasures_client.s2.control_types.FRBC.utils import follow_operation_mode + """The CEM follows a step exactly, the example's buffer a minute at a time: two discretisations of S2's definitions. + + Through the cold element and into the warm one they can differ only within the minute of the crossing, + by at most a minute of the difference between the two elements' fill rates. + """ + from flexmeasures_client.s2.control_types.FRBC.utils import ( + UsageProfile, + follow_operation_mode, + ) buffer = _heating_buffer(5.0) heating = buffer.modes[buffer.mode_id] @@ -333,10 +343,237 @@ def test_the_example_and_the_cem_agree_across_an_element_boundary(): pd.Timestamp(NOON + step * timedelta(minutes=15)), timedelta(minutes=15), leakage_behaviour=leakage_behaviour(NOON), - usage_forecast=usage_forecast(NOON), + usage=UsageProfile(usage_forecast(NOON)), ) buffer.advance(NOON + timedelta(hours=1)) assert ( cem_fill_level > 6.0 ) # the boundary between the cold and the warm element was crossed - assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) + a_minute_of_the_difference = (heat(6000) - heat(4500)) * 60 + assert buffer.fill_level == pytest.approx( + cem_fill_level, abs=a_minute_of_the_difference + ) + + +# --------------------------------------------------------------------------- +# What the review of #259 found unpinned +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + COLD_BELOW, + PHYSICAL_CAPACITY, + answer_instructions, + expected_usage, + fill_level_target_profile, + refresh_forecasts, + report_fill_level, + system_description, +) + + +class _Collect: + """A websocket that collects what is sent and hands over what it is given, then ends.""" + + def __init__(self, incoming: list[str] | None = None): + self.sent: list[dict] = [] + self._incoming = list(incoming or []) + + async def send_str(self, data: str) -> None: + self.sent.append(json.loads(data)) + + def __aiter__(self): + async def messages(): + for data in self._incoming: + yield _Text(data) + await asyncio.sleep(0) + + return messages() + + +def test_the_forecast_the_example_sends_is_the_usage_it_simulates(): + """Hour by hour, through a whole day, so that what is planned for is what is drawn.""" + for start_hour in (0, 13): + start = datetime(2024, 1, 1, start_hour, tzinfo=timezone.utc) + forecast = usage_forecast(start) + for h, element in enumerate(forecast.elements): + assert element.usage_rate_expected == pytest.approx( + per_second(expected_usage((start_hour + h) % 24)) + ) + + +def test_the_comfort_profile_preheats_before_the_morning_showers(): + profile = fill_level_target_profile(datetime(2024, 1, 1, tzinfo=timezone.utc)) + lows = [e.fill_level_range.start_of_range for e in profile.elements] + assert lows[6] == 6.0 and expected_usage(7) > expected_usage(6) + assert set(lows[:6] + lows[7:]) == {3.0} + + +def test_at_the_boundary_the_buffer_heats_at_the_warm_coefficient(): + buffer = _heating_buffer(COLD_BELOW) + fill, _, _ = buffer.rates(NOON) + assert fill == pytest.approx(heat(4500)) + + +def test_the_buffer_never_holds_more_than_it_physically_can(): + buffer = _heating_buffer(PHYSICAL_CAPACITY - 0.1) + buffer.advance(NOON + timedelta(hours=2)) + assert buffer.fill_level == PHYSICAL_CAPACITY + + +def test_the_example_declares_what_it_sends_and_how_it_can_switch(): + modes = operation_modes() + description = system_description(modes, NOON) + storage = description.storage + assert storage.provides_leakage_behaviour and storage.provides_usage_forecast + assert storage.provides_fill_level_target_profile + switches = {(str(t.from_), str(t.to)) for t in description.actuators[0].transitions} + ids = [str(m.id) for m in modes] + assert switches == {(ids[0], ids[1]), (ids[1], ids[0])} + + +@pytest.mark.asyncio +async def test_the_reported_fill_level_is_where_the_buffer_has_got_to(): + buffer = _heating_buffer(5.0) + buffer.at = datetime.now(timezone.utc) - timedelta(minutes=30) + ws = _Collect() + reporting = asyncio.create_task(report_fill_level(ws, buffer, interval=0.01)) + await _until(lambda: ws.sent) + reporting.cancel() + assert ( + ws.sent[0]["present_fill_level"] > 5.5 + ) # half an hour of heating, not where it started + + +@pytest.mark.asyncio +async def test_a_revoked_instruction_is_not_followed(): + """A newer plan replaced it; following it anyway left the heat pump in whatever the oldest plan said last.""" + from s2python.common import RevokableObjects, RevokeObject + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + due_soon = FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=get_unique_id(), + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + timedelta(milliseconds=200), + abnormal_condition=False, + ) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=due_soon.id, + ) + ws = _Collect([due_soon.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await asyncio.sleep(0.4) + assert not [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + acknowledged = { + m["subject_message_id"] + for m in ws.sent + if m["message_type"] == "ReceptionStatus" + } + assert str(revoke.message_id) in acknowledged + + +@pytest.mark.asyncio +async def test_the_forecasts_are_sent_again_from_now_on(): + ws = _Collect() + refreshing = asyncio.create_task( + refresh_forecasts(ws, leakage_behaviour(NOON), interval=0.01) + ) + await _until(lambda: len(ws.sent) >= 3) + refreshing.cancel() + assert [m["message_type"] for m in ws.sent[:3]] == [ + "FRBC.LeakageBehaviour", + "FRBC.UsageForecast", + "FRBC.FillLevelTargetProfile", + ] + assert pd.Timestamp(ws.sent[0]["valid_from"]) > NOON + + +@pytest.mark.asyncio +async def test_the_server_plans_from_now_in_the_site_time_zone_and_revokes_what_it_replaces(): + from unittest.mock import patch + + made = [] + real = server.FRBCSimple + + def capture(**kwargs): + frbc = real(**kwargs) + made.append((kwargs, frbc)) + return frbc + + resource_manager, cem = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "FRBCSimple", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem, _flexmeasures(), sensors) + ) + await _until(lambda: made) + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + kwargs, frbc = made[0] + assert kwargs["timezone"] == server.SITE_TIMEZONE + assert kwargs["valid_from_shift"] == timedelta(0) + assert frbc.send_revocations is True + + +def test_the_site_sensors_are_what_the_cem_records_in(): + assert server.SITE_SENSORS["state of charge"][:2] == ("kWh", "PT0M") + assert server.SITE_SENSORS["storage efficiency"][0] == "%" + assert server.SITE_SENSORS["usage forecast"][0] == "kWh/s" + + +@pytest.mark.asyncio +async def test_the_demo_prices_are_seeded_again_while_the_server_runs(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + assert fm.post_sensor_data.await_args.kwargs["sensor_id"] == 10 + + +def test_the_loss_is_a_share_of_the_fill_level_a_schedule_starts_from(): + """1.5 kWh a day from a 5 kWh buffer is a larger share of it than of a 12 kWh one.""" + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + leakage = leakage_behaviour(NOON) + at_five = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=7.0, share_of=5.0 + ) + rate = per_second(1.5 / 24) # the warm element, which applies at 7 + assert at_five == pytest.approx(100 * (1 - rate / 5.0) ** 900, rel=1e-12) + + +@pytest.mark.asyncio +async def test_a_failed_reseed_is_tried_again_at_the_next_interval(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index e783d876..b1af0a3b 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -9,8 +9,9 @@ from __future__ import annotations +import asyncio import logging -from datetime import timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch import pandas as pd @@ -265,6 +266,7 @@ async def test_usage_forecast_is_recorded_from_a_resolution_boundary( async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( system_description, ): + """Recorded when a schedule is asked for, over the window it covers, rather than on arrival.""" frbc = _frbc(system_description, with_sensors=True) leakage = FRBCLeakageBehaviour( message_id=get_unique_id(), @@ -279,12 +281,22 @@ async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( ], ) - await frbc.send_leakage_behaviour(leakage) + await frbc.handle_message(leakage) + await asyncio.gather(*frbc.background_tasks) + assert not [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] # nothing on arrival + + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, (RANGE_BOTTOM + RANGE_TOP) / 2) posted = _posted(frbc, LEAKAGE_SENSOR_ID) assert posted["unit"] == "%" assert len(posted["values"]) == 1 - assert posted["start"].minute % 15 == 0 and posted["start"].second == 0 + assert posted["start"] == start + assert posted["duration"] == frbc._schedule_duration # --- Without the sensors: nothing recorded, and still no exception --- @@ -601,3 +613,58 @@ async def test_an_actuator_s_fill_rate_is_posted_in_the_fill_rate_unit( ) # The power sensor really does carry a power, and is untouched by this. assert _posted(frbc, 1)["unit"] == frbc.power_unit + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_already_recorded_for_the_window_is_not_recorded_again( + system_description, +): + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) + assert await frbc._record_storage_efficiency(start, 50.0) + posts = [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_that_could_not_be_recorded_is_not_planned_against( + system_description, caplog +): + """And it is tried again next time, rather than remembered as recorded.""" + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + frbc._fm_client.post_sensor_data.side_effect = ConnectionError("unreachable") + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) is False + assert frbc._storage_efficiency_recorded is None + assert "could not record" in caplog.text diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 06ef021c..5909b4a1 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -525,8 +525,10 @@ def test_the_storage_efficiency_at_a_fill_level_is_that_elements(): async def test_the_handler_records_leakage_at_the_latest_fill_level(): leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) - _report_fill_level(frbc, 20.0) - await frbc.send_leakage_behaviour(leakage) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), 20.0 + ) posted = [ call.kwargs["values"] for call in frbc._fm_client.post_sensor_data.await_args_list @@ -535,7 +537,10 @@ async def test_the_handler_records_leakage_at_the_latest_fill_level(): assert posted == [ [ leakage_behaviour_to_storage_efficiency( - leakage, resolution=timedelta(minutes=15), fill_level=20.0 + leakage, + resolution=timedelta(minutes=15), + fill_level=20.0, + share_of=20.0, ) ] ] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index e3c0a745..f8b9f8d3 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -334,3 +334,64 @@ async def test_the_soc_is_not_reposted_when_a_description_asks_for_a_schedule(): await settle(frbc) assert frbc._fm_client.post_measurements.await_count == posts_after_the_status + + +def test_a_step_boundary_in_the_repeated_autumn_hour_is_not_an_hour_early(): + """At 02:07 the second time round, on the night the clocks go back, the step began at 02:00 of that second pass.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 7, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + aligned = frbc.align_to_resolution(second_pass) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +@pytest.mark.asyncio +async def test_a_plan_asked_for_by_a_late_description_supersedes_one_in_flight(): + """A status's plan is still out when a changed description asks for its own: only the newer batch goes out.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + assert sent[2] > 0 # the description's plan, which is the newer request + assert sent[1] == 0 # the status's plan, overtaken while it was out + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_recorded_as_a_reading_at_an_instant(): + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + assert frbc._fm_client.post_measurements.await_args.kwargs["duration"] == timedelta( + 0 + ) diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 1aa734bd..64055d78 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -156,7 +156,20 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned system_description, ): """A storage that is drawn from and leaks is not planned for by its prices alone.""" + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + frbc = _frbc(system_description) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T00:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage flex_model, _ = await _asked_for(frbc, system_description) @@ -164,6 +177,16 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned assert flex_model["storage-efficiency"] == {"sensor": LEAKAGE_SENSOR_ID} +@pytest.mark.asyncio +async def test_no_storage_efficiency_is_named_before_a_leakage_behaviour_is_known( + system_description, +): + """A sensor holding nothing for the window would be planned against as a storage that keeps everything.""" + frbc = _frbc(system_description) + flex_model, _ = await _asked_for(frbc, system_description) + assert "storage-efficiency" not in flex_model + + @pytest.mark.asyncio async def test_the_hard_bounds_are_wider_than_the_declared_range( system_description, From e3a556755bf483ce81a11e229885668ceb1ee65c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:33:43 +0100 Subject: [PATCH 24/59] fix(example, FRBC): what the second review of the example round found - The example now tells the CEM where each instruction stands: accepted on arrival, succeeded once carried out, revoked once revoked. Without that every instruction stayed NEW, so each replan revoked the whole previous batch, the instructions already carried out included. - The mode a transition reports leaving is the one the last carried-out instruction put the heat pump in. It was the mode the previous instruction to arrive asked for, which after a revocation the heat pump was never in. - A storage efficiency takes the loss as a share of the start fill level or of the middle of the storage's range, whichever is larger. A share of a near-empty start left almost nothing per step, and overstated the loss many times over once the plan filled the storage. - A state of charge is posted as an instant only on a sensor that records instants: FlexMeasures refuses a single value of no duration on a sensor with a resolution, so there it is posted over a minute, as before. - align_to_resolution floors on the moment's own clock, which is the grid FlexMeasures floors to, with the pass of an autumn DST hour taken from the moment. Flooring in UTC put hourly steps half an hour off in zones that are, and daily steps on the wrong midnight. - The example server's price reseeding survives a failed ingestion job, a wrong content type and a wrong API version, not only a lost connection. - Statuses about different steps that interleave within a second get belief times a second apart per step, where only the latest status was compared. The demo gate's first run had thirty statuses for one quarter hour arrive at once, and one was refused as a replacement. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 92 +++++++++++---- .../s2/script/websockets_client.py | 39 +++++-- .../s2/script/websockets_server.py | 13 ++- tests/s2/test_example_end_to_end.py | 108 ++++++++++++++++++ tests/s2/test_frbc_element_selection.py | 55 ++++++++- tests/s2/test_frbc_simple_planning.py | 60 +++++++++- 7 files changed, 334 insertions(+), 35 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 7aec9f68..d343b2ef 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -47,7 +47,7 @@ Removed Fixed ----- -* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels [`#259 `__] +* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] * A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index d6331642..a266a8cc 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -27,7 +27,7 @@ import pandas as pd from flexmeasures_client.client import convert_units -from flexmeasures_client.exceptions import JobFailedError +from flexmeasures_client.exceptions import ContentTypeError, JobFailedError from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( clip_fill_level_target_profile, @@ -144,7 +144,7 @@ class FRBCSimple(FRBC): _power_sensor_takes_consumption_as_positive: bool | None #: When a power value was last recorded for the actuator, and for which moment, #: so that a second status inside one event does not collide with the first. - _last_power_recorded: tuple[datetime, datetime] | None + _power_recorded_at: dict[datetime, datetime] #: The charging efficiency last recorded, and the window it was recorded for, #: so that the same value for the same window is not recorded again on every schedule. _charging_efficiency_recorded: tuple[datetime, float] | None @@ -243,7 +243,8 @@ def __init__( ) self._fill_level_scale = fill_level_scale self._power_sensor_takes_consumption_as_positive = None - self._last_power_recorded = None + self._power_recorded_at = {} + self._soc_sensor_records_instants: bool | None = None if hard_bound_factor < 1: # Below one, the hard upper bound sits inside the declared range, # and FlexMeasures clips the band into the hard bounds, @@ -290,14 +291,17 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - # Floored as an instant, in UTC, and handed back in the moment's own time zone. - # Flooring the local clock instead took the repeated hour of an autumn DST change for its first pass, - # an hour earlier than meant. For Europe/Amsterdam and a resolution dividing an hour the two agree everywhere else. - seconds = int(moment.timestamp()) - floored = datetime.fromtimestamp( - seconds - seconds % resolution_seconds, tz=ZoneInfo("UTC") + # Floored on the moment's own clock, which is the grid FlexMeasures floors a series to, + # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. + # In the repeated hour of an autumn DST change, the pass the moment is in decides which of the two boundaries it is, + # where an earlier version took the first pass every time, an hour earlier than meant. + # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. + floored = pd.Timestamp(moment).floor( + self._resolution, + ambiguous=moment.fold == 0, + nonexistent="shift_forward", ) - return floored.astimezone(moment.tzinfo) + return floored.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. @@ -311,6 +315,34 @@ def schedule_start(self, valid_from: datetime) -> datetime: max(valid_from + self._valid_from_shift, self.now()) ) + async def _soc_duration(self) -> timedelta: + """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. + + Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. + FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, + so on any other sensor, or one that cannot be read, it is posted over a minute, as before. + The sensor is asked once, since its resolution is a property of how it was set up. + """ + if self._soc_sensor_records_instants is None: + try: + sensor = await self._fm_client.get_sensor(self._soc_sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not read the resolution of state-of-charge sensor %s, so a fill level is posted over a minute: %s", + self._soc_sensor_id, + exc, + ) + return timedelta(minutes=1) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + self._soc_sensor_records_instants = resolution is not None and pd.Timedelta( + resolution + ) == pd.Timedelta(0) + return ( + timedelta(0) if self._soc_sensor_records_instants else timedelta(minutes=1) + ) + async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): @@ -321,8 +353,7 @@ async def send_storage_status( # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, - # A reading at an instant, not over the minute after it, which FlexMeasures took for a forecast. - duration=timedelta(0), + duration=await self._soc_duration(), ) # The state of charge this status reports is what a schedule starts from, so this is @@ -513,15 +544,21 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # and this handler sends several at once, so that is the ordinary case rather than an edge. # The comparison is on the event as FlexMeasures keys it, # since two statuses seconds apart still land on the same resolution step. + # Kept per event, since statuses for different steps can interleave within a second, + # and a comparison with the latest status alone then missed the one before it about the same step. event = self.align_to_resolution(start) recorded_at = self.now() - if self._last_power_recorded is not None: - last_event, last_recorded_at = self._last_power_recorded - if last_event == event and recorded_at - last_recorded_at < timedelta( - seconds=1 - ): - recorded_at = last_recorded_at + timedelta(seconds=1) - self._last_power_recorded = (event, recorded_at) + last_recorded_at = self._power_recorded_at.get(event) + if last_recorded_at is not None and recorded_at - last_recorded_at < timedelta( + seconds=1 + ): + recorded_at = last_recorded_at + timedelta(seconds=1) + self._power_recorded_at[event] = recorded_at + # Steps more than a day older than this one are forgotten, to keep this bounded: a status that late is not expected. + for old_event in [ + e for e in self._power_recorded_at if e < event - timedelta(days=1) + ]: + del self._power_recorded_at[old_event] # The value and the unit it is in travel together, # so that what is reported below is a pair that agrees even where the conversion is what failed. @@ -1066,8 +1103,21 @@ async def _record_storage_efficiency( :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. """ leakage_behaviour = self._latest(self._leakage_behaviour_history) - if self._leakage_behaviour_sensor_id is None or leakage_behaviour is None: + if ( + self._leakage_behaviour_sensor_id is None + or leakage_behaviour is None + or not self._system_description_history + ): return False + # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate + # only at one fill level, and the loss scales with the stock away from it. + # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: + # from a near-empty start, a share of the start level would let almost nothing remain per step, + # and the loss would be overstated many times over as soon as the plan filled the storage. + storage_range = list(self._system_description_history.values())[ + -1 + ].storage.fill_level_range + middle = (storage_range.start_of_range + storage_range.end_of_range) / 2 # What remains after one step of this handler's resolution, # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. efficiency = leakage_behaviour_to_storage_efficiency( @@ -1075,7 +1125,7 @@ async def _record_storage_efficiency( resolution=self._resolution, fill_level_scale=self._fill_level_scale, fill_level=fill_level, - share_of=fill_level, + share_of=max(fill_level, middle), ) if self._storage_efficiency_recorded == (start, efficiency): return True diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index e17b7e37..ad7b646e 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -24,6 +24,8 @@ Duration, EnergyManagementRole, Handshake, + InstructionStatus, + InstructionStatusUpdate, NumberRange, PowerRange, ReceptionStatus, @@ -284,10 +286,24 @@ def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: self.factor = instruction.operation_mode_factor +async def send_instruction_status( + ws, instruction_id, status: InstructionStatus +) -> None: + """Tell the CEM where an instruction stands, which is what it revokes by: only one not yet carried out.""" + await ws.send_str( + InstructionStatusUpdate( + message_id=get_unique_id(), + instruction_id=instruction_id, + status_type=status, + timestamp=datetime.now(timezone.utc), + ).model_dump_json() + ) + + async def apply_at_execution_time( ws, instruction: FRBCInstruction, - previous_operation_mode_id=None, + applied: dict | None = None, buffer: Buffer | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -297,6 +313,9 @@ async def apply_at_execution_time( A resource manager that answers on arrival reports a mode it is not in yet, and a customer energy manager recording that status has a measurement of a power the device is not drawing, timestamped now. + + The mode it reports leaving is the one the last instruction carried out put it in, kept in applied, + rather than whatever the instruction before it asked for, which may have been revoked and never carried out. """ delay = (instruction.execution_time - datetime.now(timezone.utc)).total_seconds() @@ -312,13 +331,16 @@ async def apply_at_execution_time( actuator_id=instruction.actuator_id, active_operation_mode_id=instruction.operation_mode, operation_mode_factor=instruction.operation_mode_factor, - previous_operation_mode_id=previous_operation_mode_id, + previous_operation_mode_id=(applied or {}).get("mode"), # The moment the transition happened, so that what is recorded about it is placed there, # rather than wherever the message is read. transition_timestamp=now, ) + if applied is not None: + applied["mode"] = instruction.operation_mode print("SENDING: FRBC.ActuatorStatus") await ws.send_str(status.model_dump_json()) + await send_instruction_status(ws, instruction.id, InstructionStatus.SUCCEEDED) async def answer_instructions(ws, buffer: Buffer | None = None) -> None: @@ -328,7 +350,7 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. """ - active_operation_mode_id = None + applied: dict = {"mode": None} pending: dict[str, asyncio.Task] = {} async for msg in ws: payload = msg.json() @@ -337,14 +359,17 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: if message_type == "RevokeObject": revocation = RevokeObject.model_validate(payload) task = pending.pop(str(revocation.object_id), None) - if task is not None: - task.cancel() await ws.send_str( ReceptionStatus( subject_message_id=revocation.message_id, status=ReceptionStatusValues.OK, ).model_dump_json() ) + if task is not None: + task.cancel() + await send_instruction_status( + ws, revocation.object_id, InstructionStatus.REVOKED + ) continue if message_type != "FRBC.Instruction": continue @@ -355,9 +380,10 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: status=ReceptionStatusValues.OK, ).model_dump_json() ) + await send_instruction_status(ws, instruction.id, InstructionStatus.ACCEPTED) # In a task of its own, so that an instruction due later does not hold up the ones behind it. task = asyncio.create_task( - apply_at_execution_time(ws, instruction, active_operation_mode_id, buffer) + apply_at_execution_time(ws, instruction, applied, buffer) ) key = str(instruction.id) pending[key] = task @@ -366,7 +392,6 @@ def forget(done: asyncio.Task, key: str = key) -> None: pending.pop(key, None) task.add_done_callback(forget) - active_operation_mode_id = instruction.operation_mode async def report_fill_level( diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 9b1a8b5f..49d81d82 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -12,6 +12,11 @@ from s2python.common import ControlType from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ( + ContentTypeError, + JobFailedError, + WrongAPIVersionError, +) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple @@ -249,7 +254,13 @@ async def reseed(): await asyncio.sleep(SEED_PRICES_EVERY.total_seconds()) try: await seed_prices(app[FM_CLIENT], app[SENSORS]["price"]["id"]) - except (ConnectionError, ValueError): + except ( + ConnectionError, + ValueError, + ContentTypeError, + JobFailedError, + WrongAPIVersionError, + ): LOGGER.exception( "Could not seed the demo prices again; trying at the next interval." ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index bc4741e6..3ef96f46 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -577,3 +577,111 @@ async def test_a_failed_reseed_is_tried_again_at_the_next_interval(): await _until(lambda: fm.post_sensor_data.await_count >= 2) with pytest.raises(StopAsyncIteration): await seeding.__anext__() + + +def _instruction(mode, due_in: timedelta): + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + return FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=mode, + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + due_in, + abnormal_condition=False, + ) + + +@pytest.mark.asyncio +async def test_the_mode_reported_as_left_is_the_one_last_carried_out(): + """A revoked instruction was never carried out, so the next transition does not start from its mode.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + a, b = get_unique_id(), get_unique_id() + first = _instruction(a, timedelta(0)) + revoked = _instruction(b, timedelta(seconds=30)) + later = _instruction(a, timedelta(milliseconds=200)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [ + first.model_dump_json(), + revoked.model_dump_json(), + revoke.model_dump_json(), + later.model_dump_json(), + ] + ) + await answer_instructions(ws) + await _until( + lambda: len([m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"]) + >= 2 + ) + statuses = [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + assert statuses[1]["previous_operation_mode_id"] == str(a) + + +@pytest.mark.asyncio +async def test_the_example_tells_the_cem_where_each_instruction_stands(): + """Accepted on arrival, succeeded once carried out, revoked once revoked: the CEM revokes only what is not yet carried out.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + done = _instruction(get_unique_id(), timedelta(0)) + revoked = _instruction(get_unique_id(), timedelta(seconds=30)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [done.model_dump_json(), revoked.model_dump_json(), revoke.model_dump_json()] + ) + await answer_instructions(ws) + await _until( + lambda: any( + m.get("status_type") == "SUCCEEDED" + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + ) + ) + updates = { + (m["instruction_id"], m["status_type"]) + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + } + assert updates == { + (str(done.id), "ACCEPTED"), + (str(done.id), "SUCCEEDED"), + (str(revoked.id), "ACCEPTED"), + (str(revoked.id), "REVOKED"), + } + + +@pytest.mark.asyncio +async def test_a_failed_ingestion_of_the_reseeded_prices_does_not_end_the_reseeding(): + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.exceptions import JobFailedError + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [JobFailedError("ingestion failed"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py index 5909b4a1..992df47e 100644 --- a/tests/s2/test_frbc_element_selection.py +++ b/tests/s2/test_frbc_element_selection.py @@ -540,7 +540,60 @@ async def test_the_handler_records_leakage_at_the_latest_fill_level(): leakage, resolution=timedelta(minutes=15), fill_level=20.0, - share_of=20.0, + share_of=max(20.0, _middle_of_the_range(frbc)), ) ] ] + + +def _middle_of_the_range(frbc) -> float: + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + return (storage_range.start_of_range + storage_range.end_of_range) / 2 + + +@pytest.mark.asyncio +async def test_a_near_empty_start_takes_the_loss_as_a_share_of_the_middle_of_the_range(): + """A share of a near-empty start would leave almost nothing per step, and overstate the loss once the plan fills up.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + await frbc._record_storage_efficiency(start, 0.01) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + middle = _middle_of_the_range(frbc) + assert middle > 0.01 + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, + resolution=timedelta(minutes=15), + fill_level=0.01, + share_of=middle, + ) + assert posted > 99 + + +@pytest.mark.asyncio +async def test_a_start_above_the_middle_takes_the_loss_as_a_share_of_the_start(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + high = (storage_range.end_of_range + _middle_of_the_range(frbc)) / 2 + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), high + ) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=high, share_of=high + ) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index f8b9f8d3..6c8ce9db 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -389,9 +389,61 @@ async def recording(*args, **kwargs): @pytest.mark.asyncio async def test_a_state_of_charge_is_recorded_as_a_reading_at_an_instant(): + """On a sensor that records instants, and over a minute on one with a resolution, which refuses an instant.""" + for resolution, duration in ( + ("PT0M", timedelta(0)), + ("PT15M", timedelta(minutes=1)), + ): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": resolution} + await frbc.handle_message(storage_status()) + await settle(frbc) + posted = frbc._fm_client.post_measurements.await_args.kwargs + assert posted["duration"] == duration + + +@pytest.mark.asyncio +async def test_interleaved_statuses_about_one_step_get_belief_times_a_second_apart(): + """A status about step A, one about step B, then A again, all within a second: the two about A must not collide.""" + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + frbc = frbc_simple() - await frbc.handle_message(storage_status()) - await settle(frbc) - assert frbc._fm_client.post_measurements.await_args.kwargs["duration"] == timedelta( - 0 + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + mode = actuator.operation_modes[0].id + a = datetime(2022, 12, 2, 0, 5, tzinfo=timezone.utc) + b = datetime(2022, 12, 2, 0, 20, tzinfo=timezone.utc) + for when in (a, b, a): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + priors = [ + c.kwargs["prior"] + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("start") == a + ] + assert len(priors) == 2 + assert priors[1] - priors[0] >= timedelta(seconds=1) + + +def test_a_step_boundary_is_on_the_local_clock_where_the_zone_is_half_an_hour_off_utc(): + """FlexMeasures floors a series on the sensor's clock, so an hourly step in Kolkata starts on the hour there.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=1) + moment = datetime(2026, 11, 5, 14, 40, tzinfo=ZoneInfo("Asia/Kolkata")) + assert frbc.align_to_resolution(moment) == datetime( + 2026, 11, 5, 14, 0, tzinfo=ZoneInfo("Asia/Kolkata") ) From c7fe0a7fc9fa038e25a09ca15ce615ad677f69e1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:34:15 +0100 Subject: [PATCH 25/59] docs(changelog): describe the example round as it now stands Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d343b2ef..279a3356 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -37,7 +37,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] Removed ------- @@ -48,8 +48,8 @@ Fixed ----- * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] -* A resolution boundary in the repeated hour of an autumn DST change is found in that hour, where it used to be taken for the first pass of it, an hour earlier, so a schedule could start in the past [`#259 `__] -* A state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast [`#259 `__] +* A resolution boundary is found on the site's own clock, as FlexMeasures finds it, and in the repeated hour of an autumn DST change in the pass the moment is in, where it used to be taken for the first pass, an hour earlier, so a schedule could start in the past [`#259 `__] +* On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] * Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] @@ -62,7 +62,7 @@ Fixed * ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] * A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] * ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] -* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] +* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind. Statuses about different steps may interleave, so the spacing is kept per step [`#252 `__, `#259 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] * ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] * The example server creates its power sensor with ``consumption_is_positive``, as S2 states power, so that an actuator's measured power and the schedule planned for it point the same way on the sensor that holds both [`#252 `__] From 41a8f637ac5c87c073d75014f430f6a4f59e5d67 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 16:58:03 +0100 Subject: [PATCH 26/59] fix(example, FRBC): what the third review of the example round found - In the repeated hour of an autumn DST change, a boundary is the later of the two candidates that is not after the moment. Taking the pass from the moment put 03:10 at two-hourly steps on 02:00 of the first pass, three hours back. The candidates are compared as pandas timestamps, which compare instants. - The changelog says what the floor does: on the clock of the moment, in the offset it carries, which is how FlexMeasures floors a posted series. "The site's own clock" was not true for series a resource manager anchors in UTC. - A state-of-charge sensor stating a resolution that is not a duration is posted over a minute and logged, where it aborted the status before planning. The docstring says the sensor is asked until it answers, which it is. - An instruction that has become due is no longer pending, so a revocation arriving while it is carried out is not answered as REVOKED for an instruction the heat pump is following. - The comment on the storage efficiency's reference says that below the middle of the range the loss is understated, and by how much at most. - Tests pin the forgetting of belief times more than a day old. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 40 +++++++---- .../s2/script/websockets_client.py | 16 +++-- tests/s2/test_example_end_to_end.py | 27 +++++++ tests/s2/test_frbc_simple_planning.py | 72 +++++++++++++++++++ 5 files changed, 139 insertions(+), 18 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 279a3356..f8c2a2a8 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -48,7 +48,7 @@ Fixed ----- * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] -* A resolution boundary is found on the site's own clock, as FlexMeasures finds it, and in the repeated hour of an autumn DST change in the pass the moment is in, where it used to be taken for the first pass, an hour earlier, so a schedule could start in the past [`#259 `__] +* A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] * ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] * ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index a266a8cc..1f669a29 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -291,17 +291,22 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - # Floored on the moment's own clock, which is the grid FlexMeasures floors a series to, + # Floored on the moment's own clock, in the offset it carries, which is how FlexMeasures floors a series posted with it, # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. - # In the repeated hour of an autumn DST change, the pass the moment is in decides which of the two boundaries it is, - # where an earlier version took the first pass every time, an hour earlier than meant. + # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two + # that is not after the moment: taking the first pass every time put a boundary an hour or more too early. # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. - floored = pd.Timestamp(moment).floor( - self._resolution, - ambiguous=moment.fold == 0, - nonexistent="shift_forward", - ) - return floored.to_pydatetime() + # Compared as pandas timestamps, which compare instants: two datetimes on one local clock compare by wall time, + # whichever pass each is on. + at = pd.Timestamp(moment) + candidates = [ + at.floor( + self._resolution, ambiguous=first_pass, nonexistent="shift_forward" + ) + for first_pass in (True, False) + ] + boundary = max(c for c in candidates if c <= at) + return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. @@ -321,7 +326,7 @@ async def _soc_duration(self) -> timedelta: Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, so on any other sensor, or one that cannot be read, it is posted over a minute, as before. - The sensor is asked once, since its resolution is a property of how it was set up. + The sensor is asked until it answers, and its answer kept, since its resolution is a property of how it was set up. """ if self._soc_sensor_records_instants is None: try: @@ -336,9 +341,17 @@ async def _soc_duration(self) -> timedelta: resolution = ( sensor.get("event_resolution") if isinstance(sensor, dict) else None ) - self._soc_sensor_records_instants = resolution is not None and pd.Timedelta( - resolution - ) == pd.Timedelta(0) + try: + self._soc_sensor_records_instants = resolution is not None and ( + pd.Timedelta(resolution) == pd.Timedelta(0) + ) + except ValueError: + self._logger.warning( + "State-of-charge sensor %s states a resolution of %r, which is not a duration, so a fill level is posted over a minute.", + self._soc_sensor_id, + resolution, + ) + self._soc_sensor_records_instants = False return ( timedelta(0) if self._soc_sensor_records_instants else timedelta(minutes=1) ) @@ -1114,6 +1127,7 @@ async def _record_storage_efficiency( # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: # from a near-empty start, a share of the start level would let almost nothing remain per step, # and the loss would be overstated many times over as soon as the plan filled the storage. + # Below the middle the loss is understated instead, by at most the ratio of the middle to the stock, which is the milder error. storage_range = list(self._system_description_history.values())[ -1 ].storage.fill_level_range diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index ad7b646e..b73ce4c9 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -13,6 +13,7 @@ import asyncio import os +from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone @@ -305,6 +306,7 @@ async def apply_at_execution_time( instruction: FRBCInstruction, applied: dict | None = None, buffer: Buffer | None = None, + on_due: Callable[[], None] | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -323,6 +325,9 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) + # From here on it is being carried out, so a revocation arriving now is too late to stop it. + if on_due is not None: + on_due() now = datetime.now(timezone.utc) if buffer is not None: buffer.follow(instruction, now) @@ -382,15 +387,18 @@ async def answer_instructions(ws, buffer: Buffer | None = None) -> None: ) await send_instruction_status(ws, instruction.id, InstructionStatus.ACCEPTED) # In a task of its own, so that an instruction due later does not hold up the ones behind it. - task = asyncio.create_task( - apply_at_execution_time(ws, instruction, applied, buffer) - ) key = str(instruction.id) - pending[key] = task + + def due(key: str = key) -> None: + pending.pop(key, None) def forget(done: asyncio.Task, key: str = key) -> None: pending.pop(key, None) + task = asyncio.create_task( + apply_at_execution_time(ws, instruction, applied, buffer, on_due=due) + ) + pending[key] = task task.add_done_callback(forget) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 3ef96f46..0e6b680c 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -685,3 +685,30 @@ async def test_a_failed_ingestion_of_the_reseeded_prices_does_not_end_the_reseed await _until(lambda: fm.post_sensor_data.await_count >= 2) with pytest.raises(StopAsyncIteration): await seeding.__anext__() + + +@pytest.mark.asyncio +async def test_a_revocation_arriving_once_an_instruction_is_due_is_too_late_to_stop_it(): + """It is carried out and reported as succeeded, not reported as revoked while the heat pump follows it.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.script.websockets_client import apply_at_execution_time + from flexmeasures_client.s2.utils import get_unique_id + + instruction = _instruction(get_unique_id(), timedelta(0)) + became_due = [] + ws = _Collect() + await apply_at_execution_time( + ws, instruction, {"mode": None}, on_due=lambda: became_due.append(True) + ) + assert became_due == [True] + + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=instruction.id, + ) + ws = _Collect([instruction.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await _until(lambda: any(m.get("status_type") == "SUCCEEDED" for m in ws.sent)) + assert not [m for m in ws.sent if m.get("status_type") == "REVOKED"] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 6c8ce9db..92ca9b5b 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -447,3 +447,75 @@ def test_a_step_boundary_is_on_the_local_clock_where_the_zone_is_half_an_hour_of assert frbc.align_to_resolution(moment) == datetime( 2026, 11, 5, 14, 0, tzinfo=ZoneInfo("Asia/Kolkata") ) + + +def test_a_boundary_in_the_repeated_hour_is_the_later_one_not_after_the_moment(): + """At two-hourly steps, 03:10 on the night the clocks go back is past 02:00 of the second pass, which is the boundary meant.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=2) + amsterdam = ZoneInfo("Europe/Amsterdam") + moment = datetime(2026, 10, 25, 3, 10, tzinfo=amsterdam) + aligned = frbc.align_to_resolution(moment) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +def test_a_naive_moment_is_floored_on_its_clock(): + frbc = frbc_simple() + assert frbc.align_to_resolution(datetime(2026, 1, 1, 12, 7)) == datetime( + 2026, 1, 1, 12, 0 + ) + + +@pytest.mark.asyncio +async def test_a_state_of_charge_sensor_stating_no_duration_is_posted_over_a_minute( + caplog, +): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": "fortnightly"} + assert await frbc._soc_duration() == timedelta(minutes=1) + assert "not a duration" in caplog.text + + +@pytest.mark.asyncio +async def test_a_sensor_that_could_not_be_read_is_asked_again(): + frbc = frbc_simple() + frbc._fm_client.get_sensor.side_effect = [ + ConnectionError("down"), + {"event_resolution": "PT0M"}, + ] + assert await frbc._soc_duration() == timedelta(minutes=1) + assert await frbc._soc_duration() == timedelta(0) + + +@pytest.mark.asyncio +async def test_belief_times_are_forgotten_for_steps_more_than_a_day_old(): + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + + frbc = frbc_simple() + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + for when in ( + datetime(2022, 12, 1, 0, 0, tzinfo=timezone.utc), + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc), + ): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + assert list(frbc._power_recorded_at) == [ + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc) + ] From bc90b2bd1bcb1faa10ccd32aacf1abd84bbf7808 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 17:19:31 +0100 Subject: [PATCH 27/59] fix(FRBC): plan from the next boundary, not the start of the step under way The demo gate's second run, against a sandboxed FlexMeasures, heated the buffer in the quarter hour the plan chose, but every replan inside that quarter replanned the same quarter, at 0.72 kW, then 0.67, 0.63 and on down to 0.51 kW. A schedule asked for mid-step started at the boundary that step began at, while it was planned from the fill level reported now, which FlexMeasures takes to be the level at the schedule's start. So each replan counted on the whole step where only part of it was left, asked for less power over what remained, and the step delivered less than any plan meant it to. A schedule now starts at the next boundary, or at now where now is one. The step under way keeps the instruction it is already following, which a newer plan does not revoke, since it has been carried out. The end-to-end test stands the CEM's clock on a boundary, so that its schedules start there and the first instruction is due within the test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 19 +++++++++----- tests/s2/test_example_end_to_end.py | 14 ++++++++++- tests/s2/test_frbc_simple_planning.py | 25 +++++++++++++++++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index f8c2a2a8..243ee8d5 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -79,7 +79,7 @@ Fixed * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] * The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] * An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and never from a step before the current one, which a description days old made it slide into [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and from the next resolution boundary rather than the start of the step under way, which a description days old made it slide into the past, and which, planned from the fill level reported mid-step, counted on the whole step where only part was left, so each replan within it asked for less power than the step needed [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 1f669a29..8bce09af 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -309,16 +309,23 @@ def align_to_resolution(self, moment: datetime) -> datetime: return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary, but not into an earlier step than now's. + """Where a schedule asked for at valid_from starts: shifted and floored to a boundary, but never before now. - Not into an earlier step, since planning happens on every storage status while a description may be days old: + Never before now, since planning happens on every storage status while a description may be days old: anchored to that description alone, every status in a long session was answered with the same window, which slid into the past, so its instructions were all due already and arrived as one burst. - The step now falls in has begun already, so its first instruction is due at once. + Where now is past that boundary, the schedule starts at the next one, not at the start of the step now is in. + A schedule is planned from the fill level reported now, and FlexMeasures takes that to be the level at its start, + so a schedule starting at a step already under way was planned as if the whole step were left, + and every replan within it asked for less power over what remained of it, delivering less than planned. + The step under way keeps the instruction it is already following. """ - return self.align_to_resolution( - max(valid_from + self._valid_from_shift, self.now()) - ) + now = self.now() + start = self.align_to_resolution(valid_from + self._valid_from_shift) + if start >= now: + return start + current = self.align_to_resolution(now) + return current if current == now else current + self._resolution async def _soc_duration(self) -> timedelta: """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 0e6b680c..c01016cf 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -118,7 +118,18 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): loop = asyncio.get_running_loop() loop.set_exception_handler(lambda _, context: unhandled.append(context)) began = datetime.now(timezone.utc) + # The CEM's clock stands on the boundary the step under way began at, so that is where its schedules start, + # and the first instruction is due at once rather than at the next boundary, up to a quarter of an hour away. + on_a_boundary = began.replace( + minute=began.minute - began.minute % 15, second=0, microsecond=0 + ) + + from unittest.mock import patch + clock = patch.object( + server.FRBCSimple, "now", lambda self: on_a_boundary.astimezone(self._timezone) + ) + clock.start() tasks = [ asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), asyncio.create_task( @@ -133,6 +144,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) + clock.stop() gc.collect() loop.set_exception_handler(None) @@ -150,7 +162,7 @@ async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): assert refused == [] start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] - assert began - timedelta(minutes=15) < start <= began + timedelta(seconds=5) + assert start == on_a_boundary assert unhandled == [] # Every declaration reached the sensor the server made for it, to be seen next to the schedule planned against it. diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 92ca9b5b..8b314168 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -284,12 +284,33 @@ def test_a_handler_turns_away_a_model_it_does_not_handle(): def test_a_schedule_never_starts_before_now(): - """A description days old still plans from now, on a boundary, rather than a window that has already passed.""" + """A description days old still plans from the next boundary, rather than a window that has already passed. + + Nor from the start of the step under way: the fill level reported now is taken as the level at the start, + so the plan would count on the whole step where only part of it is left. + """ frbc = frbc_simple() frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) assert frbc.schedule_start( datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) - ) == datetime(2022, 12, 10, 14, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_asked_for_on_a_boundary_starts_there(): + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_whose_shifted_start_is_in_the_step_under_way_starts_at_the_next_boundary(): + frbc = frbc_simple() + frbc._valid_from_shift = timedelta(0) + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 10, 14, 6, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) @pytest.mark.asyncio From 9824ea5dbb5e5ffc70f207c66c39dbbeb28ecdd8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 17:23:06 +0100 Subject: [PATCH 28/59] style(FRBC): keep the soc-at-start TODO off the line it annotates Black had split the expression around the trailing comment after the merge. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 6f21e10f..9805fc87 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -912,8 +912,8 @@ async def trigger_schedule( soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { "soc-unit": soc_unit, - "soc-at-start": soc_at_start - * to_soc_unit, # TODO: use forecast of the SOC instead + # TODO: use forecast of the SOC instead + "soc-at-start": soc_at_start * to_soc_unit, "soc-min": soc_min * to_soc_unit, "soc-max": soc_max * to_soc_unit, } From 93e3ae3ea77d4b8fbd2d280799c2f2c7ad01a5ad Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:06:28 +0100 Subject: [PATCH 29/59] feat(FRBC): a production price of its own, an opt-in relaxed site capacity, and assets without a location Carved from #188, which needed these for a community of homes steered by one CEM. - FRBCSimple takes production_price_sensor_id, for a site where feeding power back is priced differently from drawing it, as under a feed-in tariff. Without one, both directions are priced by price_sensor_id, as before. - FRBCSimple takes relax_site_capacity, off by default. For a home the connection is a physical limit; for a member of a community whose capacity is a target set by others, a breach at a price is what is meant. - A schedule overtaken by a newer storage status during its round trip is no longer translated into instructions: send_instruction_batch would drop them anyway, and translating follows every step of the schedule. - The instructions of a batch are logged once, at debug level, by mode name, rather than as a list growing on every step. - add_asset no longer requires a location, which FlexMeasures does not require either and an asset standing for a community has no use for. Arguments keep their positions, so generic_asset_type_id is optional in the signature only, and its absence is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/client.py | 19 ++-- .../s2/control_types/FRBC/frbc_simple.py | 30 ++++++- .../s2/control_types/FRBC/utils.py | 14 +++ tests/client/test_asset.py | 47 ++++++++++ tests/s2/test_frbc_charging_efficiency.py | 88 +++++++++++++++++++ 5 files changed, 191 insertions(+), 7 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 698f5bc6..5aa6cb1f 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1511,9 +1511,9 @@ async def add_asset( self, name: str, account_id: int, - latitude: float, - longitude: float, - generic_asset_type_id: int, + latitude: float | None = None, + longitude: float | None = None, + generic_asset_type_id: int | None = None, parent_asset_id: int | None = None, sensors_to_show: list | None = None, flex_context: dict | None = None, @@ -1539,13 +1539,20 @@ async def add_asset( This function raises a ValueError when an unhandled status code is returned. """ - asset = dict( + if generic_asset_type_id is None: + # Optional in the signature only so that latitude and longitude, before it, can be left out, + # while every argument keeps its position. + raise ValueError("add_asset needs a generic_asset_type_id.") + asset: dict = dict( name=name, account_id=account_id, - latitude=latitude, - longitude=longitude, generic_asset_type_id=generic_asset_type_id, ) + # A location is optional to FlexMeasures, which many assets have no use for, such as one standing for a community. + if latitude is not None: + asset["latitude"] = latitude + if longitude is not None: + asset["longitude"] = longitude if parent_asset_id: asset["parent_asset_id"] = parent_asset_id if sensors_to_show: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 110b3f26..8e88ce64 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -122,6 +122,8 @@ def _operation_mode_power_bands( class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int + _production_price_sensor_id: int + _relax_site_capacity: bool _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None @@ -159,6 +161,13 @@ def __init__( soc_sensor_id: int, rm_discharge_sensor_id: int, price_sensor_id: int, + # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. + # Without one, both directions are priced by price_sensor_id. + production_price_sensor_id: int | None = None, + # Whether the site's connection capacity may be breached at a price rather than never. + # Off by default: for a home, the connection is a physical limit. + # A site whose capacity is a target set by others, as in a community steering its members, can turn it on. + relax_site_capacity: bool = False, # Optional so existing callers keep working; when absent the actuator's fill rate is # simply not recorded, rather than being posted onto the power sensor. fill_rate_sensor_id: int | None = None, @@ -204,6 +213,12 @@ def __init__( super().__init__(max_size) self._power_sensor_id = power_sensor_id self._price_sensor_id = price_sensor_id + self._production_price_sensor_id = ( + production_price_sensor_id + if production_price_sensor_id is not None + else price_sensor_id + ) + self._relax_site_capacity = relax_site_capacity self._schedule_duration = schedule_duration self._resolution = resolution self._soc_sensor_id = soc_sensor_id @@ -922,7 +937,7 @@ async def trigger_schedule( self._logger.debug("flex model: %s", flex_model) flex_context: dict = { - "production-price": {"sensor": self._price_sensor_id}, + "production-price": {"sensor": self._production_price_sensor_id}, "consumption-price": {"sensor": self._price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", } @@ -932,10 +947,23 @@ async def trigger_schedule( # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. flex_context["relax-soc-constraints"] = True + if self._relax_site_capacity: + flex_context["relax-site-capacity-constraints"] = True schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) self._warn_where_a_fallback_scheduler_answered(schedule, start) + # send_instruction_batch checks this once more, but translating a schedule follows every step of it, + # which is wasted on a request that is already out of date. + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not turning the schedule for planning request %s into instructions: a newer storage status arrived while it was being planned.", + request_number, + ) + return + # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor instructions = fm_schedule_to_instructions( schedule, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index a513d9eb..d30cfd3a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -189,6 +189,7 @@ def fm_schedule_to_instructions( ) actuator = system_description.actuators[0] operation_modes: list[FRBCOperationMode] = actuator.operation_modes + modes_by_id = {str(mode.id): mode for mode in operation_modes} # Held between the hard bounds the schedule was planned against, # so that each step is answered with the operation mode for the fill level the scheduler planned. @@ -295,6 +296,19 @@ def fm_schedule_to_instructions( bounds=(fill_level_min, fill_level_max), ) + if logger.isEnabledFor(logging.DEBUG): + # The whole batch once, by name, rather than as a growing list on every step. + logger.debug( + "Instructions: %s", + [ + ( + instruction.execution_time.isoformat(), + om_label(modes_by_id[str(instruction.operation_mode)]), + round(instruction.operation_mode_factor, 6), + ) + for instruction in instructions + ], + ) return instructions diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index 25101873..78f0f723 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -560,3 +560,50 @@ async def test_delete_asset_confirm_no(): (k[0], str(k[1])) for k in m.requests.keys() ] await client.close() + + +def _posted_asset(m) -> dict: + """The body of the one asset post aioresponses saw.""" + ((_, calls),) = [ + (key, calls) for key, calls in m.requests.items() if key[0] == "POST" + ] + return calls[0].kwargs["json"] + + +@pytest.mark.asyncio +async def test_an_asset_can_be_added_without_a_location(): + """FlexMeasures takes no location for an asset that has none, such as one standing for a community.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 27}) + await client.add_asset(name="community", account_id=1, generic_asset_type_id=5) + posted = _posted_asset(m) + assert "latitude" not in posted and "longitude" not in posted + await client.close() + + +@pytest.mark.asyncio +async def test_a_location_is_posted_as_numbers_where_given(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 28}) + await client.add_asset( + name="home", + account_id=1, + latitude=52.0, + longitude=4.0, + generic_asset_type_id=5, + ) + posted = _posted_asset(m) + assert (posted["latitude"], posted["longitude"]) == (52.0, 4.0) + await client.close() + + +@pytest.mark.asyncio +async def test_an_asset_needs_a_type(): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="generic_asset_type_id"): + await client.add_asset(name="untyped", account_id=1) + await client.close() diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 6baea1dd..dc4a3460 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -448,3 +448,91 @@ async def test_the_measured_state_of_charge_is_not_named_in_the_flex_model(): flex_model = await _asked_for(frbc, system_description) assert "state-of-charge" not in flex_model assert isinstance(flex_model["soc-at-start"], float) + + +@pytest.mark.asyncio +async def test_feeding_back_is_priced_by_its_own_sensor_where_there_is_one(): + system_description = _heater() + frbc = _frbc(system_description, production_price_sensor_id=42) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 42} + assert flex_context["consumption-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +async def test_without_its_own_sensor_feeding_back_is_priced_as_drawing(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("relax", [False, True]) +async def test_the_site_capacity_is_relaxed_only_where_asked(relax): + """A home's connection is a physical limit; a community member's may be a target it can breach at a price.""" + system_description = _heater() + frbc = _frbc(system_description, relax_site_capacity=relax) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context.get("relax-site-capacity-constraints", False) is relax + + +@pytest.mark.asyncio +async def test_a_schedule_overtaken_while_it_was_planned_is_not_translated(): + """A newer storage status arrived during the round trip, so turning this schedule into instructions is wasted.""" + from unittest.mock import patch + + from flexmeasures_client.s2.control_types.FRBC import STORAGE_STATUS_REQUESTS + + system_description = _heater() + frbc = _frbc(system_description) + request_number = frbc._count_request(STORAGE_STATUS_REQUESTS) + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def overtaken(**kwargs): + frbc._count_request(STORAGE_STATUS_REQUESTS) # a newer status, mid round trip + return planned + + frbc._fm_client.trigger_and_get_schedule = AsyncMock(side_effect=overtaken) + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions" + ) as translate: + await frbc.trigger_schedule( + system_description.message_id, request_number=request_number + ) + translate.assert_not_called() + frbc.send_instruction_batch.assert_not_awaited() + + +def test_a_batch_of_instructions_is_logged_once_and_by_name(caplog): + from flexmeasures_client.s2.control_types.FRBC.utils import ( + fm_schedule_to_instructions, + ) + + system_description = _heater() + with caplog.at_level(logging.DEBUG): + fm_schedule_to_instructions( + { + "start": "2024-01-01T00:00:00+00:00", + "duration": "PT1H", + "values": [0.0, HEATER_POWER, HEATER_POWER, 0.0], + }, + system_description, + initial_fill_level=FILL_LEVEL_AT_START, + ) + batches = [ + r.getMessage() + for r in caplog.records + if r.getMessage().startswith("Instructions: ") + ] + assert len(batches) == 1 + assert "'on'" in batches[0] and "'off'" in batches[0] From 6408b6fc8356b1cd9c94e71a9ebb740448128ede Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:13:07 +0100 Subject: [PATCH 30/59] fix(FRBC): what the review of the energy-unit change found - soc-min is converted like soc-at-start and soc-max, but nothing checked it: it is zero unless a storage is reported below its range, and zero converts to zero. A test now reports one below it, in joules. - A test docstring and a changelog line from #244 said that a state-of-charge sensor in another unit would hold readings a thousand times off. FlexMeasures converts what is posted into the sensor's own unit, so it does not. - A comment broke between its subject and its verb. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 5 +++-- tests/s2/test_frbc_charging_efficiency.py | 13 +++++++++++++ tests/s2/test_frbc_declared_profiles.py | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 61c953e5..9dcbd5a7 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -34,7 +34,7 @@ Changed * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] * ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, rather than as the hard ``soc-min`` and ``soc-max``, which become wide bounds at zero and half again above the range, each widened to take in a state of charge beyond them. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band [`#245 `__] -* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] +* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] * The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 9805fc87..fee4e462 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -906,8 +906,9 @@ async def trigger_schedule( # and the window a charging efficiency has to be recorded over to be planned against. start = self.schedule_start(system_description.valid_from) - # FlexMeasures takes a state of charge in kWh or MWh only, so a fill level recorded in another energy unit - # is converted here, at the edge, rather than refused, and the bounds read back below stay in the recorded unit. + # FlexMeasures takes a state of charge in kWh or MWh only. + # So a fill level recorded in another energy unit is converted here, at the edge, rather than refused, + # and the bounds read back below stay in the recorded unit. # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 6baea1dd..37c14be8 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -418,6 +418,19 @@ async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( assert flex_model["soc-max"] == pytest.approx(1.5 * RANGE_TOP * per_unit) +@pytest.mark.asyncio +async def test_a_lower_bound_below_zero_goes_out_in_the_same_unit(): + """Reported below its range, a storage widens the lower bound to take it in, which then needs the conversion too.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit="J") + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=-3.6e6 + ) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-at-start"] == pytest.approx(-1.0) + assert flex_model["soc-min"] == pytest.approx(-1.0) + + @pytest.mark.asyncio async def test_the_bounds_read_back_stay_in_the_recorded_unit(): """Only what goes to FlexMeasures is converted: the schedule is matched against the operation modes in their own units.""" diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index b1af0a3b..201ed323 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -409,7 +409,7 @@ async def test_the_energy_unit_is_the_deployment_s_to_choose(system_description) async def test_a_schedule_is_asked_for_in_the_unit_the_fill_level_is_recorded_in( system_description, ): - """soc-unit has to follow energy_unit, or the scheduler plans against bounds a thousand times its measurements.""" + """soc-unit follows energy_unit where FlexMeasures takes it, so the values stated inline are in the unit they are recorded in.""" frbc = _frbc(system_description, with_sensors=True) # A schedule is only asked for once a storage status has said where to start from. frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( From 11e9a397891bcb46c4838fc5b1509efa35217d51 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:20:33 +0100 Subject: [PATCH 31/59] test(FRBC): an overtaken plan sends nothing, whether dropped before or after translation Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_frbc_simple_planning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 8b314168..c35ec8b1 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -405,7 +405,8 @@ async def recording(*args, **kwargs): assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 sent = dict(batches) assert sent[2] > 0 # the description's plan, which is the newer request - assert sent[1] == 0 # the status's plan, overtaken while it was out + # The status's plan, overtaken while it was out, sends nothing, whether it is dropped before or after translation. + assert sent.get(1, 0) == 0 @pytest.mark.asyncio From 7c8ae45ff7bc497fdb14e5127288ea6b7dda263e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:21:24 +0100 Subject: [PATCH 32/59] docs(changelog): the handler options, the dropped translation, and assets without a location Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 ++++ CHANGELOG.rst | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 9dcbd5a7..ecd80cc9 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -22,10 +22,14 @@ Added * ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] +* ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``, off by default, which lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others rather than a physical limit [`#261 `__] +* An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] Changed ------- +* An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] * ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes, which a schedule's start is lined up with [`#240 `__] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 05d31188..4a939f62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,7 @@ Added Changed ------- +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] Removed From 47d295dc3bd716be40240bf950e0ffdd62a9a4ed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:34:55 +0100 Subject: [PATCH 33/59] fix(FRBC): send relax_site_capacity either way, or leave it to FlexMeasures Review found that relax_site_capacity=False, the default, kept nothing hard. Leaving the key out lets FlexMeasures, since v1.0, read its relax-constraints umbrella, which relaxes by default at a 10000 EUR/kW breach price, so a home's connection was soft whichever way the option was set. relax_site_capacity is now True, False or unset. True and False are sent as they are; unset leaves the server's own default in place, which is what the handler has always done, so nothing changes for a caller that does not pass it. The test asserts what is sent, where it asserted a missing key, which passed on the bug. Also from the review: - add_asset refuses half a location, which FlexMeasures would store, and its docstring says when it raises before any request is made. - The comment on the batch log line said it replaced a growing per-step list that this branch never had. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- CHANGELOG.rst | 2 +- src/flexmeasures_client/client.py | 7 ++++++- .../s2/control_types/FRBC/frbc_simple.py | 16 ++++++++------- .../s2/control_types/FRBC/utils.py | 2 +- tests/client/test_asset.py | 11 ++++++++++ tests/s2/test_frbc_charging_efficiency.py | 20 ++++++++++++++++--- 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index ecd80cc9..b3f40ebe 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -23,7 +23,7 @@ Added * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] -* ``FRBCSimple`` takes ``relax_site_capacity``, off by default, which lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others rather than a physical limit [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] Changed diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a939f62..4d0d1f34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,7 +27,7 @@ Added Changed ------- -* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required [`#261 `__] +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required, and half a location is refused [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] Removed diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 5aa6cb1f..9252c477 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1537,12 +1537,17 @@ async def add_asset( 'status': 200 } - This function raises a ValueError when an unhandled status code is returned. + This function raises a ValueError when an unhandled status code is returned, + when no generic_asset_type_id is given, and when only one of latitude and longitude is. """ if generic_asset_type_id is None: # Optional in the signature only so that latitude and longitude, before it, can be left out, # while every argument keeps its position. raise ValueError("add_asset needs a generic_asset_type_id.") + if (latitude is None) != (longitude is None): + raise ValueError( + "add_asset takes a location as both a latitude and a longitude, or neither." + ) asset: dict = dict( name=name, account_id=account_id, diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 735b29a7..694541e2 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -123,7 +123,7 @@ class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int _production_price_sensor_id: int - _relax_site_capacity: bool + _relax_site_capacity: bool | None _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None @@ -164,10 +164,12 @@ def __init__( # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. # Without one, both directions are priced by price_sensor_id. production_price_sensor_id: int | None = None, - # Whether the site's connection capacity may be breached at a price rather than never. - # Off by default: for a home, the connection is a physical limit. - # A site whose capacity is a target set by others, as in a community steering its members, can turn it on. - relax_site_capacity: bool = False, + # Whether the site's connection capacity may be breached at a price (True) or never (False). + # Unset, it is left to FlexMeasures, which since v1.0 relaxes it at a high default breach price, + # under its relax-constraints umbrella, and before that did not. + # A home whose connection is a physical limit can pass False; + # a site whose capacity is a target set by others, as in a community steering its members, can pass True. + relax_site_capacity: bool | None = None, # Optional so existing callers keep working; when absent the actuator's fill rate is # simply not recorded, rather than being posted onto the power sensor. fill_rate_sensor_id: int | None = None, @@ -1016,8 +1018,8 @@ async def trigger_schedule( # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. flex_context["relax-soc-constraints"] = True - if self._relax_site_capacity: - flex_context["relax-site-capacity-constraints"] = True + if self._relax_site_capacity is not None: + flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) self._warn_where_a_fallback_scheduler_answered(schedule, start) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 2dbfd0b7..cec7fa08 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -383,7 +383,7 @@ def fm_schedule_to_instructions( ) if logger.isEnabledFor(logging.DEBUG): - # The whole batch once, by name, rather than as a growing list on every step. + # The whole batch in one line, by mode name, which the per-step lines above do not give. logger.debug( "Instructions: %s", [ diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index 78f0f723..85c4b238 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -607,3 +607,14 @@ async def test_an_asset_needs_a_type(): with pytest.raises(ValueError, match="generic_asset_type_id"): await client.add_asset(name="untyped", account_id=1) await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("location", [dict(latitude=52.0), dict(longitude=4.0)]) +async def test_half_a_location_is_refused(location): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="both a latitude and a longitude"): + await client.add_asset( + name="half", account_id=1, generic_asset_type_id=5, **location + ) + await client.close() diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index bc12cbab..2a899d82 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -488,15 +488,29 @@ async def test_without_its_own_sensor_feeding_back_is_priced_as_drawing(): @pytest.mark.asyncio @pytest.mark.parametrize("relax", [False, True]) -async def test_the_site_capacity_is_relaxed_only_where_asked(relax): - """A home's connection is a physical limit; a community member's may be a target it can breach at a price.""" +async def test_the_site_capacity_is_relaxed_or_kept_hard_as_asked(relax): + """Sent either way: FlexMeasures reads a missing flag as its relax-constraints umbrella, which relaxes by default. + + So a home whose connection is a physical limit is only kept to it by an explicit False. + """ system_description = _heater() frbc = _frbc(system_description, relax_site_capacity=relax) await frbc.trigger_schedule(system_description.message_id) flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ "flex_context" ] - assert flex_context.get("relax-site-capacity-constraints", False) is relax + assert flex_context["relax-site-capacity-constraints"] is relax + + +@pytest.mark.asyncio +async def test_unset_the_site_capacity_is_left_to_flexmeasures(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert "relax-site-capacity-constraints" not in flex_context @pytest.mark.asyncio From bcdbd79c53d031f21cc4f0c27e65f7f9586642df Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:41:16 +0100 Subject: [PATCH 34/59] feat(example): the site's graphs show the buffer, its comfort band, and what fills and draws it The demo site's graphs page showed only the price and the power, which says nothing about whether the plan kept the buffer warm. The example server now sets the site's sensors_to_show: the price; the heat pump's planned and measured power; the buffer's fill level between the comfort band's minima and maxima; and the heat pump's fill rate next to the hot-water usage forecast, both in kWh/s. Checked against a sandboxed FlexMeasures v1.1.0rc3: all four graphs render, with the pre-heat before the morning showers visible as a raised minimum. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/script/websockets_server.py | 31 +++++++++++++++++++ tests/s2/test_example_end_to_end.py | 29 +++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 49d81d82..820808a3 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -217,10 +217,41 @@ async def configure_site( **({"attributes": attributes} if attributes else {}), ) + # The site's graphs page shows what the plan is about, not only its price and power: + # the buffer with the comfort band around it, what is drawn from it, and what the heat pump puts in. + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": site_graphs(sensors)}, + parse_json_fields=True, + ) + await seed_prices(fm_client, sensors["price"]["id"]) return sensors +def site_graphs(sensors: dict[str, dict]) -> list[dict]: + """The site's graphs, top to bottom, in the format FlexMeasures takes for an asset's sensors_to_show.""" + + def ids(*names: str) -> list[int]: + return [sensors[name]["id"] for name in names] + + return [ + {"title": "Price", "plots": [{"sensors": ids("price")}]}, + { + "title": "Heat pump: planned and measured power", + "plots": [{"sensors": ids("power")}], + }, + { + "title": "Buffer: fill level and comfort band", + "plots": [{"sensors": ids("state of charge", "soc minima", "soc maxima")}], + }, + { + "title": "Buffer: filled by the heat pump, drawn by hot-water use", + "plots": [{"sensors": ids("fill rate", "usage forecast")}], + }, + ] + + async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: """Demo prices, so that a site has something to schedule against. diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index c01016cf..1406388b 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -724,3 +724,32 @@ async def test_a_revocation_arriving_once_an_instruction_is_due_is_too_late_to_s await answer_instructions(ws) await _until(lambda: any(m.get("status_type") == "SUCCEEDED" for m in ws.sent)) assert not [m for m in ws.sent if m.get("status_type") == "REVOKED"] + + +@pytest.mark.asyncio +async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_draws_it(): + """The graphs page showed only price and power, which says nothing about whether the plan kept the buffer warm.""" + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[{"id": 5, "name": server.SITE_NAME, "sensors": []}] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + sensors = await server.configure_site(server.SITE_NAME, fm) + shown = [ + call.args[1]["sensors_to_show"] + for call in fm.update_asset.await_args_list + if "sensors_to_show" in call.args[1] + ] + assert len(shown) == 1 + plotted = {graph["title"]: set(graph["plots"][0]["sensors"]) for graph in shown[0]} + by_id = {sensor["id"]: name for name, sensor in sensors.items()} + named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} + assert {"state of charge", "soc minima", "soc maxima"} in named.values() + assert {"fill rate", "usage forecast"} in named.values() + assert {"power"} in named.values() and {"price"} in named.values() From cc54bb683f5c7db7bcc450c1205a1a31e289f52b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 18:41:47 +0100 Subject: [PATCH 35/59] docs(changelog): the example site's graphs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b3f40ebe..101f6b47 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -41,7 +41,7 @@ Changed * ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] -* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, and seeds its demo prices again every 12 hours [`#259 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] Removed ------- From c46675257df5d1c25cdf4a613917586d09d0e110 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 19:16:38 +0100 Subject: [PATCH 36/59] fix(example): set the site graphs in the form every FlexMeasures reads Review found that the form with plots is refused before FlexMeasures v0.32, so the example server would not start against an older server. A title and a list of sensors per graph is read by every version and draws the same graphs (checked against v1.1.0rc3). The docstring now also says the server sets them on every start, so changes made in the UI do not outlive a restart. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/script/websockets_server.py | 18 ++++++++++-------- tests/s2/test_example_end_to_end.py | 12 +++++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 820808a3..ee7d2152 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -230,24 +230,26 @@ async def configure_site( def site_graphs(sensors: dict[str, dict]) -> list[dict]: - """The site's graphs, top to bottom, in the format FlexMeasures takes for an asset's sensors_to_show.""" + """The site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + + In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, + rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. + The server sets them on every start, so changes made to them in the UI do not outlive a restart. + """ def ids(*names: str) -> list[int]: return [sensors[name]["id"] for name in names] return [ - {"title": "Price", "plots": [{"sensors": ids("price")}]}, - { - "title": "Heat pump: planned and measured power", - "plots": [{"sensors": ids("power")}], - }, + {"title": "Price", "sensors": ids("price")}, + {"title": "Heat pump: planned and measured power", "sensors": ids("power")}, { "title": "Buffer: fill level and comfort band", - "plots": [{"sensors": ids("state of charge", "soc minima", "soc maxima")}], + "sensors": ids("state of charge", "soc minima", "soc maxima"), }, { "title": "Buffer: filled by the heat pump, drawn by hot-water use", - "plots": [{"sensors": ids("fill rate", "usage forecast")}], + "sensors": ids("fill rate", "usage forecast"), }, ] diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 1406388b..952c2867 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -747,9 +747,19 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ if "sensors_to_show" in call.args[1] ] assert len(shown) == 1 - plotted = {graph["title"]: set(graph["plots"][0]["sensors"]) for graph in shown[0]} + plotted = {graph["title"]: set(graph["sensors"]) for graph in shown[0]} by_id = {sensor["id"]: name for name, sensor in sensors.items()} named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} assert {"state of charge", "soc minima", "soc maxima"} in named.values() assert {"fill rate", "usage forecast"} in named.values() assert {"power"} in named.values() and {"price"} in named.values() + + +def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): + """A title and a list of sensors per graph; the form with plots is refused before FlexMeasures v0.32.""" + graphs = server.site_graphs( + {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + ) + for graph in graphs: + assert set(graph) == {"title", "sensors"} + assert all(isinstance(i, int) for i in graph["sensors"]) From 00284949114efb2999fa83444b2f1c6a7f8e7865 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:46:20 +0100 Subject: [PATCH 37/59] fix(S2): a lost confirmation does not stall a session, and a closed session stops Carved from #188, rewritten for the library rather than transcribed. - A control type's message arriving while no control type is selected activates that control type, if the CEM has a handler for it, with a warning. A resource manager sends a control type's messages only after accepting the SelectControlType for it, so such a message is that acceptance; without this, a lost ReceptionStatus for the selection left every FRBC message answered with TEMPORARY_ERROR, which a resource manager retries forever, and a whole co-simulation hung at its first step. #188 did this for FRBC alone and also before a resource manager's details had arrived, when nothing can have been selected; here it covers each control type's message prefix, and only after the details. - Closing a control-type handler cancels the work it still has in flight, such as a schedule being planned, which otherwise went on posting and planning for a resource manager no longer connected. TUNES no longer overrides close() with a copy of the default. - The example server's session ends with its connection: the CEM is closed and the producer and the watchdog are stopped. Left running, they outlived the connection, and a resource manager that reconnected was served by two sessions planning for it at once. - A test now pins the case of a system description replaced while the plan it asked for is out: only the newer plan is sent. The behaviour came with the request counting in #259; #188 had no equivalent. Not carried from #188: the futures paired with each outgoing message, which nothing awaited and which changed get_message's public return type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 41 +++++++++++ .../s2/control_types/FRBC/frbc_tunes.py | 6 -- .../s2/control_types/__init__.py | 5 ++ .../s2/script/websockets_server.py | 22 ++++-- tests/s2/test_cem.py | 70 +++++++++++++++++++ tests/s2/test_example_end_to_end.py | 33 +++++++++ tests/s2/test_frbc_simple_planning.py | 37 ++++++++++ 7 files changed, 202 insertions(+), 12 deletions(-) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 72ef9b3d..ceb27934 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -42,6 +42,15 @@ _LOGGER = logging.getLogger(__name__) +#: The control type each family of control-type messages belongs to, by the prefix of its message type. +CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { + "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, + "PEBC.": ControlType.POWER_ENVELOPE_BASED_CONTROL, + "PPBC.": ControlType.POWER_PROFILE_BASED_CONTROL, + "OMBC.": ControlType.OPERATION_MODE_BASED_CONTROL, + "DDBC.": ControlType.DEMAND_DRIVEN_BASED_CONTROL, +} + class CEM(Handler): __version__ = "0.0.2-beta" @@ -188,6 +197,7 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): ) # what a message is from here on, whatever form it came in self._logger.debug(f"Received: {message}") + self._activate_implicitly(message) # try to handle the message with the control_type handle if ( @@ -220,6 +230,37 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if response is not None: await self._sending_queue.put(response) + def _activate_implicitly(self, message: dict) -> None: + """Take a control type's message as proof that the resource manager selected that control type. + + A resource manager sends a control type's messages only after accepting the SelectControlType for it. + So where one arrives while no control type is selected here, the confirmation was lost on the way, + and without this every such message was answered with a TEMPORARY_ERROR, which a resource manager retries forever. + Only a control type this CEM has a handler for is activated. + """ + if self._control_type != ControlType.NO_SELECTION: + return + message_type = message.get("message_type") + if not isinstance(message_type, str): + return + control_type = next( + ( + ct + for prefix, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() + if message_type.startswith(prefix) + ), + None, + ) + if control_type is None or control_type not in self._control_types_handlers: + return + self._logger.warning( + "Received %s while no control type is selected, which a resource manager sends only after accepting %s; " + "taking the confirmation as lost and activating it.", + message_type, + control_type.value, + ) + self._control_type = control_type + def update_control_type(self, control_type: ControlType): """ Callback function that is triggered when we receive diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index d13e5708..c6a33e50 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -664,12 +664,6 @@ async def update_flex_model(self, system_description: FRBCSystemDescription): parse_json_fields=True, ) - async def close(self): - """ - Can be used to stop recurring tasks. - """ - self._logger.debug(f"Closing {self.__class__.__name__} handler") - async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): """ Send FRBC.UsageForecast to FlexMeasures. diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index 09f526f3..70f32b72 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -53,6 +53,11 @@ async def close(self): killing the CEM's request handler and hanging the resource manager. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") + # Work still in flight, such as a schedule being planned, would otherwise go on after the session has ended: + # it posts to FlexMeasures and plans for a resource manager that is no longer there, + # racing the session that replaces it when the resource manager reconnects. + for task in list(self.background_tasks): + task.cancel() @register(InstructionStatusUpdate) def handle_instruction_status_update(self, message: InstructionStatusUpdate): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index ee7d2152..279e7605 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -162,12 +162,22 @@ async def serve_resource_manager( frbc.send_revocations = True cem.register_control_type(frbc) - # create "parallel" tasks for the message producer and consumer - await asyncio.gather( - websocket_consumer(ws, cem), - websocket_producer(ws, cem), - rm_details_watchdog(ws, cem), - ) + # The session lasts as long as the resource manager's messages do. + # When they end, the CEM is closed, which stops what its handlers still have in flight, + # and the producer and the watchdog stop with it: left running, they outlived the connection, + # and a resource manager that reconnected was served by two sessions planning for it at once. + consumer = asyncio.create_task(websocket_consumer(ws, cem)) + side_tasks = [ + asyncio.create_task(websocket_producer(ws, cem)), + asyncio.create_task(rm_details_watchdog(ws, cem)), + ] + try: + await consumer + finally: + await cem.close() + for task in side_tasks: + task.cancel() + await asyncio.gather(*side_tasks, return_exceptions=True) async def configure_site( diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 37f3aaff..190f1841 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -242,3 +242,73 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" assert response["status"] == "OK" + + +async def _details_received(cem, rm_handshake, resource_manager_details) -> None: + """Handshake and details, with every answer taken off the queue: the CEM selects no control type yet.""" + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + while not cem._sending_queue.empty(): + await cem.get_message() + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_control_types_message_activates_it_where_its_confirmation_was_lost( + frbc_system_description, resource_manager_details, rm_handshake +): + """A resource manager sends FRBC messages only once it has accepted FRBC, so one arriving is that acceptance. + + Without this, a lost ReceptionStatus for the SelectControlType left every FRBC message answered with a + TEMPORARY_ERROR, which the resource manager retried forever, and a whole co-simulation hung at its first step. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_control_type_without_a_handler_is_not_activated_implicitly( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + await _details_received(cem, rm_handshake, resource_manager_details) + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + + +@pytest.mark.asyncio +async def test_a_message_before_the_details_activates_nothing(frbc_system_description): + """Before its details, a resource manager has not been offered a control type, so it cannot have accepted one.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await cem.handle_message(frbc_system_description) + assert cem.control_type is None + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_what_its_handlers_have_in_flight(): + """A schedule still being planned for a session that has ended would post and plan for a resource manager no longer there.""" + import asyncio + + cem = CEM(fm_client=None) + frbc = FRBCTest() + cem.register_control_type(frbc) + planning = asyncio.create_task(asyncio.sleep(3600)) + frbc.background_tasks.add(planning) + + await cem.close() + await asyncio.wait([planning], timeout=1) + + assert planning.cancelled() + planning.cancel() # in case it was not, so the test does not leave it running diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 952c2867..50c65bd7 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -763,3 +763,36 @@ def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): for graph in graphs: assert set(graph) == {"title", "sensors"} assert all(isinstance(i, int) for i in graph["sensors"]) + + +@pytest.mark.asyncio +async def test_a_session_ends_with_its_connection(): + """Its producer, its watchdog and its planning stop, so a resource manager that reconnects is served by one session.""" + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0].is_closed() + leftover = [ + t + for t in asyncio.all_tasks() + if t is not asyncio.current_task() + and getattr(t.get_coro(), "__qualname__", "") + in ("websocket_producer", "rm_details_watchdog") + ] + assert leftover == [] diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index c35ec8b1..046c4fb2 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -541,3 +541,40 @@ async def test_belief_times_are_forgotten_for_steps_more_than_a_day_old(): assert list(frbc._power_recorded_at) == [ datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc) ] + + +@pytest.mark.asyncio +async def test_a_description_replaced_while_its_plan_is_out_sends_only_the_newer_plan(): + """The plan for a description that a newer one replaced during the round trip is for a device that no longer is.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(storage_status()) + await settle(frbc) + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + newest = max(n for n, _ in batches) + assert sent[newest] > 0 + assert all(count == 0 for n, count in batches if n != newest) From 1cc6261a6c4cf15691a01d7b6c0e9acea5b1b1d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:49:15 +0100 Subject: [PATCH 38/59] test(S2): a message without a type activates nothing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_cem.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 190f1841..6a6fcb0a 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -312,3 +312,16 @@ async def test_closing_the_cem_stops_what_its_handlers_have_in_flight(): assert planning.cancelled() planning.cancel() # in case it was not, so the test does not leave it running + + +@pytest.mark.asyncio +async def test_a_message_without_a_type_activates_nothing( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + from flexmeasures_client.s2.utils import get_unique_id + + await cem.handle_message({"message_id": get_unique_id()}) + assert cem.control_type == ControlType.NO_SELECTION From 9053b861a40626771cfd1a0439932f47119b9f29 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 20:49:50 +0100 Subject: [PATCH 39/59] docs(changelog): implicit activation and session teardown Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 101f6b47..4ae271a1 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -51,6 +51,8 @@ Removed Fixed ----- +* A control type's message arriving while no control type is selected activates that control type, where the CEM has a handler for it, since a resource manager sends it only after accepting the selection. A lost confirmation used to leave every such message answered with a temporary error, which a resource manager retries forever [`#263 `__] +* Closing a control-type handler cancels the work it still has in flight, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] From 6b1d3eaa0a14f0db41eb61b1d7eb220d6f737593 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:03:50 +0100 Subject: [PATCH 40/59] fix(S2): take only the control type being selected as confirmed, and close any handler What the review of this PR found: - Implicit activation activated whichever control type a message belonged to, so a PEBC message while the CEM selected FRBC switched it to PEBC for good, and the watchdog waited for FRBC forever. Now only a message of the control type the CEM is selecting counts as the resource manager's confirmation, only from NO_SELECTION (a switch between control types is confirmed to the active handler, which is left to it), and the pending confirmation callback is then removed rather than left behind. - ControlTypeHandler.close() read background_tasks, which only FRBC handlers have, so closing any other handler raised AttributeError, which stopped CEM.close() and skipped the teardown of the producer and the watchdog: the very leak this PR set out to fix. A handler without them has nothing to stop. - The account of the hang is what #188 saw, not a lost confirmation it never established: a CEM left in NO_SELECTION while a co-simulation's reconnecting resource manager kept sending FRBC messages and retrying their temporary errors. The docstrings, the changelog and the close() comment now say so, and that closing stops recording in flight too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 4 +- src/flexmeasures_client/s2/cem.py | 41 +++++---- .../s2/control_types/__init__.py | 18 ++-- tests/s2/test_cem.py | 86 ++++++++++++++++++- 4 files changed, 119 insertions(+), 30 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 4ae271a1..d67516f4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -51,8 +51,8 @@ Removed Fixed ----- -* A control type's message arriving while no control type is selected activates that control type, where the CEM has a handler for it, since a resource manager sends it only after accepting the selection. A lost confirmation used to leave every such message answered with a temporary error, which a resource manager retries forever [`#263 `__] -* Closing a control-type handler cancels the work it still has in flight, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] +* A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] +* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] * A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] * On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index ceb27934..f46c4fdb 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -97,6 +97,8 @@ def __init__( self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() + #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. + self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type if not logger: @@ -231,35 +233,42 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): await self._sending_queue.put(response) def _activate_implicitly(self, message: dict) -> None: - """Take a control type's message as proof that the resource manager selected that control type. - - A resource manager sends a control type's messages only after accepting the SelectControlType for it. - So where one arrives while no control type is selected here, the confirmation was lost on the way, - and without this every such message was answered with a TEMPORARY_ERROR, which a resource manager retries forever. - Only a control type this CEM has a handler for is activated. + """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. + + A resource manager sends a control type's messages only once that control type is active on its side. + So where one arrives for the control type this CEM asked for, while the CEM still waits for the confirmation, + the resource manager is working in it already. #188 saw the CEM stay in NO_SELECTION while a resource manager, + reconnecting during setup in a co-simulation, kept sending FRBC messages, each answered with a TEMPORARY_ERROR + and retried; whether the confirmation was lost or never sent was not established. + Only the control type being selected is activated, never another one a message happens to belong to, + and its pending confirmation is then no longer waited for. """ - if self._control_type != ControlType.NO_SELECTION: + if self._control_type != ControlType.NO_SELECTION or self._selecting is None: return + control_type, selection_message_id = self._selecting message_type = message.get("message_type") if not isinstance(message_type, str): return - control_type = next( + prefix = next( ( - ct - for prefix, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() - if message_type.startswith(prefix) + p + for p, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() + if ct == control_type ), None, ) - if control_type is None or control_type not in self._control_types_handlers: + if prefix is None or not message_type.startswith(prefix): + return + if control_type not in self._control_types_handlers: return self._logger.warning( - "Received %s while no control type is selected, which a resource manager sends only after accepting %s; " - "taking the confirmation as lost and activating it.", + "Received %s before the resource manager confirmed the selection of %s; taking it as the confirmation.", message_type, control_type.value, ) - self._control_type = control_type + self.success_callbacks.pop(selection_message_id, None) + self.failure_callbacks.pop(selection_message_id, None) + self.update_control_type(control_type) def update_control_type(self, control_type: ControlType): """ @@ -267,6 +276,7 @@ def update_control_type(self, control_type: ControlType): a confirmation that the message has been received. """ self._control_type = control_type + self._selecting = None async def get_message(self) -> dict: """Call this function to get the messages to be sent to the RM @@ -317,6 +327,7 @@ async def activate_control_type( message_id, self.update_control_type, control_type=control_type ) + self._selecting = (control_type, message_id) await self._sending_queue.put( SelectControlType(message_id=message_id, control_type=control_type) ) diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index 70f32b72..c708c2b2 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -44,19 +44,19 @@ async def send_message(self, message: BaseModel) -> None: await self._sending_queue.put(message) async def close(self): - """Release any resources / stop recurring tasks for this handler. + """Stop this handler's work in flight, when the CEM closes as its websocket tears down. - Default no-op so CEM.close(), which calls close() on every registered handler when a - websocket tears down, works for any control-type handler. - Subclasses that own recurring tasks override this. - Without it, a handler lacking close() raised AttributeError inside CEM.close(), + Defined here so that CEM.close(), which calls close() on every registered handler, works for any of them: + a handler lacking close() raised AttributeError inside CEM.close(), killing the CEM's request handler and hanging the resource manager. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") - # Work still in flight, such as a schedule being planned, would otherwise go on after the session has ended: - # it posts to FlexMeasures and plans for a resource manager that is no longer there, - # racing the session that replaces it when the resource manager reconnects. - for task in list(self.background_tasks): + # Work still in flight would otherwise go on after the session has ended, + # planning for a resource manager that is no longer there and racing the session that replaces it. + # That includes recording what the resource manager last reported, which is stopped too: + # a reading cut off mid-post is reported again by a resource manager that reconnects. + # Not every handler keeps background tasks, so a handler without them has nothing to stop. + for task in list(getattr(self, "background_tasks", ())): task.cancel() @register(InstructionStatusUpdate) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 6a6fcb0a..752793da 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -4,6 +4,7 @@ from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types import ControlTypeHandler from flexmeasures_client.s2.control_types.FRBC import FRBCTest @@ -244,6 +245,12 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh assert response["status"] == "OK" +class _PlainPEBC(ControlTypeHandler): + """A handler for a second control type, which keeps no background tasks.""" + + _control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + + async def _details_received(cem, rm_handshake, resource_manager_details) -> None: """Handshake and details, with every answer taken off the queue: the CEM selects no control type yet.""" await cem.handle_message(rm_handshake) @@ -254,23 +261,81 @@ async def _details_received(cem, rm_handshake, resource_manager_details) -> None @pytest.mark.asyncio -async def test_a_control_types_message_activates_it_where_its_confirmation_was_lost( +async def test_a_message_of_the_control_type_being_selected_counts_as_its_confirmation( frbc_system_description, resource_manager_details, rm_handshake ): - """A resource manager sends FRBC messages only once it has accepted FRBC, so one arriving is that acceptance. + """A resource manager sends a control type's messages only once it is working in that control type. - Without this, a lost ReceptionStatus for the SelectControlType left every FRBC message answered with a - TEMPORARY_ERROR, which the resource manager retried forever, and a whole co-simulation hung at its first step. + #188 saw a CEM stay in NO_SELECTION while a reconnecting resource manager kept sending FRBC messages, + each answered with a temporary error and retried. """ cem = CEM(fm_client=None) cem.register_control_type(FRBCTest()) await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() await cem.handle_message(frbc_system_description) assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL response = await cem.get_message() assert response["status"] == ReceptionStatusValues.OK.value + assert selection["message_id"] not in cem.success_callbacks # no longer waited for + + +@pytest.mark.asyncio +async def test_a_message_of_another_control_type_is_not_taken_as_a_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """Selecting one control type and receiving another's message switches to neither.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.POWER_ENVELOPE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_switch_between_control_types_is_left_to_its_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """While one control type is active, a message of the one being switched to is still that of an unconfirmed switch. + + Its confirmation is awaited by the active handler, which this CEM would not clear, so it is left to arrive. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.POWER_ENVELOPE_BASED_CONTROL + + +@pytest.mark.asyncio +async def test_a_confirmed_selection_is_no_longer_pending( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], + status=ReceptionStatusValues.OK, + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + assert cem._selecting is None @pytest.mark.asyncio @@ -279,6 +344,7 @@ async def test_a_control_type_without_a_handler_is_not_activated_implicitly( ): cem = CEM(fm_client=None) await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") await cem.handle_message(frbc_system_description) @@ -325,3 +391,15 @@ async def test_a_message_without_a_type_activates_nothing( await cem.handle_message({"message_id": get_unique_id()}) assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_handler_without_background_tasks_closes(): + """Closing every handler is what ends a session, so one that keeps no tasks must close too.""" + cem = CEM(fm_client=None) + plain = _PlainPEBC() + cem.register_control_type(plain) + if hasattr(plain, "background_tasks"): + del plain.background_tasks + await cem.close() + assert cem.is_closed() From 69f5dbed16d616ec3aa1d5b2fe0be6eacf76d79b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:04:45 +0100 Subject: [PATCH 41/59] test(S2): a typeless message activates nothing while a selection is pending Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- tests/s2/test_cem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 752793da..4299fa8a 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -387,6 +387,7 @@ async def test_a_message_without_a_type_activates_nothing( cem = CEM(fm_client=None) cem.register_control_type(FRBCTest()) await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") from flexmeasures_client.s2.utils import get_unique_id await cem.handle_message({"message_id": get_unique_id()}) From 80ad945dfba7862fb6608e01535744acb62d7026 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:17:10 +0100 Subject: [PATCH 42/59] feat(S2): a clock the CEM and its handlers go by, which a co-simulation can set Carved from #188, rewritten for the library rather than transcribed. #188 let a co-simulation's resource manager wrap each S2 message with the simulated time, {"message": ..., "metadata": {"dt": ...}}, and patched a lambda returning that time onto the CEM and every handler, for every resource manager, live ones included. The lambda returned dt as it came, an ISO string, and it reached neither the CEM's timer, its measurement binning, nor TUNES' timer, which kept the wall clock. - flexmeasures_client.s2.clock holds a Clock protocol, a WallClock, and a SimulatedClock, which stands still between settings, refuses a time without a zone, and goes by the wall clock until it is first set: the handshake of each connection runs before any simulated time has arrived. - Handler.now() reads its handler's clock, in its time zone. The CEM gives its clock to every handler it registers, so they share one time. FRBCSimple and TUNES lose their copies of now(). The CEM's and TUNES' timers, and the CEM's measurement binning, go by now() instead of the wall clock; a timer handed in without a zone, as they all were, is read in the handler's time zone. - CEM(accept_simulation_metadata=True) takes the wrapper, sets its clock to the time it carries and handles the message inside. It needs a SimulatedClock and makes one where none is given. Off, which is the default, a wrapped message is refused with INVALID_MESSAGE and the reason, since the wrapper is not S2 and a live CEM should not let a resource manager set its time. A time that cannot be read is refused the same way. - The example server turns it on with S2_ACCEPT_SIMULATION_METADATA, which the co-simulation's stack will set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/__init__.py | 8 +- src/flexmeasures_client/s2/cem.py | 71 +++++- src/flexmeasures_client/s2/clock.py | 55 +++++ .../s2/control_types/FRBC/frbc_simple.py | 3 - .../s2/control_types/FRBC/frbc_tunes.py | 9 +- .../s2/script/websockets_server.py | 9 + tests/s2/test_clock.py | 208 ++++++++++++++++++ tests/s2/test_example_end_to_end.py | 27 +++ 8 files changed, 379 insertions(+), 11 deletions(-) create mode 100644 src/flexmeasures_client/s2/clock.py create mode 100644 tests/s2/test_clock.py diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index a1fcadc5..04a7cbcb 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -21,6 +21,7 @@ ) +from flexmeasures_client.s2.clock import Clock, WallClock from flexmeasures_client.s2.utils import SizeLimitOrderedDict, get_message_id @@ -126,6 +127,8 @@ def __init__(self, max_size: int = 100, timezone: str = "UTC") -> None: self.outgoing_messages_status = SizeLimitOrderedDict(max_size=max_size) self._timezone = ZoneInfo(timezone) + #: The time this handler goes by; a CEM gives each handler it registers its own clock. + self._clock: Clock = WallClock() self.discover() @@ -249,5 +252,6 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: subject_message_id=str(message.message_id), status=ReceptionStatusValues.OK ) - def now(self): - return datetime.now(self._timezone) + def now(self) -> datetime: + """The current moment by this handler's clock, in its time zone.""" + return self._clock.now().astimezone(self._timezone) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index f46c4fdb..5f2676d9 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -33,12 +33,14 @@ from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2 import Handler, register +from flexmeasures_client.s2.clock import Clock, SimulatedClock from flexmeasures_client.s2.control_types import ControlTypeHandler from flexmeasures_client.s2.utils import ( get_latest_compatible_version, get_reception_status, get_unique_id, ) +from flexmeasures_client.s2.wrapper import MetaData _LOGGER = logging.getLogger(__name__) @@ -85,12 +87,32 @@ def __init__( timers: dict[str, datetime] | None = None, datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, + clock: Clock | None = None, + accept_simulation_metadata: bool = False, **kwargs, ) -> None: """ Customer Energy Manager (CEM) + + :param clock: the time this CEM and every handler it registers go by; + the wall clock by default. + :param accept_simulation_metadata: whether to take messages wrapped with a simulated time + (see flexmeasures_client.s2.wrapper.S2Wrapper), + as a co-simulation sends them, and set the clock to that time. + Off by default, since the wrapper is not part of S2, + and a live CEM should not let a resource manager set its time. + On, the clock has to be a SimulatedClock, and one is made where none is given. """ super(CEM, self).__init__() + if accept_simulation_metadata: + clock = clock if clock is not None else SimulatedClock() + if not isinstance(clock, SimulatedClock): + raise ValueError( + "A CEM that accepts simulation metadata sets its clock from it, so it needs a SimulatedClock." + ) + if clock is not None: + self._clock = clock + self._accept_simulation_metadata = accept_simulation_metadata self._fm_client = fm_client self._sending_queue = Queue() @@ -111,8 +133,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. + due_time = due_time.replace(tzinfo=self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -169,6 +195,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): # add fm_client to control_type handler control_type_handler._fm_client = self._fm_client + # The handler goes by the CEM's time, which a co-simulation sets. + control_type_handler._clock = self._clock + # add sending queue control_type_handler._sending_queue = self._sending_queue @@ -198,6 +227,12 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): message, dict ) # what a message is from here on, whatever form it came in + if "metadata" in message and "message" in message: + unwrapped = self._unwrap(message) + if unwrapped is None: + return + message = unwrapped + self._logger.debug(f"Received: {message}") self._activate_implicitly(message) @@ -232,6 +267,38 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if response is not None: await self._sending_queue.put(response) + def _unwrap(self, wrapped: dict) -> dict | None: + """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. + + Where this CEM does not accept the wrapper, or the time cannot be read, the message is refused, + and None is returned so that it is not handled. + """ + inner = wrapped["message"] if isinstance(wrapped["message"], dict) else {} + reason = None + if not self._accept_simulation_metadata: + reason = ( + "This CEM does not accept messages wrapped with simulation metadata." + ) + else: + try: + moment = MetaData.model_validate(wrapped["metadata"]).dt + assert isinstance(self._clock, SimulatedClock) + self._clock.set(moment) + except (pydantic.ValidationError, ValueError, TypeError) as exc: + reason = f"The simulated time in this message's metadata cannot be used: {exc}" + if reason is None: + return inner + self._logger.warning("%s Refusing %s.", reason, inner.get("message_type")) + if inner.get("message_id") is not None: + self._sending_queue.put_nowait( + ReceptionStatus( + subject_message_id=inner["message_id"], + status=ReceptionStatusValues.INVALID_MESSAGE, + diagnostic_label=reason, + ) + ) + return None + def _activate_implicitly(self, message: dict) -> None: """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. @@ -399,7 +466,7 @@ async def handle_power_measurement(self, message: PowerMeasurement): ) # Compute bin - now = datetime.now(self._timezone) + now = self.now() m = self._minimum_measurement_period // pd.Timedelta(minutes=1) bin_end = now.replace( second=0, microsecond=0, minute=(now.minute // m) * m diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py new file mode 100644 index 00000000..15fdaa2f --- /dev/null +++ b/src/flexmeasures_client/s2/clock.py @@ -0,0 +1,55 @@ +"""The time an S2 CEM and its handlers go by. + +A CEM that runs live goes by the wall clock. +One that steers a co-simulation goes by the time the simulation says it is, +which a resource manager sends along with its messages (see CEM's accept_simulation_metadata), +so that what the CEM records and plans is placed at simulated moments rather than at the moment it ran. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Protocol + + +class Clock(Protocol): + def now(self) -> datetime: + """The current moment, timezone-aware.""" + ... + + +class WallClock: + """The time it is.""" + + def now(self) -> datetime: + return datetime.now(timezone.utc) + + +class SimulatedClock: + """The time a simulation says it is, as last set. + + It stands still between settings, as a simulation's time does between the messages that carry it. + Until it is first set it goes by the wall clock, which is the time the handshake of each connection runs on, + since a resource manager sends the simulated time only with the messages that follow it. + """ + + def __init__(self, start: datetime | None = None) -> None: + self._now: datetime | None = None + if start is not None: + self.set(start) + + def set(self, moment: datetime) -> None: + if moment.tzinfo is None or moment.utcoffset() is None: + raise ValueError( + f"A simulated time has to say which time zone it is in, and {moment.isoformat()} does not." + ) + self._now = moment + + @property + def is_set(self) -> bool: + return self._now is not None + + def now(self) -> datetime: + if self._now is None: + return datetime.now(timezone.utc) + return self._now diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 694541e2..ccd43d1f 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -284,9 +284,6 @@ def __init__( # of the FRBC.SystemDescription. self._valid_from_shift = valid_from_shift - def now(self): - return datetime.now(self._timezone) - def fill_rate_unit(self) -> str: """The unit a rate of change of the fill level is recorded in. diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index c6a33e50..ea11080a 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -169,8 +169,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. + due_time = due_time.replace(tzinfo=self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -195,9 +199,6 @@ def _is_timer_due(self, name: str) -> bool: ) return False - def now(self): - return datetime.now(self._timezone) - async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 279e7605..9c3f9988 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -32,6 +32,13 @@ # Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, # with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 + +#: Whether resource managers may send messages wrapped with a simulated time, as a co-simulation does. +ACCEPT_SIMULATION_METADATA = os.getenv("S2_ACCEPT_SIMULATION_METADATA", "").lower() in ( + "1", + "true", + "yes", +) SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. @@ -141,6 +148,8 @@ async def serve_resource_manager( sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, + # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. + accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) frbc = FRBCSimple( power_sensor_id=sensors["power"]["id"], diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py new file mode 100644 index 00000000..f1394969 --- /dev/null +++ b/tests/s2/test_clock.py @@ -0,0 +1,208 @@ +"""The time an S2 CEM goes by, and a co-simulation's way of setting it.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +import pandas as pd +import pytest +from s2python.common import ReceptionStatusValues + +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.clock import SimulatedClock, WallClock +from flexmeasures_client.s2.control_types.FRBC import FRBCTest +from flexmeasures_client.s2.utils import get_unique_id + +SIMULATED = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) + + +def test_a_simulated_clock_stands_still_between_settings(): + clock = SimulatedClock() + clock.set(SIMULATED) + assert clock.now() == SIMULATED + assert clock.now() == SIMULATED + + +def test_a_simulated_clock_goes_by_the_wall_clock_until_it_is_set(): + clock = SimulatedClock() + assert not clock.is_set + assert abs(clock.now() - WallClock().now()) < timedelta(seconds=5) + + +def test_a_simulated_time_without_a_zone_is_refused(): + """#188 took naive times as Europe/Amsterdam, which a simulation in UTC shifts by an hour or two.""" + with pytest.raises(ValueError, match="time zone"): + SimulatedClock().set(datetime(2022, 3, 1, 12, 0)) + + +def test_a_handler_registered_with_a_cem_goes_by_its_clock(): + clock = SimulatedClock(SIMULATED) + cem = CEM(fm_client=None, clock=clock) + frbc = FRBCTest() + cem.register_control_type(frbc) + assert frbc.now() == SIMULATED + assert cem.now() == SIMULATED + later = SIMULATED + timedelta(hours=3) + clock.set(later) + assert frbc.now() == later # the same clock, not a copy of its time + + +def test_now_is_given_in_the_handlers_time_zone(): + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED)) + frbc = FRBCTest() + frbc._timezone = ZoneInfo("Europe/Amsterdam") + cem.register_control_type(frbc) + assert frbc.now().tzinfo == ZoneInfo("Europe/Amsterdam") + assert frbc.now() == SIMULATED + + +def test_accepting_simulation_metadata_needs_a_simulated_clock(): + with pytest.raises(ValueError, match="SimulatedClock"): + CEM(fm_client=None, clock=WallClock(), accept_simulation_metadata=True) + + +def _wrapped(message: dict, dt) -> dict: + return {"message": message, "metadata": {"dt": dt}} + + +@pytest.mark.asyncio +async def test_a_wrapped_message_sets_the_clock_and_is_handled(rm_handshake): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + _wrapped(rm_handshake.model_dump(mode="json"), "2022-03-01T12:00:00+00:00") + ) + assert cem.now() == SIMULATED + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_wrapped_message_is_refused_by_a_cem_that_does_not_accept_it( + rm_handshake, +): + """The wrapper is not S2, and a live CEM does not let a resource manager set its time.""" + cem = CEM(fm_client=None) + before = cem.now() + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, "2022-03-01T12:00:00+00:00")) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert reception["subject_message_id"] == message["message_id"] + assert cem._sending_queue.empty() # the handshake itself was not answered + assert cem.now() - before < timedelta(seconds=5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dt", ["2022-03-01T12:00:00", "not a time"]) +async def test_a_wrapped_message_with_an_unusable_time_is_refused(rm_handshake, dt): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, dt)) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert not cem._clock.is_set + + +@pytest.mark.asyncio +async def test_a_wrapper_around_no_message_is_refused_without_an_answer(): + """Nothing to answer: a reception status needs the id of the message it is about.""" + cem = CEM(fm_client=None) + await cem.handle_message({"message": "garbage", "metadata": {}}) + assert cem._sending_queue.empty() + + +def test_a_cem_timer_runs_on_simulated_time(): + # Off a period boundary: on one, the next due time rounds to the moment itself. + start = SIMULATED + timedelta(minutes=1) + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + assert cem._is_timer_due("power") + assert not cem._is_timer_due("power") + clock.set(start + timedelta(hours=1)) + assert cem._is_timer_due("power") + + +def test_a_timer_handed_in_without_a_zone_is_read_in_the_handlers_zone(): + cem = CEM( + fm_client=None, + clock=SimulatedClock(SIMULATED), + timers={"power": datetime(2022, 3, 1, 13, 0)}, + ) + assert not cem._is_timer_due("power") + + +def test_the_example_server_accepts_simulation_metadata_only_when_told_to(monkeypatch): + import importlib + + from flexmeasures_client.s2.script import websockets_server + + monkeypatch.setenv("S2_ACCEPT_SIMULATION_METADATA", "1") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is True + monkeypatch.delenv("S2_ACCEPT_SIMULATION_METADATA") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is False + + +def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) + + start = SIMULATED + timedelta(minutes=1) + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, + ) + cem.register_control_type(tunes) + tunes._timers["usage_forecast"] = datetime( + 2022, 3, 1, 12, 30 + ) # zoneless, as timers used to be + assert not tunes._is_timer_due("usage_forecast") + clock.set(start + timedelta(hours=1)) + assert tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +async def test_a_power_measurement_is_binned_by_simulated_time(): + """Binned by the wall clock, a co-simulation's measurements, stamped in 2022, fell outside every bin and were dropped.""" + from unittest.mock import AsyncMock + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.client import FlexMeasuresClient + + fm = AsyncMock(FlexMeasuresClient) + clock = SimulatedClock(datetime(2022, 3, 1, 12, 7, tzinfo=timezone.utc)) + cem = CEM(fm_client=fm, clock=clock, power_sensor_id={"ELECTRIC.POWER.L1": 7}) + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, 2, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + posted = fm.post_sensor_data.await_args + assert posted is not None + assert pd.Timestamp(posted.kwargs["start"]) == pd.Timestamp( + "2022-03-01T12:00:00+00:00" + ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 50c65bd7..e2eca6c4 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -796,3 +796,30 @@ def capture(**kwargs): in ("websocket_producer", "rm_details_watchdog") ] assert leftover == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("accept", [False, True]) +async def test_the_server_hands_its_simulation_flag_to_each_cem(accept): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with ( + patch.object(server, "CEM", side_effect=capture), + patch.object(server, "ACCEPT_SIMULATION_METADATA", accept), + ): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["accept_simulation_metadata"] is accept From 5b9793ec286ca1d4818db0337d04e312e68c85fc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:18:17 +0100 Subject: [PATCH 43/59] docs(changelog): the clock and the simulation metadata Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d67516f4..30f19ad3 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -25,6 +25,8 @@ Added * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] +* ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] Changed ------- From 47c695a262d74a71903bc410ac3ddb606dfd7d33 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:30:34 +0100 Subject: [PATCH 44/59] fix(S2): what the review of the clock found - On a period boundary, the timers' next due time rounded to the moment itself, so a clock standing still there, as a co-simulation's does between messages, found every timer due on every call. The next due time is now a whole period later. Both tests had started a minute off the boundary to dodge it; they now start on it. - A wrapper with no S2 message inside set the clock, and then crashed the consumer on a reception status for a message without an id. It is refused before the clock is touched. - A timer handed in without a zone was compared with the host's local time before, and is read as that again, where the first version read it in the handler's time zone, which is UTC by default. - The measurement binning was claimed as moved onto simulated time, but binning on now() never catches a co-simulation's measurements, each stamped at the start of a quarter hour that has passed by the time it arrives. #188 bypassed the binning instead. The claim and its test are dropped, and posting a co-simulation's measurements is left to the recorder that is planned next. - Docs: the co-simulation's resource manager wraps every message from its handshake on, so a CEM without accept_simulation_metadata refuses it from the first message, which the docstring now says; the simulated clock's docstring no longer claims the handshake runs on wall time, and says that background work reads the clock as it stands. The server's flag moved out of the middle of the price constants, and a docstring line breaks after punctuation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 28 ++++-- src/flexmeasures_client/s2/clock.py | 5 +- .../s2/control_types/FRBC/frbc_tunes.py | 8 +- .../s2/script/websockets_server.py | 2 +- tests/s2/test_clock.py | 92 +++++++++++-------- 5 files changed, 81 insertions(+), 54 deletions(-) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 5f2676d9..e1adf215 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -96,11 +96,12 @@ def __init__( :param clock: the time this CEM and every handler it registers go by; the wall clock by default. - :param accept_simulation_metadata: whether to take messages wrapped with a simulated time - (see flexmeasures_client.s2.wrapper.S2Wrapper), - as a co-simulation sends them, and set the clock to that time. + :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, + and set the clock to that time (see flexmeasures_client.s2.wrapper.S2Wrapper). Off by default, since the wrapper is not part of S2, and a live CEM should not let a resource manager set its time. + A resource manager that wraps its messages, from its handshake on, + is then refused with INVALID_MESSAGE and cannot connect. On, the clock has to be a SimulatedClock, and one is made where none is given. """ super(CEM, self).__init__() @@ -137,8 +138,8 @@ def _is_timer_due(self, name: str) -> bool: now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. - due_time = due_time.replace(tzinfo=self._timezone) + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -155,6 +156,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: @@ -270,12 +275,17 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): def _unwrap(self, wrapped: dict) -> dict | None: """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. - Where this CEM does not accept the wrapper, or the time cannot be read, the message is refused, - and None is returned so that it is not handled. + Where this CEM does not accept the wrapper, the time cannot be read, or there is no message inside, + the message is refused, the clock is left as it was, and None is returned so that nothing is handled. """ - inner = wrapped["message"] if isinstance(wrapped["message"], dict) else {} + inner = wrapped["message"] reason = None - if not self._accept_simulation_metadata: + if not isinstance(inner, dict) or not isinstance( + inner.get("message_type"), str + ): + reason = "There is no S2 message inside this wrapper." + inner = inner if isinstance(inner, dict) else {} + elif not self._accept_simulation_metadata: reason = ( "This CEM does not accept messages wrapped with simulation metadata." ) diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py index 15fdaa2f..55646905 100644 --- a/src/flexmeasures_client/s2/clock.py +++ b/src/flexmeasures_client/s2/clock.py @@ -29,8 +29,9 @@ class SimulatedClock: """The time a simulation says it is, as last set. It stands still between settings, as a simulation's time does between the messages that carry it. - Until it is first set it goes by the wall clock, which is the time the handshake of each connection runs on, - since a resource manager sends the simulated time only with the messages that follow it. + Until it is first set it goes by the wall clock, for a resource manager that does not wrap its first messages; + one that wraps every message from its handshake on, as the co-simulation's does, sets it with the first. + Background work, such as a schedule being planned, reads it as it stands, which may be a later message's time. """ def __init__(self, start: datetime | None = None) -> None: diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index ea11080a..23babaf3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -173,8 +173,8 @@ def _is_timer_due(self, name: str) -> bool: now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) if due_time.tzinfo is None: - # A timer handed in as a wall-clock time without a zone, as they all were before, is read in this handler's. - due_time = due_time.replace(tzinfo=self._timezone) + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -191,6 +191,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 9c3f9988..21cdd974 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -32,6 +32,7 @@ # Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, # with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 +SEED_PRICES_EVERY = timedelta(hours=12) #: Whether resource managers may send messages wrapped with a simulated time, as a co-simulation does. ACCEPT_SIMULATION_METADATA = os.getenv("S2_ACCEPT_SIMULATION_METADATA", "").lower() in ( @@ -39,7 +40,6 @@ "true", "yes", ) -SEED_PRICES_EVERY = timedelta(hours=12) #: Each sensor the site needs: its unit, its resolution, and any attributes. #: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py index f1394969..946bae72 100644 --- a/tests/s2/test_clock.py +++ b/tests/s2/test_clock.py @@ -5,14 +5,12 @@ from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo -import pandas as pd import pytest from s2python.common import ReceptionStatusValues from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.clock import SimulatedClock, WallClock from flexmeasures_client.s2.control_types.FRBC import FRBCTest -from flexmeasures_client.s2.utils import get_unique_id SIMULATED = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) @@ -113,23 +111,22 @@ async def test_a_wrapper_around_no_message_is_refused_without_an_answer(): def test_a_cem_timer_runs_on_simulated_time(): - # Off a period boundary: on one, the next due time rounds to the moment itself. - start = SIMULATED + timedelta(minutes=1) - clock = SimulatedClock(start) + # On a period boundary, which a co-simulation's times are: the ceiling of the moment is the moment itself. + clock = SimulatedClock(SIMULATED) cem = CEM(fm_client=None, clock=clock) assert cem._is_timer_due("power") - assert not cem._is_timer_due("power") - clock.set(start + timedelta(hours=1)) + assert not cem._is_timer_due("power") # a clock standing still on a boundary + clock.set(SIMULATED + timedelta(hours=1)) assert cem._is_timer_due("power") -def test_a_timer_handed_in_without_a_zone_is_read_in_the_handlers_zone(): - cem = CEM( - fm_client=None, - clock=SimulatedClock(SIMULATED), - timers={"power": datetime(2022, 3, 1, 13, 0)}, - ) +def test_a_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + """As it was compared before, with a zoneless datetime.now(), which is the host's local time.""" + local = (SIMULATED + timedelta(minutes=30)).astimezone().replace(tzinfo=None) + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED), timers={"power": local}) assert not cem._is_timer_due("power") + cem._clock.set(SIMULATED + timedelta(minutes=31)) + assert cem._is_timer_due("power") def test_the_example_server_accepts_simulation_metadata_only_when_told_to(monkeypatch): @@ -148,7 +145,7 @@ def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): FillRateBasedControlTUNES, ) - start = SIMULATED + timedelta(minutes=1) + start = SIMULATED clock = SimulatedClock(start) cem = CEM(fm_client=None, clock=clock) tunes = FillRateBasedControlTUNES( @@ -170,39 +167,54 @@ def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): state_of_charge_sensor_id=18, ) cem.register_control_type(tunes) - tunes._timers["usage_forecast"] = datetime( - 2022, 3, 1, 12, 30 - ) # zoneless, as timers used to be - assert not tunes._is_timer_due("usage_forecast") + assert tunes._is_timer_due("usage_forecast") + assert not tunes._is_timer_due( + "usage_forecast" + ) # a clock standing still on a boundary clock.set(start + timedelta(hours=1)) assert tunes._is_timer_due("usage_forecast") @pytest.mark.asyncio -async def test_a_power_measurement_is_binned_by_simulated_time(): - """Binned by the wall clock, a co-simulation's measurements, stamped in 2022, fell outside every bin and were dropped.""" - from unittest.mock import AsyncMock +@pytest.mark.parametrize("inner", ["garbage", {"no": "type"}]) +async def test_a_wrapper_around_no_message_leaves_the_clock_alone(inner): + """With the flag on, too: the clock is set only for a message that will be handled.""" + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + {"message": inner, "metadata": {"dt": "2022-03-01T12:00:00+00:00"}} + ) + assert not cem._clock.is_set + assert cem._sending_queue.empty() - from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue - from flexmeasures_client.client import FlexMeasuresClient +def test_a_tunes_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) - fm = AsyncMock(FlexMeasuresClient) - clock = SimulatedClock(datetime(2022, 3, 1, 12, 7, tzinfo=timezone.utc)) - cem = CEM(fm_client=fm, clock=clock, power_sensor_id={"ELECTRIC.POWER.L1": 7}) - await cem.handle_message( - PowerMeasurement( - message_id=get_unique_id(), - measurement_timestamp=datetime(2022, 3, 1, 12, 2, tzinfo=timezone.utc), - values=[ - PowerValue( - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 - ) - ], - ) + start = SIMULATED + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, ) - posted = fm.post_sensor_data.await_args - assert posted is not None - assert pd.Timestamp(posted.kwargs["start"]) == pd.Timestamp( - "2022-03-01T12:00:00+00:00" + cem.register_control_type(tunes) + tunes._timers["usage_forecast"] = ( + (start + timedelta(minutes=30)).astimezone().replace(tzinfo=None) ) + assert not tunes._is_timer_due("usage_forecast") From 6a12ea4cb5088eb7551bda9932b12909e156ffac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:44:35 +0100 Subject: [PATCH 45/59] feat(S2): a site of its own for each resource manager, and the co-simulation stack Carved from #188, rewritten for the library rather than transcribed. #188 made every CEM find or create a FlexMeasures asset named after the resource manager that connected, with its sensors (config_utils.configure_site, a 12-tuple, a module-level id cache and logging configured at import), and registered an FRBCSimple wired to them, for every deployment. It also seeded constant prices from the wall clock's today with a belief time in 2026, which a simulation of 2022 never reads. - flexmeasures_client.s2.site holds FRBC_SITE_SENSORS, the sensors an FRBC site records on, which the example's table becomes; ensure_site, which finds or makes a site of a name with its sensors and graphs, and is safe to run again; and FRBCSiteConfigurator, which gives each resource manager a site named after it and an FRBCSimple wired to it, planning from now over 24 hours as #188 did, with the asset type, time zone, asset specs and the power sensor's attributes as options, and the site's id remembered for a reconnect. The sensor names the co-simulation reads by ("price", "production price", "power", "measured-power", "state of charge") are kept. - CEM(site_configurator=...) configures in the background once a resource manager's details arrive, and wires the site's measured-power sensor. Until the handler exists, the active control type's messages are answered with TEMPORARY_ERROR, which the resource manager retries; before, the CEM raised a KeyError on them. Closing the CEM stops a configuration in flight. - The example server takes S2_SITE_PER_RESOURCE_MANAGER: no fixed site and no demo prices, and a configurator for the server's lifetime, whose power sensors say the heat pumps cannot produce (is_strictly_non_positive), as #188's did. Its fixed site is set up with ensure_site, and gains the production price and measured-power sensors. - examples/S2-co-simulation holds the docker-compose overlay the co-simulation runs on, moved from #188's root docker-compose.override.yml, which compose applied to anyone running the repo's base file. It sets both switches, uses s2-python as this repo locks it (0.8.1, which the co-simulation's resource managers use too, where #188 pinned 0.8.2), runs one worker command for all three workers, logs at INFO, and drops the py-spy debugging setup. Its README says how to keep it apart from a development database: FlexMeasures keeps that database's data inside its checkout, so a project name does not. Not carried from #188: mirroring measured power onto the community's aggregate-power sensor (the maintainer dropped it, since nothing reads it back), and the price seeding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 30 ++ .../S2-co-simulation/docker-compose.cem.yml | 164 +++++++++++ src/flexmeasures_client/s2/cem.py | 60 +++- .../s2/script/websockets_server.py | 177 +++++------- src/flexmeasures_client/s2/site.py | 264 ++++++++++++++++++ tests/s2/test_example_end_to_end.py | 83 +++++- tests/s2/test_site.py | 264 ++++++++++++++++++ 7 files changed, 933 insertions(+), 109 deletions(-) create mode 100644 examples/S2-co-simulation/README.md create mode 100644 examples/S2-co-simulation/docker-compose.cem.yml create mode 100644 src/flexmeasures_client/s2/site.py create mode 100644 tests/s2/test_site.py diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md new file mode 100644 index 00000000..fa95fa8e --- /dev/null +++ b/examples/S2-co-simulation/README.md @@ -0,0 +1,30 @@ +# A FlexMeasures stack for an S2 co-simulation + +`docker-compose.cem.yml` runs FlexMeasures with one S2 CEM per simulated home, +for a co-simulation whose resource managers connect over websockets, such as a +community of homes steered at once. How to start it is at the top of the file. + +Each CEM is the example server (`src/flexmeasures_client/s2/script/websockets_server.py`) +with two switches on: + +- `S2_ACCEPT_SIMULATION_METADATA`: a resource manager wraps each message with the + simulated time, `{"message": ..., "metadata": {"dt": ...}}`, and the CEM, and every + handler it registers, go by that time (see `flexmeasures_client.s2.clock`). + `dt` has to name its time zone. Without the switch, a wrapped message is refused + as invalid, from the handshake on. +- `S2_SITE_PER_RESOURCE_MANAGER`: each resource manager gets a site of its own, + named after it, with the sensors in `flexmeasures_client.s2.site.FRBC_SITE_SENSORS`. + Messages are answered with a temporary error until its site is set up, which a + resource manager retries. Its schedules start from the simulated now and cover + 24 hours. The site's prices are for whoever runs the simulation to post. + +A co-simulation reads the sites by sensor name: `price`, `production price`, +`power` (the schedule), `measured-power` (what the home measured) and +`state of charge`. + +The stack uses FlexMeasures' own development database service, `dev-db`, whose +data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the +flexmeasures checkout. A compose project name of its own does not separate that, +so run the stack from a checkout of its own (a git worktree will do) to keep a +development database you care about out of it. Its host ports, 5000, 5433, 6380 +and 8080 to 8084, have to be free too. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml new file mode 100644 index 00000000..da7f58dd --- /dev/null +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -0,0 +1,164 @@ +# ------------------------------------------------------------------ +# A FlexMeasures stack with one S2 CEM per simulated home, for a co-simulation +# whose resource managers connect over websockets, such as the community +# simulations that steer several homes at once. +# +# It is an overlay on FlexMeasures' own docker-compose.yml, and runs the CEMs +# from this checkout. With the flexmeasures and flexmeasures-client repos side +# by side, run it from the flexmeasures folder (which holds the Dockerfile): +# +# docker compose \ +# -f docker-compose.yml \ +# -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \ +# up +# +# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws +# and so on, one per home. Each gets a site of its own, named after it, and the +# CEM goes by the simulated time its messages carry (see README.md). +# ------------------------------------------------------------------ + +# Shared definition for every CEM. Each listens on port 8080 inside its own +# container; only the host port differs. +x-cem: &cem + build: + context: . + dockerfile: Dockerfile + image: flexmeasures-client-cem + depends_on: + - server + restart: always + environment: + FLEXMEASURES_BASE_URL: http://server:5000 + FLEXMEASURES_USER: toy-user@flexmeasures.io + FLEXMEASURES_PASSWORD: toy-password + LOGGING_LEVEL: INFO + # A resource manager in a co-simulation wraps each message with the simulated time. + S2_ACCEPT_SIMULATION_METADATA: "1" + # And each home is a site of its own, rather than all of them sharing one. + S2_SITE_PER_RESOURCE_MANAGER: "1" + # The checkout is mounted without its git history, which is where its version would come from. + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLEXMEASURES_CLIENT: "0.0.0" + volumes: + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + command: + - | + # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`): + # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv, + # which ships without pip and cannot see what the system pip installs. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + # s2-python as this repo locks it, and as the co-simulation's resource managers use it. + python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py + +# Shared definition for every RQ worker: one per home simulated at once, +# like the CEMs and the server's gunicorn workers below, since every home's +# scheduling jobs otherwise queue behind one worker process. +x-worker: &worker + # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with. + build: + context: . + dockerfile: Dockerfile + depends_on: + - dev-db + - queue-db + - mailhog + restart: on-failure + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" + FLEXMEASURES_REDIS_URL: queue-db + FLEXMEASURES_REDIS_PASSWORD: fm-redis-pass + SECRET_KEY: notsecret + SECURITY_TOTP_SECRETS: '{"1": "something-secret"}' + FLEXMEASURES_ENV: development + MAIL_SERVER: mailhog + MAIL_PORT: 1025 + LOGGING_LEVEL: INFO + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Drop any lingering registration for the ingestion queue before this worker starts. + # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, + # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. + # No worker here serves that queue, so removing those entries on every start is safe, and idempotent. + python3 - <<'PYGUARD' + import os, redis + r = redis.Redis(host=os.environ.get("FLEXMEASURES_REDIS_URL", "queue-db"), + port=int(os.environ.get("FLEXMEASURES_REDIS_PORT", 6379)), + password=os.environ.get("FLEXMEASURES_REDIS_PASSWORD")) + try: + for name in r.smembers("rq:workers"): + key = name.decode() + queues = (r.hget(key, "queues") or b"").decode() + if "ingestion" in queues or not queues: + r.srem("rq:workers", name); r.delete(key) + r.delete("rq:workers:ingestion") + except redis.RedisError as exc: + print(f"ingestion-registration guard skipped: {exc}") + PYGUARD + # Named after the container, which is unique per worker. + flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting + +services: + dev-db: + ports: + - "5433:5432" + queue-db: + ports: + - "6380:6379" + server: + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + command: + - | + pip install --break-system-packages -e /app + pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt + pip install --break-system-packages -e /app/flexmeasures-client[s2] + flexmeasures db upgrade + if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then + flexmeasures add toy-account --name 'Docker Toy Account' + fi + # Worker processes rather than threads: a slow, CPU-bound request held its worker's GIL, + # and under several threads that starved the other requests into client-side timeouts. + # One worker per CEM below, since every home's CEM can be mid-request at once; + # scale the two together. + gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 5 --threads 1 --timeout 120 wsgi:application + # One worker per home simulated at once; add or remove them along with the CEMs. + worker: + <<: *worker + worker-2: + <<: *worker + worker-3: + <<: *worker + # One CEM per simulated home; add one by copying an entry and taking the next host port. + cem: + <<: *cem + ports: + - "8080:8080" + cem-2: + <<: *cem + ports: + - "8081:8080" + cem-3: + <<: *cem + ports: + - "8082:8080" + cem-4: + <<: *cem + ports: + - "8083:8080" + cem-5: + <<: *cem + ports: + - "8084:8080" diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index e1adf215..4b6749ce 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json import logging import math @@ -7,7 +8,7 @@ from collections import defaultdict from datetime import datetime, timedelta from logging import Logger -from typing import Dict, Optional +from typing import Awaitable, Callable, Dict, Optional, Protocol import pandas as pd import pydantic @@ -44,6 +45,23 @@ _LOGGER = logging.getLogger(__name__) + +class ConfiguredHandler(Protocol): + """What a site configurator gives back: the handler for a resource manager, and its measured-power sensors.""" + + @property + def handler(self) -> ControlTypeHandler: ... + + @property + def power_sensor_ids(self) -> dict[str, int]: ... + + @property + def asset_id(self) -> int: ... + + +#: Gives a resource manager a site and a handler of its own, from its details (see flexmeasures_client.s2.site). +SiteConfigurator = Callable[[ResourceManagerDetails], Awaitable[ConfiguredHandler]] + #: The control type each family of control-type messages belongs to, by the prefix of its message type. CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, @@ -87,6 +105,7 @@ def __init__( timers: dict[str, datetime] | None = None, datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, + site_configurator: SiteConfigurator | None = None, clock: Clock | None = None, accept_simulation_metadata: bool = False, **kwargs, @@ -94,6 +113,10 @@ def __init__( """ Customer Energy Manager (CEM) + :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; + called with its details, in the background, while its messages are answered + with TEMPORARY_ERROR until the handler is registered (see FRBCSiteConfigurator). + Without one, the handlers registered up front serve every resource manager. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -120,6 +143,8 @@ def __init__( self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() + self._site_configurator = site_configurator + self._configuring: asyncio.Task | None = None #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type @@ -173,6 +198,8 @@ def supports_control_type(self, control_type: ControlType): async def close(self): self._is_closed = True + if self._configuring is not None: + self._configuring.cancel() for control_type, handler in self._control_types_handlers.items(): self._logger.debug(f"Closing handler for {control_type}") @@ -242,15 +269,17 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): self._activate_implicitly(message) # try to handle the message with the control_type handle + # A control type without a handler yet, as while a resource manager's site is being configured, + # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. + handler = self._control_types_handlers.get(self._control_type) if ( self._control_type is not None and ( self._control_type not in [ControlType.NO_SELECTION, ControlType.NOT_CONTROLABLE] ) - and self._control_types_handlers[self._control_type].supports_message( - message - ) + and handler is not None + and handler.supports_message(message) ): response = await self._control_types_handlers[ self._control_type @@ -431,6 +460,10 @@ async def handle_handshake(self, message: Handshake): @register(ResourceManagerDetails) async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message + if self._site_configurator is not None and self._configuring is None: + # In the background, since a site takes a few requests to FlexMeasures to set up, + # and the resource manager waits for this message's reception status meanwhile. + self._configuring = asyncio.create_task(self._configure_site(message)) if ( not self._control_type @@ -444,6 +477,25 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) + async def _configure_site(self, details: ResourceManagerDetails) -> None: + """Give the resource manager its site, and register the handler wired to it.""" + assert self._site_configurator is not None + try: + site = await self._site_configurator(details) + except (ValueError, ConnectionError) as exc: + # Its messages go on being answered with TEMPORARY_ERROR, which the resource manager sees and retries. + self._logger.error( + "Could not configure a site for resource manager %r: %s", + details.name, + exc, + ) + return + self.power_sensor_id = {**(self.power_sensor_id or {}), **site.power_sensor_ids} + self.register_control_type(site.handler) + self._logger.info( + "Resource manager %r is served on site %s.", details.name, site.asset_id + ) + @register(PowerMeasurement) async def handle_power_measurement(self, message: PowerMeasurement): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 21cdd974..3b8636eb 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -19,6 +19,11 @@ ) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + ensure_site, +) log_level = os.getenv("LOGGING_LEVEL", "WARNING").upper() logging.basicConfig( @@ -41,28 +46,27 @@ "yes", ) -#: Each sensor the site needs: its unit, its resolution, and any attributes. -#: The units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. -SITE_SENSORS: dict[str, tuple[str, str, dict]] = { - "price": ("EUR/kWh", "PT15M", {}), - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. - "power": ("kW", "PT15M", {"consumption_is_positive": True}), - "state of charge": ("kWh", "PT0M", {}), - "RM discharge": ("dimensionless", "PT15M", {}), - "fill rate": ("kWh/s", "PT15M", {}), - "soc minima": ("kWh", "PT15M", {}), - "soc maxima": ("kWh", "PT15M", {}), - "usage forecast": ("kWh/s", "PT15M", {}), - "storage efficiency": ("%", "PT15M", {}), - "charging efficiency": ("dimensionless", "PT15M", {}), -} +#: Each sensor the site needs, which is the library's table of what an FRBC site records. +SITE_SENSORS = FRBC_SITE_SENSORS + +#: What every site asset is made with: a building, whose connection takes three phases of 25 A at 230 V. +SITE_ASSET_SPECS: dict[str, Any] = dict( + generic_asset_type_id=6, + flex_model={"power-capacity": f"{3 * 25 * 230} VA"}, +) + +#: Whether each resource manager gets a site of its own, named after it, as in a co-simulation of a community, +#: rather than all of them sharing the one site this server sets up at startup. +SITE_PER_RESOURCE_MANAGER = os.getenv("S2_SITE_PER_RESOURCE_MANAGER", "").lower() in ( + "1", + "true", + "yes", +) # Keys under which the configured site is kept on the aiohttp application. FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) SENSORS = web.AppKey("sensors", dict) +CONFIGURATOR = web.AppKey("configurator", FRBCSiteConfigurator) async def rm_details_watchdog(ws, cem: CEM): @@ -136,21 +140,40 @@ async def websocket_handler(request): # calls, which takes about ten seconds against a local server, while s2-python's # reception-status timeout is five: a resource manager gave up before the CEM had read its # first message. - await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + if SITE_PER_RESOURCE_MANAGER: + await serve_resource_manager( + ws, request.app[FM_CLIENT], configurator=request.app[CONFIGURATOR] + ) + else: + await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) return ws async def serve_resource_manager( - ws, fm_client: FlexMeasuresClient, sensors: dict[str, dict] + ws, + fm_client: FlexMeasuresClient, + sensors: dict[str, dict] | None = None, + configurator: FRBCSiteConfigurator | None = None, ) -> None: - """Act as the CEM for one resource manager connected over ws, until it disconnects.""" + """Act as the CEM for one resource manager connected over ws, until it disconnects. + + On the site whose sensors are given, or, with a configurator, on a site of the resource manager's own, + which is set up once its details arrive. + """ cem = CEM( - sensor_id=sensors["power"]["id"], fm_client=fm_client, logger=LOGGER, + site_configurator=configurator, # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) + if sensors is not None: + cem.register_control_type(fixed_site_handler(sensors)) + await serve(ws, cem) + + +def fixed_site_handler(sensors: dict[str, dict]) -> FRBCSimple: + """The handler for a resource manager on the one site this server sets up at startup.""" frbc = FRBCSimple( power_sensor_id=sensors["power"]["id"], price_sensor_id=sensors["price"]["id"], @@ -169,8 +192,11 @@ async def serve_resource_manager( ) # A newer plan revokes what is left of the one before, so the resource manager does not follow both. frbc.send_revocations = True - cem.register_control_type(frbc) + return frbc + +async def serve(ws, cem: CEM) -> None: + """Serve one resource manager with this CEM, for as long as its connection lasts.""" # The session lasts as long as the resource manager's messages do. # When they end, the CEM is closed, which stops what its handlers still have in flight, # and the producer and the watchdog stop with it: left running, they outlived the connection, @@ -192,87 +218,18 @@ async def serve_resource_manager( async def configure_site( site_name: str, fm_client: FlexMeasuresClient ) -> dict[str, dict]: - """Create or update the site asset and its sensors, and seed it with demo prices.""" - account = await fm_client.get_account() - if account is None: - raise ValueError( - "The demo user belongs to no organisation, so there is nowhere to create the site." - ) - assets = await fm_client.get_assets(parse_json_fields=True) - - site_asset = None - for asset in assets: - if asset["name"] == site_name: - site_asset = asset - break - - site_asset_specs: dict[str, Any] = dict( - latitude=0, - longitude=0, - generic_asset_type_id=6, # Building asset type - flex_model={ - "power-capacity": f"{3 * 25 * 230} VA", - }, - ) - - if not site_asset: - site_asset = await fm_client.add_asset( - name=site_name, account_id=account["id"], **site_asset_specs - ) - # Update site asset with the latest specs - await fm_client.update_asset( - site_asset["id"], site_asset_specs, parse_json_fields=True - ) - - existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} - sensors: dict[str, dict] = {} - for name, (unit, resolution, attributes) in SITE_SENSORS.items(): - sensors[name] = existing.get(name) or await fm_client.add_sensor( - name=name, - event_resolution=resolution, - unit=unit, - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - **({"attributes": attributes} if attributes else {}), - ) - - # The site's graphs page shows what the plan is about, not only its price and power: - # the buffer with the comfort band around it, what is drawn from it, and what the heat pump puts in. - await fm_client.update_asset( - site_asset["id"], - {"sensors_to_show": site_graphs(sensors)}, - parse_json_fields=True, + """Create or update the site asset, its sensors and its graphs, and seed it with demo prices.""" + _, sensors = await ensure_site( + fm_client, + site_name, + asset_specs=SITE_ASSET_SPECS, + sensors=SITE_SENSORS, + timezone=SITE_TIMEZONE, ) - await seed_prices(fm_client, sensors["price"]["id"]) return sensors -def site_graphs(sensors: dict[str, dict]) -> list[dict]: - """The site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. - - In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, - rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. - The server sets them on every start, so changes made to them in the UI do not outlive a restart. - """ - - def ids(*names: str) -> list[int]: - return [sensors[name]["id"] for name in names] - - return [ - {"title": "Price", "sensors": ids("price")}, - {"title": "Heat pump: planned and measured power", "sensors": ids("power")}, - { - "title": "Buffer: fill level and comfort band", - "sensors": ids("state of charge", "soc minima", "soc maxima"), - }, - { - "title": "Buffer: filled by the heat pump, drawn by hot-water use", - "sensors": ids("fill rate", "usage forecast"), - }, - ] - - async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: """Demo prices, so that a site has something to schedule against. @@ -299,7 +256,13 @@ async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> No async def keep_prices_seeded(app: web.Application): - """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them.""" + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them. + + Not where each resource manager has a site of its own, whose prices are whoever runs them to post. + """ + if SITE_PER_RESOURCE_MANAGER: + yield + return async def reseed(): while True: @@ -338,6 +301,20 @@ async def configure_site_once(app: web.Application) -> None: ssl=parsed.scheme == "https", ) app[FM_CLIENT] = fm_client + if SITE_PER_RESOURCE_MANAGER: + # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. + app[CONFIGURATOR] = FRBCSiteConfigurator( + fm_client, + asset_type_id=SITE_ASSET_SPECS["generic_asset_type_id"], + asset_specs={"flex_model": SITE_ASSET_SPECS["flex_model"]}, + timezone=SITE_TIMEZONE, + # The co-simulation's resource managers are heat pumps, which cannot feed power back: + # FlexMeasures then keeps their production at zero even where constraints are relaxed. + power_sensor_attributes={"is_strictly_non_positive": True}, + logger=LOGGER, + ) + LOGGER.info("Each resource manager gets a site of its own; ready for them.") + return app[SENSORS] = await configure_site(SITE_NAME, fm_client) LOGGER.info("Site '%s' is configured; ready for resource managers.", SITE_NAME) diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py new file mode 100644 index 00000000..0807c80f --- /dev/null +++ b/src/flexmeasures_client/s2/site.py @@ -0,0 +1,264 @@ +"""A FlexMeasures site for an S2 resource manager: its asset, its sensors, its graphs, and the handler wired to them. + +The example server keeps one site for whichever resource manager connects (see ensure_site). +A CEM serving many resource managers, as a co-simulation of a community does, gives each one a site of its own, +named after it, with FRBCSiteConfigurator. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import timedelta +from logging import Logger, getLogger +from typing import Any + +try: + from s2python.common import ResourceManagerDetails +except ImportError: + raise ImportError( + "The 's2-python' package is required for this functionality. " + "Install it using `pip install flexmeasures-client[s2]`." + ) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + +_LOGGER = getLogger(__name__) + + +@dataclass(frozen=True) +class SiteSensor: + """What a site's sensor is made with: its unit, its resolution, and any attributes.""" + + unit: str + resolution: str + attributes: dict = field(default_factory=dict) + + +#: The sensors an FRBC site needs, by name. +#: Their units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. +#: The names are a contract with whoever reads the site, a co-simulation among them, +#: which looks up "price", "production price", "power", "state of charge" and "measured-power" by name. +FRBC_SITE_SENSORS: dict[str, SiteSensor] = { + "price": SiteSensor("EUR/kWh", "PT15M"), + "production price": SiteSensor("EUR/kWh", "PT15M"), + # S2 states a power as positive when it is consumed, + # and this sensor holds both an actuator's power and the schedule planned for it. + # Left unsaid, FlexMeasures reads a power sensor the other way round, + # and flips a schedule's sign on the way in, which puts the two in opposite directions. + "power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + # What the resource manager measures its site drawing, kept apart from the schedule on "power". + "measured-power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + "state of charge": SiteSensor("kWh", "PT0M"), + "RM discharge": SiteSensor("dimensionless", "PT15M"), + "fill rate": SiteSensor("kWh/s", "PT15M"), + "soc minima": SiteSensor("kWh", "PT15M"), + "soc maxima": SiteSensor("kWh", "PT15M"), + "usage forecast": SiteSensor("kWh/s", "PT15M"), + "storage efficiency": SiteSensor("%", "PT15M"), + "charging efficiency": SiteSensor("dimensionless", "PT15M"), +} + + +def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: + """A site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + + In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, + rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. + A graph whose sensors the site does not have is left out. + """ + graphs = [ + ("Prices", ["price", "production price"]), + ("Power: planned and measured", ["power", "measured-power"]), + ( + "Buffer: fill level and comfort band", + ["state of charge", "soc minima", "soc maxima"], + ), + ( + "Buffer: filled by the actuator, drawn by usage", + ["fill rate", "usage forecast"], + ), + ] + shown = [] + for title, names in graphs: + ids = [sensors[name]["id"] for name in names if name in sensors] + if ids: + shown.append({"title": title, "sensors": ids}) + return shown + + +async def ensure_site( + fm_client: FlexMeasuresClient, + name: str, + *, + asset_specs: dict[str, Any], + sensors: dict[str, SiteSensor], + timezone: str, + known_asset_id: int | None = None, +) -> tuple[dict, dict[str, dict]]: + """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. + + It is safe to run again: what exists is kept, and what is missing is made. + + :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. + :returns: the asset, and its sensors by name. + """ + site_asset = None + if known_asset_id is not None: + try: + candidate = await fm_client.get_asset( + known_asset_id, parse_json_fields=True + ) + except (ValueError, ConnectionError): + candidate = None + if candidate is not None and candidate.get("name") == name: + site_asset = candidate + if site_asset is None: + for asset in await fm_client.get_assets(parse_json_fields=True): + if asset["name"] == name: + site_asset = asset + break + if site_asset is None: + account = await fm_client.get_account() + if account is None: + raise ValueError( + f"The user belongs to no organisation, so there is nowhere to create the site {name!r}." + ) + site_asset = await fm_client.add_asset( + name=name, account_id=account["id"], **asset_specs + ) + else: + await fm_client.update_asset( + site_asset["id"], asset_specs, parse_json_fields=True + ) + + existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} + made: dict[str, dict] = {} + for sensor_name, spec in sensors.items(): + made[sensor_name] = existing.get(sensor_name) or await fm_client.add_sensor( + name=sensor_name, + event_resolution=spec.resolution, + unit=spec.unit, + generic_asset_id=site_asset["id"], + timezone=timezone, + **({"attributes": spec.attributes} if spec.attributes else {}), + ) + + # Set on every run, so changes made to the graphs in the UI do not outlive a restart. + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": frbc_site_graphs(made)}, + parse_json_fields=True, + ) + return site_asset, made + + +@dataclass +class ConfiguredSite: + """What a CEM needs to serve one resource manager: its handler, and where its measured power goes.""" + + handler: FRBCSimple + #: The sensor each commodity quantity's measured power is recorded on, as the CEM's power_sensor_id takes it. + power_sensor_ids: dict[str, int] + asset_id: int + + +class FRBCSiteConfigurator: + """Gives each resource manager a site of its own, named after it, and an FRBC handler wired to its sensors. + + Made once per server and handed to each CEM it makes (see CEM's site_configurator), + so that a resource manager that reconnects finds its site by the id remembered here, + rather than by a listing of every asset on the server. + """ + + def __init__( + self, + fm_client: FlexMeasuresClient, + *, + asset_type_id: int, + timezone: str = "UTC", + asset_specs: dict[str, Any] | None = None, + sensors: dict[str, SiteSensor] | None = None, + power_sensor_attributes: dict[str, Any] | None = None, + handler_options: dict[str, Any] | None = None, + send_revocations: bool = True, + logger: Logger | None = None, + ) -> None: + """ + :param asset_type_id: the FlexMeasures asset type each site is made with. + :param timezone: the time zone of each site's sensors, and of its handler. + :param asset_specs: anything else each site asset is made or updated with, such as a flex model. + :param sensors: the sensors each site gets; FRBC_SITE_SENSORS by default. + :param power_sensor_attributes: attributes added to the "power" sensor's, which a device's nature decides, + such as that it cannot produce. + :param handler_options: keyword arguments for each FRBCSimple beyond its sensors. + By default it plans from now, over 24 hours, as a co-simulation steps through days. + :param send_revocations: whether a newer plan revokes what is left of the one before. + """ + self._fm_client = fm_client + self._timezone = timezone + self._asset_specs = { + "generic_asset_type_id": asset_type_id, + **(asset_specs or {}), + } + sensors = dict(sensors if sensors is not None else FRBC_SITE_SENSORS) + if power_sensor_attributes: + power = sensors["power"] + sensors["power"] = SiteSensor( + power.unit, + power.resolution, + {**power.attributes, **power_sensor_attributes}, + ) + self._sensors = sensors + self._handler_options = { + "valid_from_shift": timedelta(0), + "schedule_duration": timedelta(hours=24), + **(handler_options or {}), + } + self._send_revocations = send_revocations + self._asset_ids: dict[str, int] = {} + self._logger = logger or _LOGGER + + async def __call__(self, details: ResourceManagerDetails) -> ConfiguredSite: + name = details.name + if not name: + raise ValueError( + "A resource manager without a name cannot be given a site of its own, which is found by its name." + ) + asset, sensors = await ensure_site( + self._fm_client, + name, + asset_specs=self._asset_specs, + sensors=self._sensors, + timezone=self._timezone, + known_asset_id=self._asset_ids.get(name), + ) + self._asset_ids[name] = asset["id"] + self._logger.info("Resource manager %r has site %s.", name, asset["id"]) + + def sensor_id(sensor_name: str) -> int | None: + sensor = sensors.get(sensor_name) + return sensor["id"] if sensor else None + + handler = FRBCSimple( + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + production_price_sensor_id=sensor_id("production price"), + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + fill_rate_sensor_id=sensor_id("fill rate"), + soc_minima_sensor_id=sensor_id("soc minima"), + soc_maxima_sensor_id=sensor_id("soc maxima"), + usage_forecast_sensor_id=sensor_id("usage forecast"), + leakage_behaviour_sensor_id=sensor_id("storage efficiency"), + charging_efficiency_sensor_id=sensor_id("charging efficiency"), + timezone=self._timezone, + **self._handler_options, + ) + handler.send_revocations = self._send_revocations + measured = sensor_id("measured-power") + return ConfiguredSite( + handler=handler, + power_sensor_ids={"ELECTRIC.POWER.L1": measured} if measured else {}, + asset_id=asset["id"], + ) diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index e2eca6c4..3678e18b 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -534,9 +534,12 @@ def capture(**kwargs): def test_the_site_sensors_are_what_the_cem_records_in(): - assert server.SITE_SENSORS["state of charge"][:2] == ("kWh", "PT0M") - assert server.SITE_SENSORS["storage efficiency"][0] == "%" - assert server.SITE_SENSORS["usage forecast"][0] == "kWh/s" + assert ( + server.SITE_SENSORS["state of charge"].unit, + server.SITE_SENSORS["state of charge"].resolution, + ) == ("kWh", "PT0M") + assert server.SITE_SENSORS["storage efficiency"].unit == "%" + assert server.SITE_SENSORS["usage forecast"].unit == "kWh/s" @pytest.mark.asyncio @@ -752,12 +755,15 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} assert {"state of charge", "soc minima", "soc maxima"} in named.values() assert {"fill rate", "usage forecast"} in named.values() - assert {"power"} in named.values() and {"price"} in named.values() + assert {"power", "measured-power"} in named.values() + assert {"price", "production price"} in named.values() def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): """A title and a list of sensors per graph; the form with plots is refused before FlexMeasures v0.32.""" - graphs = server.site_graphs( + from flexmeasures_client.s2.site import frbc_site_graphs + + graphs = frbc_site_graphs( {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} ) for graph in graphs: @@ -823,3 +829,70 @@ def capture(**kwargs): await cem_end._inbox.put(_CLOSED) await asyncio.wait_for(serving, timeout=5) assert made[0]["accept_simulation_metadata"] is accept + + +@pytest.mark.asyncio +async def test_with_a_site_per_resource_manager_the_server_configures_none_at_startup(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "configure_site", AsyncMock()) as configure, + ): + await server.configure_site_once(app) + configure.assert_not_awaited() + assert isinstance(app[server.CONFIGURATOR], server.FRBCSiteConfigurator) + assert ( + app[server.CONFIGURATOR] + ._sensors["power"] + .attributes["is_strictly_non_positive"] + ) + app[server.SENSORS] = { + "price": {"id": 1} + } # so that seeding, were it to happen, would reach FlexMeasures + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)), + patch.object(server, "seed_prices", AsyncMock()) as seed, + ): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() # nothing to seed: the prices are the simulation's to post + await asyncio.sleep(0.05) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + seed.assert_not_awaited() + await app[server.FM_CLIENT].close() + + +@pytest.mark.asyncio +async def test_a_resource_manager_with_a_site_of_its_own_is_served_once_it_is_configured(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + async def configurator(details): + raise AssertionError("not called in this test") + + resource_manager, cem_end = _connected() + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), configurator=configurator + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]._site_configurator is configurator + assert ( + made[0]._control_types_handlers == {} + ) # no handler until its site is configured diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py new file mode 100644 index 00000000..54e81673 --- /dev/null +++ b/tests/s2/test_site.py @@ -0,0 +1,264 @@ +"""A site of its own for each resource manager, and a CEM that waits for it.""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from unittest.mock import AsyncMock + +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + SiteSensor, + ensure_site, + frbc_site_graphs, +) + + +def _flexmeasures(existing: list[dict] | None = None) -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock(return_value=existing or []) + fm.add_asset = AsyncMock(side_effect=lambda **kw: {"id": 50, "sensors": [], **kw}) + fm.made = {} + + def add_sensor(**kw): + sensor = {"id": 100 + len(fm.made), **kw} + fm.made[kw["name"]] = sensor + return sensor + + fm.add_sensor = AsyncMock(side_effect=add_sensor) + return fm + + +@pytest.fixture +def named_details(resource_manager_details): + return resource_manager_details.model_copy(update={"name": "home A"}) + + +@pytest.mark.asyncio +async def test_a_site_is_made_with_every_sensor_and_its_graphs(): + fm = _flexmeasures() + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 50 + assert set(sensors) == set(FRBC_SITE_SENSORS) + made = {c.kwargs["name"]: c.kwargs for c in fm.add_sensor.await_args_list} + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["state of charge"]["event_resolution"] == "PT0M" + graphs = fm.update_asset.await_args.args[1]["sensors_to_show"] + assert graphs == frbc_site_graphs(sensors) + + +@pytest.mark.asyncio +async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): + fm = _flexmeasures( + [ + {"id": 6, "name": "home B", "sensors": []}, + {"id": 7, "name": "home A", "sensors": [{"id": 70, "name": "price"}]}, + ] + ) + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 7 + assert sensors["price"]["id"] == 70 + assert "price" not in {c.kwargs["name"] for c in fm.add_sensor.await_args_list} + fm.add_asset.assert_not_awaited() + specs_update = fm.update_asset.await_args_list[0] + assert specs_update.args == (7, {"generic_asset_type_id": 6}) + + +@pytest.mark.asyncio +async def test_a_known_site_is_fetched_by_id_rather_than_found_in_a_listing(): + """A resource manager reconnects every round of a co-simulation; a listing of every asset each time is slow.""" + fm = _flexmeasures() + fm.get_asset = AsyncMock(return_value={"id": 7, "name": "home A", "sensors": []}) + await ensure_site( + fm, + "home A", + asset_specs={}, + sensors={"price": SiteSensor("EUR/kWh", "PT15M")}, + timezone="UTC", + known_asset_id=7, + ) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_known_id_that_now_names_another_asset_is_looked_up_again(): + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock( + return_value={"id": 7, "name": "someone else", "sensors": []} + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_site_needs_an_organisation_to_be_made_in(): + fm = _flexmeasures() + fm.get_account = AsyncMock(return_value=None) + with pytest.raises(ValueError, match="no organisation"): + await ensure_site(fm, "home A", asset_specs={}, sensors={}, timezone="UTC") + + +def test_a_graph_without_sensors_on_the_site_is_left_out(): + graphs = frbc_site_graphs({"price": {"id": 1}, "power": {"id": 2}}) + assert graphs == [ + {"title": "Prices", "sensors": [1]}, + {"title": "Power: planned and measured", "sensors": [2]}, + ] + + +@pytest.mark.asyncio +async def test_each_resource_manager_gets_a_site_named_after_it_and_a_handler_wired_to_it( + named_details, +): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator( + fm, asset_type_id=6, power_sensor_attributes={"is_strictly_non_positive": True} + ) + site = await configurator(named_details) + assert fm.add_asset.await_args.kwargs["name"] == "home A" + made = fm.made + assert made["power"]["attributes"] == { + "consumption_is_positive": True, + "is_strictly_non_positive": True, + } + handler = site.handler + assert handler._power_sensor_id == made["power"]["id"] + assert handler._price_sensor_id == made["price"]["id"] + assert handler._production_price_sensor_id == made["production price"]["id"] + assert handler._soc_sensor_id == made["state of charge"]["id"] + assert handler._valid_from_shift == timedelta(0) + assert handler._schedule_duration == timedelta(hours=24) + assert handler.send_revocations is True + assert site.power_sensor_ids == {"ELECTRIC.POWER.L1": made["measured-power"]["id"]} + + +@pytest.mark.asyncio +async def test_a_reconnecting_resource_manager_finds_its_site_by_id(named_details): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + await configurator(named_details) + fm.get_asset = AsyncMock( + return_value={"id": 50, "name": "home A", "sensors": list(fm.made.values())} + ) + fm.get_assets.reset_mock() + await configurator(named_details) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_resource_manager_without_a_name_gets_no_site(resource_manager_details): + configurator = FRBCSiteConfigurator(_flexmeasures(), asset_type_id=6) + with pytest.raises(ValueError, match="without a name"): + await configurator(resource_manager_details.model_copy(update={"name": None})) + + +# --- The CEM, while the site is being configured --- + + +async def _details_then_selection(cem, rm_handshake, resource_manager_details): + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + messages = [] + while not cem._sending_queue.empty(): + messages.append(await cem.get_message()) + return [m for m in messages if m["message_type"] == "SelectControlType"][0] + + +@pytest.mark.asyncio +async def test_messages_are_answered_with_a_temporary_error_until_the_site_is_configured( + rm_handshake, named_details, frbc_system_description +): + """The resource manager retries a temporary error, so it is served once its handler exists.""" + release = asyncio.Event() + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + + async def slow(details): + await release.wait() + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=slow) + selection = await _details_then_selection(cem, rm_handshake, named_details) + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], status=ReceptionStatusValues.OK + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + + await cem.handle_message(frbc_system_description) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + + release.set() + await cem._configuring + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + assert cem.power_sensor_id and "ELECTRIC.POWER.L1" in cem.power_sensor_id + await cem.handle_message(frbc_system_description) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_site_that_cannot_be_configured_leaves_the_resource_manager_retrying( + rm_handshake, resource_manager_details, caplog +): + async def failing(details): + raise ConnectionError("FlexMeasures is down") + + cem = CEM(fm_client=_flexmeasures(), site_configurator=failing) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + await cem._configuring + assert cem._control_types_handlers == {} + assert "Could not configure a site" in caplog.text + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_a_site_being_configured( + rm_handshake, resource_manager_details +): + async def forever(details): + await asyncio.sleep(3600) + + cem = CEM(fm_client=_flexmeasures(), site_configurator=forever) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + configuring = cem._configuring + await cem.close() + await asyncio.wait([configuring], timeout=1) + assert configuring.cancelled() + + +@pytest.mark.asyncio +async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): + """Deleted since, say: the listing finds it by name, or it is made again.""" + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock(side_effect=ValueError("404")) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 From bffe6c9df24dadb1f5ac886288ccb46806e9f5c9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 21:45:12 +0100 Subject: [PATCH 46/59] docs(changelog): sites per resource manager and the co-simulation stack Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 30f19ad3..8078ed8d 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -27,6 +27,9 @@ Added * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] * ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] +* ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] +* ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] Changed ------- From 65b359f5167d6fb443594f074937916e8303e067 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:02:38 +0100 Subject: [PATCH 47/59] fix(S2): what the review of the sites per resource manager found Against the co-simulation it is for: - A reconnect overwrote the apartment's flex model with the configurator's, which reset what the co-simulation's controller writes into it (the stock's value at the end), and replaced its graphs, dropping the charts the co-simulation adds. A flex model is now merged into the asset's, and graphs are set only where an asset has none (the demo's own fixed site resets them, as before). They carry #188's titles again, since the controller adds to the graph titled "Power". - #188 always relaxed the site capacity; the configured sites in the example server do so again, explicitly. - Of several assets with a resource manager's name, the newest is taken, as the co-simulation takes it. In the CEM: - A power measurement arriving before a resource manager's site was set up was recorded on sensor 357, a fallback from #115 that named whichever sensor of whichever site had that id, and answered OK, so it was never sent again. It is now answered with TEMPORARY_ERROR while the site is being set up, and without a sensor for it, it is not recorded at all. The fixed site's measured-power sensor is now wired too. - A setup that failed was not tried again, and only two exception types were logged; anything else surfaced as a task exception never retrieved. A failed setup is now logged, whatever failed, and tried again on the next message that needs it. In the co-simulation stack: - The overlay's workers serve the reporting queue too, as the base file's do. - Its README says what isolates the database: the overlay mounts ../flexmeasures and ../flexmeasures-client, so a folder of its own with checkouts of those names, not a FlexMeasures worktree on its own. Mailhog's ports are listed. - Comments break after punctuation. Still not here: recording a co-simulation's measured power, which arrives in bursts stamped at past quarter hours and which the CEM's binning on now() does not catch. That is the next package. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + examples/S2-co-simulation/README.md | 15 +- .../S2-co-simulation/docker-compose.cem.yml | 32 ++-- src/flexmeasures_client/s2/cem.py | 65 ++++--- .../s2/script/websockets_server.py | 10 ++ src/flexmeasures_client/s2/site.py | 51 +++--- tests/s2/test_example_end_to_end.py | 40 ++++- tests/s2/test_site.py | 158 ++++++++++++++++-- 8 files changed, 293 insertions(+), 79 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 8078ed8d..c8c5187d 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -56,6 +56,7 @@ Removed Fixed ----- +* A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] * A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index fa95fa8e..9e1b8e0f 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -22,9 +22,12 @@ A co-simulation reads the sites by sensor name: `price`, `production price`, `power` (the schedule), `measured-power` (what the home measured) and `state of charge`. -The stack uses FlexMeasures' own development database service, `dev-db`, whose -data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the -flexmeasures checkout. A compose project name of its own does not separate that, -so run the stack from a checkout of its own (a git worktree will do) to keep a -development database you care about out of it. Its host ports, 5000, 5433, 6380 -and 8080 to 8084, have to be free too. +The stack uses FlexMeasures' own development database service, `dev-db`, +whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` +inside the flexmeasures checkout it runs from. +A compose project name of its own does not separate that. +To keep a development database you care about out of it, +run it from a folder of its own holding checkouts (git worktrees will do) +named `flexmeasures` and `flexmeasures-client`: +the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. +Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index da7f58dd..79a5b364 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -1,24 +1,25 @@ # ------------------------------------------------------------------ -# A FlexMeasures stack with one S2 CEM per simulated home, for a co-simulation -# whose resource managers connect over websockets, such as the community -# simulations that steer several homes at once. +# A FlexMeasures stack with one S2 CEM per simulated home, +# for a co-simulation whose resource managers connect over websockets, +# such as the community simulations that steer several homes at once. # -# It is an overlay on FlexMeasures' own docker-compose.yml, and runs the CEMs -# from this checkout. With the flexmeasures and flexmeasures-client repos side -# by side, run it from the flexmeasures folder (which holds the Dockerfile): +# It is an overlay on FlexMeasures' own docker-compose.yml, +# and runs the CEMs from this checkout. +# With checkouts named flexmeasures and flexmeasures-client side by side, +# run it from the flexmeasures folder, which holds the Dockerfile: # # docker compose \ # -f docker-compose.yml \ # -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \ # up # -# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws -# and so on, one per home. Each gets a site of its own, named after it, and the -# CEM goes by the simulated time its messages carry (see README.md). +# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws and so on, one per home. +# Each gets a site of its own, named after it, +# and the CEM goes by the simulated time its messages carry (see README.md). # ------------------------------------------------------------------ -# Shared definition for every CEM. Each listens on port 8080 inside its own -# container; only the host port differs. +# Shared definition for every CEM. +# Each listens on port 8080 inside its own container; only the host port differs. x-cem: &cem build: context: . @@ -45,15 +46,15 @@ x-cem: &cem - | # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`): # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv, - # which ships without pip and cannot see what the system pip installs. + # which may come without pip, and cannot see what the system pip installs. python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade # s2-python as this repo locks it, and as the co-simulation's resource managers use it. python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py # Shared definition for every RQ worker: one per home simulated at once, -# like the CEMs and the server's gunicorn workers below, since every home's -# scheduling jobs otherwise queue behind one worker process. +# like the CEMs and the server's gunicorn workers below, +# since every home's scheduling jobs otherwise queue behind one worker process. x-worker: &worker # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with. build: @@ -105,7 +106,8 @@ x-worker: &worker print(f"ingestion-registration guard skipped: {exc}") PYGUARD # Named after the container, which is unique per worker. - flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting + # Every queue the base file's worker serves but ingestion, which is left to the server to do as it receives the data. + flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting\|reporting services: dev-db: diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 4b6749ce..679b3e6a 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -114,8 +114,8 @@ def __init__( Customer Energy Manager (CEM) :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; - called with its details, in the background, while its messages are answered - with TEMPORARY_ERROR until the handler is registered (see FRBCSiteConfigurator). + called with its details, in the background (see FRBCSiteConfigurator). + Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. @@ -272,6 +272,8 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): # A control type without a handler yet, as while a resource manager's site is being configured, # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. handler = self._control_types_handlers.get(self._control_type) + if handler is None and self._control_type is not None: + self._ensure_configuring() if ( self._control_type is not None and ( @@ -460,10 +462,7 @@ async def handle_handshake(self, message: Handshake): @register(ResourceManagerDetails) async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message - if self._site_configurator is not None and self._configuring is None: - # In the background, since a site takes a few requests to FlexMeasures to set up, - # and the resource manager waits for this message's reception status meanwhile. - self._configuring = asyncio.create_task(self._configure_site(message)) + self._ensure_configuring() if ( not self._control_type @@ -477,19 +476,39 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) + def _ensure_configuring(self) -> None: + """Set the resource manager's site up in the background, unless that is done or under way. + + In the background, since a site takes a few requests to FlexMeasures to set up, + and the resource manager waits for a reception status meanwhile. + A setup that failed is tried again on the next message that needs it, + while the resource manager's messages are answered with TEMPORARY_ERROR, which it retries. + """ + if ( + self._site_configurator is None + or self._control_types_handlers + or (self._configuring is not None and not self._configuring.done()) + or not hasattr(self, "_resource_manager_details") + ): + return + self._configuring = asyncio.create_task( + self._configure_site(self._resource_manager_details) + ) + self._configuring.add_done_callback(self._report_configuring) + + def _report_configuring(self, task: asyncio.Task) -> None: + if task.cancelled() or task.exception() is None: + return + self._logger.error( + "Could not set a site up for resource manager %r; trying again on its next message.", + self._resource_manager_details.name, + exc_info=task.exception(), + ) + async def _configure_site(self, details: ResourceManagerDetails) -> None: """Give the resource manager its site, and register the handler wired to it.""" assert self._site_configurator is not None - try: - site = await self._site_configurator(details) - except (ValueError, ConnectionError) as exc: - # Its messages go on being answered with TEMPORARY_ERROR, which the resource manager sees and retries. - self._logger.error( - "Could not configure a site for resource manager %r: %s", - details.name, - exc, - ) - return + site = await self._site_configurator(details) self.power_sensor_id = {**(self.power_sensor_id or {}), **site.power_sensor_ids} self.register_control_type(site.handler) self._logger.info( @@ -499,15 +518,17 @@ async def _configure_site(self, details: ResourceManagerDetails) -> None: @register(PowerMeasurement) async def handle_power_measurement(self, message: PowerMeasurement): + if self._site_configurator is not None and not self._control_types_handlers: + # Its site is still being set up, so there is nowhere to record it yet: the resource manager is asked to resend it. + self._ensure_configuring() + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + for power_measurement in message.values: commodity_quantity = power_measurement.commodity_quantity.value - if ( - self.power_sensor_id is None - and commodity_quantity == "ELECTRIC.POWER.L1" - ): - sensor_id = 357 - elif self.power_sensor_id: + # Without a sensor for it, a measurement is not recorded. + # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. + if self.power_sensor_id: s_id = self.power_sensor_id.get(commodity_quantity) if s_id is None: # TODO: create a new sensor or return ReceptionStatus diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 3b8636eb..0a7deeed 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -164,6 +164,12 @@ async def serve_resource_manager( fm_client=fm_client, logger=LOGGER, site_configurator=configurator, + # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. + power_sensor_id=( + {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} + if sensors is not None and "measured-power" in sensors + else None + ), # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) @@ -225,6 +231,8 @@ async def configure_site( asset_specs=SITE_ASSET_SPECS, sensors=SITE_SENSORS, timezone=SITE_TIMEZONE, + # The demo's own graphs, on every start, so changes made to them in the UI do not outlive a restart. + reset_graphs=True, ) await seed_prices(fm_client, sensors["price"]["id"]) return sensors @@ -311,6 +319,8 @@ async def configure_site_once(app: web.Application) -> None: # The co-simulation's resource managers are heat pumps, which cannot feed power back: # FlexMeasures then keeps their production at zero even where constraints are relaxed. power_sensor_attributes={"is_strictly_non_positive": True}, + # A community's capacity is a target set by its members' steering, breached at a price, as #188 asked for. + handler_options={"relax_site_capacity": True}, logger=LOGGER, ) LOGGER.info("Each resource manager gets a site of its own; ready for them.") diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py index 0807c80f..f94185f0 100644 --- a/src/flexmeasures_client/s2/site.py +++ b/src/flexmeasures_client/s2/site.py @@ -67,17 +67,12 @@ def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. A graph whose sensors the site does not have is left out. """ + # Titled as #188 titled them, since a co-simulation's controller adds to the graph titled "Power". graphs = [ ("Prices", ["price", "production price"]), - ("Power: planned and measured", ["power", "measured-power"]), - ( - "Buffer: fill level and comfort band", - ["state of charge", "soc minima", "soc maxima"], - ), - ( - "Buffer: filled by the actuator, drawn by usage", - ["fill rate", "usage forecast"], - ), + ("Power", ["power", "measured-power"]), + ("State of charge", ["state of charge", "soc minima", "soc maxima"]), + ("Fill rate and usage", ["fill rate", "usage forecast"]), ] shown = [] for title, names in graphs: @@ -95,12 +90,17 @@ async def ensure_site( sensors: dict[str, SiteSensor], timezone: str, known_asset_id: int | None = None, + reset_graphs: bool = False, ) -> tuple[dict, dict[str, dict]]: """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. It is safe to run again: what exists is kept, and what is missing is made. + A flex model in the specs is merged into the one the asset has, since others write to it too, + such as a co-simulation's controller setting what its stock is worth at the end. + Graphs are set where the asset has none, since others add to them too, unless asked to reset them. :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. + :param reset_graphs: whether to set the graphs even where the asset has some. :returns: the asset, and its sensors by name. """ site_asset = None @@ -114,10 +114,14 @@ async def ensure_site( if candidate is not None and candidate.get("name") == name: site_asset = candidate if site_asset is None: - for asset in await fm_client.get_assets(parse_json_fields=True): - if asset["name"] == name: - site_asset = asset - break + # The newest of any of that name, as a co-simulation picks it too. + named = [ + asset + for asset in await fm_client.get_assets(parse_json_fields=True) + if asset["name"] == name + ] + site_asset = max(named, key=lambda asset: asset["id"]) if named else None + created = site_asset is None if site_asset is None: account = await fm_client.get_account() if account is None: @@ -128,9 +132,12 @@ async def ensure_site( name=name, account_id=account["id"], **asset_specs ) else: - await fm_client.update_asset( - site_asset["id"], asset_specs, parse_json_fields=True - ) + specs = dict(asset_specs) + if isinstance(specs.get("flex_model"), dict) and isinstance( + site_asset.get("flex_model"), dict + ): + specs["flex_model"] = {**site_asset["flex_model"], **specs["flex_model"]} + await fm_client.update_asset(site_asset["id"], specs, parse_json_fields=True) existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} made: dict[str, dict] = {} @@ -144,12 +151,12 @@ async def ensure_site( **({"attributes": spec.attributes} if spec.attributes else {}), ) - # Set on every run, so changes made to the graphs in the UI do not outlive a restart. - await fm_client.update_asset( - site_asset["id"], - {"sensors_to_show": frbc_site_graphs(made)}, - parse_json_fields=True, - ) + if created or reset_graphs or not site_asset.get("sensors_to_show"): + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": frbc_site_graphs(made)}, + parse_json_fields=True, + ) return site_asset, made diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 3678e18b..8874af96 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -735,7 +735,14 @@ async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_ fm = AsyncMock(FlexMeasuresClient) fm.get_account = AsyncMock(return_value={"id": 1}) fm.get_assets = AsyncMock( - return_value=[{"id": 5, "name": server.SITE_NAME, "sensors": []}] + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [], + "sensors_to_show": [{"title": "Edited in the UI", "sensors": [1]}], + } + ] ) fm.add_sensor = AsyncMock( side_effect=lambda **kwargs: { @@ -850,9 +857,9 @@ async def test_with_a_site_per_resource_manager_the_server_configures_none_at_st ._sensors["power"] .attributes["is_strictly_non_positive"] ) - app[server.SENSORS] = { - "price": {"id": 1} - } # so that seeding, were it to happen, would reach FlexMeasures + assert app[server.CONFIGURATOR]._handler_options["relax_site_capacity"] is True + # So that seeding, were it to happen, would reach FlexMeasures. + app[server.SENSORS] = {"price": {"id": 1}} with ( patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)), @@ -896,3 +903,28 @@ async def configurator(details): assert ( made[0]._control_types_handlers == {} ) # no handler until its site is configured + + +@pytest.mark.asyncio +async def test_on_the_fixed_site_measured_power_is_recorded_on_its_sensor(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["power_sensor_id"] == { + "ELECTRIC.POWER.L1": sensors["measured-power"]["id"] + } diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py index 54e81673..c67dbad9 100644 --- a/tests/s2/test_site.py +++ b/tests/s2/test_site.py @@ -64,7 +64,7 @@ async def test_a_site_is_made_with_every_sensor_and_its_graphs(): async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): fm = _flexmeasures( [ - {"id": 6, "name": "home B", "sensors": []}, + {"id": 8, "name": "home B", "sensors": []}, {"id": 7, "name": "home A", "sensors": [{"id": 70, "name": "price"}]}, ] ) @@ -123,7 +123,7 @@ def test_a_graph_without_sensors_on_the_site_is_left_out(): graphs = frbc_site_graphs({"price": {"id": 1}, "power": {"id": 2}}) assert graphs == [ {"title": "Prices", "sensors": [1]}, - {"title": "Power: planned and measured", "sensors": [2]}, + {"title": "Power", "sensors": [2]}, ] @@ -223,18 +223,95 @@ async def slow(details): @pytest.mark.asyncio -async def test_a_site_that_cannot_be_configured_leaves_the_resource_manager_retrying( - rm_handshake, resource_manager_details, caplog +async def test_a_site_that_could_not_be_set_up_is_tried_again_on_the_next_message( + rm_handshake, named_details, frbc_system_description, caplog ): - async def failing(details): - raise ConnectionError("FlexMeasures is down") + """A transient failure, such as a server error under load, would otherwise leave the resource manager retrying forever.""" + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + attempts = [] - cem = CEM(fm_client=_flexmeasures(), site_configurator=failing) + async def flaky(details): + attempts.append(details.name) + if len(attempts) == 1: + raise KeyError("a sensor the table lacks") + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=flaky) await cem.handle_message(rm_handshake) - await cem.handle_message(resource_manager_details) - await cem._configuring + await cem.handle_message(named_details) + await asyncio.wait([cem._configuring]) + assert "trying again on its next message" in caplog.text assert cem._control_types_handlers == {} - assert "Could not configure a site" in caplog.text + + cem._control_type = ControlType.FILL_RATE_BASED_CONTROL + await cem.handle_message(frbc_system_description) + await asyncio.wait([cem._configuring]) + assert attempts == ["home A", "home A"] + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + + +@pytest.mark.asyncio +async def test_a_measurement_before_the_site_is_set_up_is_asked_for_again( + rm_handshake, named_details +): + """It used to be recorded on sensor 357, whichever site that belonged to, and answered OK, so it was never resent.""" + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + release = asyncio.Event() + + async def slow(details): + await release.wait() + + fm = _flexmeasures() + cem = CEM(fm_client=fm, site_configurator=slow) + await cem.handle_message(rm_handshake) + await cem.handle_message(named_details) + while not cem._sending_queue.empty(): + await cem.get_message() + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + fm.post_sensor_data.assert_not_awaited() + release.set() + + +@pytest.mark.asyncio +async def test_without_a_sensor_for_it_a_measurement_is_not_recorded(): + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + fm = _flexmeasures() + cem = CEM(fm_client=fm) + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + fm.post_sensor_data.assert_not_awaited() @pytest.mark.asyncio @@ -262,3 +339,64 @@ async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 ) assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_reconnect_keeps_what_others_wrote_to_the_flex_model_and_the_graphs(): + """A co-simulation's controller writes its end-of-day stock value into the flex model, and adds to the graphs.""" + fm = _flexmeasures( + [ + { + "id": 7, + "name": "home A", + "sensors": [], + "flex_model": { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "1 kVA", + }, + "sensors_to_show": [{"title": "Gross vs net", "sensors": [1]}], + } + ] + ) + await ensure_site( + fm, + "home A", + asset_specs={"flex_model": {"power-capacity": "17250 VA"}}, + sensors={}, + timezone="UTC", + ) + specs = fm.update_asset.await_args_list[0].args[1] + assert specs["flex_model"] == { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "17250 VA", + } + assert all( + "sensors_to_show" not in call.args[1] + for call in fm.update_asset.await_args_list + ) + + +@pytest.mark.asyncio +async def test_the_demo_resets_its_graphs_where_asked(): + fm = _flexmeasures( + [{"id": 7, "name": "home A", "sensors": [], "sensors_to_show": [1]}] + ) + await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", reset_graphs=True + ) + assert "sensors_to_show" in fm.update_asset.await_args.args[1] + + +@pytest.mark.asyncio +async def test_of_several_sites_with_one_name_the_newest_is_taken(): + fm = _flexmeasures( + [ + {"id": 9, "name": "home A", "sensors": []}, + {"id": 12, "name": "home A", "sensors": []}, + {"id": 3, "name": "home A", "sensors": []}, + ] + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC" + ) + assert asset["id"] == 12 From ccab11bfe3275275e05e6cbc286ba0cf092dd608 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:14:40 +0100 Subject: [PATCH 48/59] feat(S2): record measured power by when it was measured, in watts, once per step Carved from #188, rewritten for the library rather than transcribed. The CEM averaged a resource manager's measured power into five-minute bins ending at its own now, posted one at most every five minutes, and dropped whatever fell outside that bin. A co-simulation's resource manager sends a window's measurements together once the window has passed, each stamped at the start of its quarter hour, so none of them ever fell in the bin, and nothing was recorded; the buffer kept them all, in a class attribute that every CEM in the process shared. The values, which S2 states in watts, also went out labelled kW, a thousand times too large. #188 bypassed the binning and posted each value at its own timestamp, deduplicated in module-level globals. flexmeasures_client.s2.measurements.MeasurementRecorder takes each measurement into the step of the sensor's resolution its timestamp falls in, and records a step once it is over by the CEM's clock, as the average of what was measured in it, in watts, with the step's end as the moment it became known. Consecutive steps go in one request, so a co-simulation's window of quarter hours costs one. A step that could not be posted is kept and tried again. A server makes one recorder and hands it to every CEM, so a resource manager that reconnects and sends its series again, as a co-simulation's does every round, is not recorded twice: the first value for a step stands, for a week. The CEM hands measurements to its recorder, its own by default, and asks the resource manager to resend one it could not take at all, such as where the sensor's resolution could not be read. get_commodity_unit, which labelled power kW, goes; nothing else used it. Not carried from #188: a barrier awaiting the posts before planning, since the co-simulation's resource manager sends its storage status before its measurements, so the barrier never covered the step being planned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/cem.py | 86 ++------ src/flexmeasures_client/s2/measurements.py | 132 ++++++++++++ .../s2/script/websockets_server.py | 18 +- tests/s2/test_example_end_to_end.py | 37 ++++ tests/s2/test_measurements.py | 195 ++++++++++++++++++ 5 files changed, 402 insertions(+), 66 deletions(-) create mode 100644 src/flexmeasures_client/s2/measurements.py create mode 100644 tests/s2/test_measurements.py diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 679b3e6a..c079bd40 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -5,7 +5,6 @@ import logging import math from asyncio import Queue -from collections import defaultdict from datetime import datetime, timedelta from logging import Logger from typing import Awaitable, Callable, Dict, Optional, Protocol @@ -33,9 +32,11 @@ from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError from flexmeasures_client.s2 import Handler, register from flexmeasures_client.s2.clock import Clock, SimulatedClock from flexmeasures_client.s2.control_types import ControlTypeHandler +from flexmeasures_client.s2.measurements import MeasurementRecorder from flexmeasures_client.s2.utils import ( get_latest_compatible_version, get_reception_status, @@ -93,10 +94,6 @@ class CEM(Handler): _datastore: dict _minimum_measurement_period: pd.Timedelta = pd.Timedelta(minutes=5) - _power_buffer: defaultdict = defaultdict( - list - ) # {commodity_quantity: [(timestamp, value), ...]} - def __init__( self, fm_client: FlexMeasuresClient, @@ -106,6 +103,7 @@ def __init__( datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, site_configurator: SiteConfigurator | None = None, + measurement_recorder: MeasurementRecorder | None = None, clock: Clock | None = None, accept_simulation_metadata: bool = False, **kwargs, @@ -117,6 +115,9 @@ def __init__( called with its details, in the background (see FRBCSiteConfigurator). Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. + :param measurement_recorder: where the resource manager's measured power is recorded; + one of its own by default. A server that makes a CEM per connection + hands each the same one, so a resource manager that reconnects is not recorded twice. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -144,6 +145,9 @@ def __init__( self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() self._site_configurator = site_configurator + self._measurement_recorder = measurement_recorder or MeasurementRecorder( + fm_client, logger=logger + ) self._configuring: asyncio.Task | None = None #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. self._selecting: tuple[ControlType, str] | None = None @@ -543,60 +547,24 @@ async def handle_power_measurement(self, message: PowerMeasurement): ) continue - # Store the value in the buffer - self._power_buffer[commodity_quantity].append( - (message.measurement_timestamp, power_measurement.value) - ) - - # Compute bin - now = self.now() - m = self._minimum_measurement_period // pd.Timedelta(minutes=1) - bin_end = now.replace( - second=0, microsecond=0, minute=(now.minute // m) * m - ) # e.g. 10:15:00 - bin_start = bin_end - self._minimum_measurement_period - - # If timer not due, just collect values - if not self._is_timer_due(f"power_measurement_{commodity_quantity}"): - self._logger.debug( - f"Collecting 5-minute average for {commodity_quantity} ({bin_start.isoformat()} – {bin_end.isoformat()})" - ) - continue - - # Compute average of all buffered values in last 5 minutes - buffer = self._power_buffer[commodity_quantity] - period_values = [v for (t, v) in buffer if bin_start <= t < bin_end] - - if not period_values: - self._logger.debug( - f"No samples found for {commodity_quantity} in {bin_start}–{bin_end}, skipping." - ) - continue - - avg_value = sum(period_values) / len(period_values) - self._logger.debug( - f"Posting 5-minute average for {commodity_quantity}: " - f"{avg_value} ({bin_start.isoformat()} – {bin_end.isoformat()})" - ) - - # Send measurement try: - await self._fm_client.post_sensor_data( + await self._measurement_recorder.add( sensor_id, - start=bin_start.isoformat(), - duration=self._minimum_measurement_period.isoformat(), # TODO: not specified in S2 Protocol - values=[avg_value], - unit=get_commodity_unit(commodity_quantity), + message.measurement_timestamp, + power_measurement.value, + now=self.now(), ) - except Exception as e: # noqa: B902 - intentional safety net + except (ValueError, ConnectionError, ContentTypeError) as exc: + # Not taken at all, as when the sensor's resolution could not be read: the resource manager is asked to resend it. self._logger.warning( - f"POSTing power measurement failed with error: {e}" + "Could not take measured power for %s at %s: %s", + commodity_quantity, + message.measurement_timestamp, + exc, + ) + return get_reception_status( + message, ReceptionStatusValues.TEMPORARY_ERROR ) - - # Keep only samples newer than this bin (for next period) - self._power_buffer[commodity_quantity] = [ - (t, v) for (t, v) in buffer if t >= bin_end - ] return get_reception_status(message) @@ -625,13 +593,3 @@ def handle_revoke_object(self, message: RevokeObject): async def send_message(self, message): self._logger.debug(f"Sent: {message}") await self._sending_queue.put(message) - - -def get_commodity_unit(commodity_quantity) -> str: - if "POWER" in commodity_quantity: - return "kW" - if "FLOW_RATE" in commodity_quantity: - return "m³/h" - if "TEMPERATURE" in commodity_quantity: - return "°C" - return "" diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py new file mode 100644 index 00000000..492cb0fc --- /dev/null +++ b/src/flexmeasures_client/s2/measurements.py @@ -0,0 +1,132 @@ +"""Recording what a resource manager measures on FlexMeasures sensors, one value per step of each sensor's resolution.""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime, timedelta +from logging import Logger, getLogger + +import pandas as pd + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError, JobFailedError + +_LOGGER = getLogger(__name__) + +#: The unit S2 states every power in, measured ones included. +S2_POWER_UNIT = "W" + +#: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. +REMEMBERED_FOR = timedelta(days=7) + + +class MeasurementRecorder: + """Records measured power on a sensor as the average of what was measured in each step of its resolution. + + A step is recorded once it is over by the CEM's clock, with the moment it ended as the moment it became known, + so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. + Steps are taken by the time each measurement is stamped with, not by when it arrives, + since a co-simulation's resource manager sends a window's measurements together once the window has passed, + each stamped at the start of its quarter hour, while a live one sends them as they are taken. + Consecutive steps that are over are recorded in one request. + + Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, + which a co-simulation makes every round, and so remembers what was recorded: + a step sent again is not recorded again, and the first value recorded for a step stands. + """ + + def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): + self._fm_client = fm_client + self._logger = logger or _LOGGER + self._resolutions: dict[int, timedelta] = {} + self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) + self._recorded: dict[int, set[datetime]] = defaultdict(set) + + async def add( + self, sensor_id: int, measured_at: datetime, watts: float, now: datetime + ) -> None: + """Take one measurement, and record every step of this sensor that is over by now.""" + resolution = await self._resolution(sensor_id) + step = self._step_of(measured_at, resolution) + if step in self._recorded[sensor_id]: + self._logger.debug( + "Not recording %s W on sensor %s for %s again.", watts, sensor_id, step + ) + else: + self._pending[sensor_id].setdefault(step, []).append(watts) + await self.record(sensor_id, now) + + async def record(self, sensor_id: int, now: datetime) -> None: + """Record every pending step of this sensor that is over by now, consecutive ones in one request each.""" + resolution = await self._resolution(sensor_id) + over = sorted( + step for step in self._pending[sensor_id] if step + resolution <= now + ) + for run in _consecutive(over, resolution): + values = [ + sum(samples) / len(samples) + for samples in (self._pending[sensor_id][step] for step in run) + ] + ended = run[-1] + resolution + try: + await self._fm_client.post_sensor_data( + sensor_id, + start=run[0], + duration=resolution * len(run) if resolution else timedelta(0), + values=values, + unit=S2_POWER_UNIT, + prior=ended, + ) + except ( + ValueError, + ConnectionError, + ContentTypeError, + JobFailedError, + ) as exc: + # Kept, and tried again with the next measurement. + self._logger.warning( + "Could not record measured power on sensor %s from %s to %s; trying again later: %s", + sensor_id, + run[0], + ended, + exc, + ) + continue + for step in run: + del self._pending[sensor_id][step] + self._recorded[sensor_id].add(step) + self._recorded[sensor_id] = { + step for step in self._recorded[sensor_id] if step >= now - REMEMBERED_FOR + } + + async def _resolution(self, sensor_id: int) -> timedelta: + """The sensor's resolution, asked once, since it is a property of how the sensor was set up.""" + if sensor_id not in self._resolutions: + sensor = await self._fm_client.get_sensor(sensor_id) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + if resolution is None: + raise ValueError( + f"Sensor {sensor_id} states no resolution, so its steps are unknown." + ) + self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() + return self._resolutions[sensor_id] + + @staticmethod + def _step_of(moment: datetime, resolution: timedelta) -> datetime: + """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it.""" + if not resolution: + return moment + return pd.Timestamp(moment).floor(resolution).to_pydatetime() + + +def _consecutive(steps: list[datetime], resolution: timedelta) -> list[list[datetime]]: + """Sorted steps, in runs of steps that follow one another; with no resolution, each on its own.""" + runs: list[list[datetime]] = [] + for step in steps: + if runs and resolution and step == runs[-1][-1] + resolution: + runs[-1].append(step) + else: + runs.append([step]) + return runs diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 0a7deeed..591540bd 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -19,6 +19,7 @@ ) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.measurements import MeasurementRecorder from flexmeasures_client.s2.site import ( FRBC_SITE_SENSORS, FRBCSiteConfigurator, @@ -67,6 +68,7 @@ FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) SENSORS = web.AppKey("sensors", dict) CONFIGURATOR = web.AppKey("configurator", FRBCSiteConfigurator) +RECORDER = web.AppKey("recorder", MeasurementRecorder) async def rm_details_watchdog(ws, cem: CEM): @@ -142,10 +144,18 @@ async def websocket_handler(request): # first message. if SITE_PER_RESOURCE_MANAGER: await serve_resource_manager( - ws, request.app[FM_CLIENT], configurator=request.app[CONFIGURATOR] + ws, + request.app[FM_CLIENT], + configurator=request.app[CONFIGURATOR], + recorder=request.app[RECORDER], ) else: - await serve_resource_manager(ws, request.app[FM_CLIENT], request.app[SENSORS]) + await serve_resource_manager( + ws, + request.app[FM_CLIENT], + request.app[SENSORS], + recorder=request.app[RECORDER], + ) return ws @@ -154,6 +164,7 @@ async def serve_resource_manager( fm_client: FlexMeasuresClient, sensors: dict[str, dict] | None = None, configurator: FRBCSiteConfigurator | None = None, + recorder: MeasurementRecorder | None = None, ) -> None: """Act as the CEM for one resource manager connected over ws, until it disconnects. @@ -164,6 +175,8 @@ async def serve_resource_manager( fm_client=fm_client, logger=LOGGER, site_configurator=configurator, + # The server's, which outlives a connection, so a resource manager that reconnects is not recorded twice. + measurement_recorder=recorder, # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. power_sensor_id=( {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} @@ -309,6 +322,7 @@ async def configure_site_once(app: web.Application) -> None: ssl=parsed.scheme == "https", ) app[FM_CLIENT] = fm_client + app[RECORDER] = MeasurementRecorder(fm_client, logger=LOGGER) if SITE_PER_RESOURCE_MANAGER: # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. app[CONFIGURATOR] = FRBCSiteConfigurator( diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 8874af96..8a7ed3b9 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -928,3 +928,40 @@ def capture(**kwargs): assert made[0]["power_sensor_id"] == { "ELECTRIC.POWER.L1": sensors["measured-power"]["id"] } + + +@pytest.mark.asyncio +async def test_every_cem_the_server_makes_records_with_the_servers_recorder(): + """It outlives a connection, so what a reconnecting resource manager sends again is not recorded twice.""" + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + with patch.object(server, "SITE_PER_RESOURCE_MANAGER", True): + await server.configure_site_once(app) + assert isinstance(app[server.RECORDER], MeasurementRecorder) + await app[server.FM_CLIENT].close() + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + recorder = MeasurementRecorder(_flexmeasures()) + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), sensors, recorder=recorder + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["measurement_recorder"] is recorder diff --git a/tests/s2/test_measurements.py b/tests/s2/test_measurements.py new file mode 100644 index 00000000..697bd7bc --- /dev/null +++ b/tests/s2/test_measurements.py @@ -0,0 +1,195 @@ +"""Recording a resource manager's measured power, live and in a co-simulation.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.measurements import MeasurementRecorder + +T = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) +Q = timedelta(minutes=15) + + +def _recorder(resolution: str = "PT15M") -> tuple[MeasurementRecorder, AsyncMock]: + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": resolution}) + return MeasurementRecorder(fm), fm + + +def _posts(fm) -> list[dict]: + return [ + c.kwargs | {"sensor_id": c.args[0]} for c in fm.post_sensor_data.await_args_list + ] + + +@pytest.mark.asyncio +async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_request(): + """Its resource manager sends a window's measurements once the window has passed, each stamped at its quarter hour's start. + + Binned by the moment they arrived, none of them fell in the step the CEM's clock was in, and none was recorded. + """ + recorder, fm = _recorder() + window_end = T + 4 * Q + for i, watts in enumerate([1000, 2000, 0, 500]): + await recorder.add(7, T + i * Q, watts, now=window_end) + (post,) = _posts(fm)[-1:] + recorded = {} + for p in _posts(fm): + for k, v in enumerate(p["values"]): + recorded[p["start"] + k * Q] = v + assert recorded == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} + assert ( + post["unit"] == "W" + ) # as S2 states power; it used to go out labelled kW, a thousand times too large + assert ( + post["prior"] == post["start"] + len(post["values"]) * Q + ) # known at the end of what it measures + + +@pytest.mark.asyncio +async def test_a_live_resource_managers_samples_are_averaged_per_step_once_it_is_over(): + recorder, fm = _recorder() + await recorder.add(7, T + timedelta(minutes=1), 1000, now=T + timedelta(minutes=1)) + await recorder.add(7, T + timedelta(minutes=8), 2000, now=T + timedelta(minutes=8)) + fm.post_sensor_data.assert_not_awaited() # the step is not over + await recorder.add( + 7, T + Q + timedelta(minutes=1), 800, now=T + Q + timedelta(minutes=1) + ) + (post,) = _posts(fm) + assert (post["start"], post["values"], post["duration"]) == (T, [1500], Q) + + +@pytest.mark.asyncio +async def test_a_step_sent_again_is_not_recorded_again(): + """A co-simulation's resource manager reconnects every round and sends its series again.""" + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add( + 7, T, 999, now=T + Q + ) # the first value recorded for a step stands + assert len(_posts(fm)) == 1 + + +@pytest.mark.asyncio +async def test_a_step_that_could_not_be_recorded_is_tried_again(caplog): + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None] + await recorder.add(7, T, 1000, now=T + Q) + assert "trying again later" in caplog.text + await recorder.add(7, T + Q, 2000, now=T + 2 * Q) + retried = _posts(fm)[-1] + assert (retried["start"], retried["values"]) == (T, [1000, 2000]) + + +@pytest.mark.asyncio +async def test_steps_that_do_not_follow_one_another_go_in_requests_of_their_own(): + recorder, fm = _recorder() + await recorder.add(7, T, 1, now=T) + await recorder.add(7, T + 2 * Q, 3, now=T) + await recorder.record(7, now=T + 3 * Q) + assert [(p["start"], p["values"]) for p in _posts(fm)] == [ + (T, [1]), + (T + 2 * Q, [3]), + ] + + +@pytest.mark.asyncio +async def test_each_sensor_is_recorded_at_its_own_resolution(): + recorder, fm = _recorder("PT1H") + await recorder.add(7, T + timedelta(minutes=20), 10, now=T + timedelta(minutes=20)) + await recorder.add(7, T + timedelta(minutes=40), 30, now=T + timedelta(hours=1)) + (post,) = _posts(fm) + assert (post["start"], post["values"], post["duration"]) == ( + T, + [20], + timedelta(hours=1), + ) + fm.get_sensor.assert_awaited_once() # asked once + + +@pytest.mark.asyncio +async def test_on_an_instantaneous_sensor_each_measurement_is_its_own_reading(): + recorder, fm = _recorder("PT0M") + await recorder.add(7, T, 10, now=T) + await recorder.add(7, T + timedelta(seconds=30), 20, now=T + timedelta(seconds=30)) + assert [(p["start"], p["values"], p["duration"]) for p in _posts(fm)] == [ + (T, [10], timedelta(0)), + (T + timedelta(seconds=30), [20], timedelta(0)), + ] + + +@pytest.mark.asyncio +async def test_a_sensor_stating_no_resolution_cannot_take_measurements(): + recorder, fm = _recorder() + fm.get_sensor = AsyncMock(return_value={}) + with pytest.raises(ValueError, match="no resolution"): + await recorder.add(7, T, 10, now=T + Q) + + +@pytest.mark.asyncio +async def test_what_was_recorded_is_forgotten_after_a_week(): + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.record(7, now=T + timedelta(days=8)) + await recorder.add(7, T, 1000, now=T + timedelta(days=8)) + assert len(_posts(fm)) == 2 + + +@pytest.mark.asyncio +async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where_it_cannot(): + from s2python.common import ( + CommodityQuantity, + PowerMeasurement, + PowerValue, + ReceptionStatusValues, + ) + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + from flexmeasures_client.s2.utils import get_unique_id + + recorder, fm = _recorder() + clock = SimulatedClock(T + timedelta(minutes=5)) + cem = CEM( + fm_client=fm, + clock=clock, + power_sensor_id={"ELECTRIC.POWER.L1": 7}, + measurement_recorder=recorder, + ) + + def measurement() -> PowerMeasurement: + return PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=T, + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=750.0 + ) + ], + ) + + await cem.handle_message(measurement()) + assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + fm.post_sensor_data.assert_not_awaited() # its step is not over by the simulated clock + clock.set(T + Q) + await cem.handle_message(measurement()) + assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + assert _posts(fm)[0]["values"] == [750.0] + + fm.get_sensor = AsyncMock(side_effect=ConnectionError("down")) + recorder._resolutions.clear() + await cem.handle_message(measurement()) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + + +def test_a_cem_has_a_recorder_of_its_own_unless_given_one(): + from flexmeasures_client.s2.cem import CEM + + assert isinstance(CEM(fm_client=None)._measurement_recorder, MeasurementRecorder) From 809b1362024858690cc7ec5ecf3ce3e98a90bbec Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:15:03 +0100 Subject: [PATCH 49/59] docs(changelog): the measurement recorder Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c8c5187d..584bb264 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -30,6 +30,7 @@ Added * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] * ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] +* ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed ------- @@ -51,11 +52,13 @@ Changed Removed ------- +* ``flexmeasures_client.s2.cem.get_commodity_unit``, which stated power in kW where S2 states it in W [`#266 `__] * ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] Fixed ----- +* A CEM records measured power by the moment it was measured, once each step of its sensor's resolution is over, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] From ab56606346da7743f7d12e160551fd29871d1eb7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 22:29:14 +0100 Subject: [PATCH 50/59] fix(S2): what the review of the measurement recorder found - A step the server kept refusing blocked its sensor for good: steps were never dropped, and consecutive ones went in one request, so every later request carried the bad step and failed with it. Each step now goes in a request of its own, is tried a few times with a wait in between, and is given up on, with an error naming its values, after that. - One request for several steps gave the earlier ones a belief time after their own end. With a request per step, each is known at its own end, as the docstring said. The batching the first version claimed only ever happened after a failure, and its test checked nothing about it; both are gone. - Two measurements for one sensor at once, from two CEMs or a sample landing while its step's post was in flight, posted the step twice and then raised a KeyError the CEM did not catch, so no reception status went out. Recording is now one post at a time per sensor. - Posting held up each measurement's reception status, which #188 had kept off the co-simulation's critical path by posting in the background. Recording happens in the background again, a defect in it is logged, and a failed post is tried again a while later without waiting for another measurement, which a co-simulation's final window never gets. - A moment in the repeated hour of an autumn DST change could not be floored to its step, so such a measurement was refused on every resend. Its step is now the later boundary not after it, as align_to_resolution finds it. - Remembering recorded steps cost a rebuild of the whole set per measurement, which on an instantaneous sensor grew large; the oldest are now dropped from the front. - A message with measurements for several sensors was taken in part where one sensor could not be read, and, asked for again, had that part taken twice. Every sensor is now made ready before any value is taken. - The server's request handler is tested for handing each CEM its recorder, and the changelog says what changes for a live resource manager: one value per step of the sensor's resolution, recorded with the first measurement after it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- src/flexmeasures_client/s2/cem.py | 60 +++--- src/flexmeasures_client/s2/measurements.py | 221 ++++++++++++++------- tests/s2/test_example_end_to_end.py | 33 +++ tests/s2/test_measurements.py | 216 ++++++++++++++------ 5 files changed, 363 insertions(+), 169 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 584bb264..d0e74592 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -58,7 +58,7 @@ Removed Fixed ----- -* A CEM records measured power by the moment it was measured, once each step of its sensor's resolution is over, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process [`#266 `__] +* A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] * A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] * A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] * Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index c079bd40..6d1f70e4 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -115,9 +115,9 @@ def __init__( called with its details, in the background (see FRBCSiteConfigurator). Until the handler is registered, its messages are answered with TEMPORARY_ERROR. Without one, the handlers registered up front serve every resource manager. - :param measurement_recorder: where the resource manager's measured power is recorded; - one of its own by default. A server that makes a CEM per connection - hands each the same one, so a resource manager that reconnects is not recorded twice. + :param measurement_recorder: where the resource manager's measured power is recorded; one of its own by default. + A server that makes a CEM per connection hands each the same one, + so a resource manager that reconnects is not recorded twice. :param clock: the time this CEM and every handler it registers go by; the wall clock by default. :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, @@ -527,44 +527,38 @@ async def handle_power_measurement(self, message: PowerMeasurement): self._ensure_configuring() return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + taken: list[tuple[int, float]] = [] for power_measurement in message.values: commodity_quantity = power_measurement.commodity_quantity.value # Without a sensor for it, a measurement is not recorded. # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. - if self.power_sensor_id: - s_id = self.power_sensor_id.get(commodity_quantity) - if s_id is None: - # TODO: create a new sensor or return ReceptionStatus - self._logger.debug( - f"No power sensor set up for {commodity_quantity}. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue - sensor_id = s_id - else: - self._logger.warning( - f"No power sensor IDs set up. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue - - try: - await self._measurement_recorder.add( - sensor_id, - message.measurement_timestamp, - power_measurement.value, - now=self.now(), - ) - except (ValueError, ConnectionError, ContentTypeError) as exc: - # Not taken at all, as when the sensor's resolution could not be read: the resource manager is asked to resend it. - self._logger.warning( - "Could not take measured power for %s at %s: %s", + sensor_id = (self.power_sensor_id or {}).get(commodity_quantity) + if sensor_id is None: + self._logger.debug( + "No power sensor set up for %s. Ignoring measurement %s at %s.", commodity_quantity, + power_measurement.value, message.measurement_timestamp, - exc, - ) - return get_reception_status( - message, ReceptionStatusValues.TEMPORARY_ERROR ) + continue + taken.append((sensor_id, power_measurement.value)) + + # Every sensor is made ready before any value is taken, so that a message asked for again is not taken in part twice. + try: + for sensor_id, _ in taken: + await self._measurement_recorder.resolution(sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not take the measured power at %s, so the resource manager is asked to send it again: %s", + message.measurement_timestamp, + exc, + ) + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + for sensor_id, watts in taken: + await self._measurement_recorder.add( + sensor_id, message.measurement_timestamp, watts, now=self.now() + ) return get_reception_status(message) diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py index 492cb0fc..5f154514 100644 --- a/src/flexmeasures_client/s2/measurements.py +++ b/src/flexmeasures_client/s2/measurements.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections import defaultdict from datetime import datetime, timedelta from logging import Logger, getLogger @@ -9,7 +10,12 @@ import pandas as pd from flexmeasures_client.client import FlexMeasuresClient -from flexmeasures_client.exceptions import ContentTypeError, JobFailedError +from flexmeasures_client.exceptions import ( + ContentTypeError, + InsufficientServerVersionError, + JobFailedError, + WrongAPIVersionError, +) _LOGGER = getLogger(__name__) @@ -19,20 +25,40 @@ #: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. REMEMBERED_FOR = timedelta(days=7) +#: How often a step is tried before it is given up on, and how long to wait between tries. +ATTEMPTS = 3 +RETRY_AFTER = timedelta(seconds=30) + +#: What a post can fail with, short of a defect in this code. +POST_FAILURES = ( + ValueError, + ConnectionError, + ContentTypeError, + JobFailedError, + InsufficientServerVersionError, + WrongAPIVersionError, +) + class MeasurementRecorder: """Records measured power on a sensor as the average of what was measured in each step of its resolution. - A step is recorded once it is over by the CEM's clock, with the moment it ended as the moment it became known, + A step is recorded once it is over by the CEM's clock, as it stands when the step's last measurement arrives, + with the moment it ended as the moment it became known, so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. Steps are taken by the time each measurement is stamped with, not by when it arrives, since a co-simulation's resource manager sends a window's measurements together once the window has passed, - each stamped at the start of its quarter hour, while a live one sends them as they are taken. - Consecutive steps that are over are recorded in one request. + each stamped at the start of its quarter hour, while a live one sends them as they are taken, + in which case a step is recorded with the first measurement after it is over. + + Recording happens in the background, so that a resource manager is not kept waiting for FlexMeasures + before its measurement is acknowledged, one post at a time per sensor. + A step that could not be recorded is tried again a while later, and given up on after a few tries. Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, which a co-simulation makes every round, and so remembers what was recorded: a step sent again is not recorded again, and the first value recorded for a step stands. + A sensor is meant to be one resource manager's: one that several share has their measurements averaged together. """ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): @@ -40,93 +66,136 @@ def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): self._logger = logger or _LOGGER self._resolutions: dict[int, timedelta] = {} self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) - self._recorded: dict[int, set[datetime]] = defaultdict(set) + self._attempts: dict[int, dict[datetime, int]] = defaultdict(dict) + #: By sensor, the steps recorded, in the order they were, which is roughly that of time. + self._recorded: dict[int, dict[datetime, None]] = defaultdict(dict) + self._locks: dict[int, asyncio.Lock] = defaultdict(asyncio.Lock) + self._tasks: set[asyncio.Task] = set() + + async def resolution(self, sensor_id: int) -> timedelta: + """The sensor's resolution, asked once, since it is a property of how the sensor was set up. + + A caller taking several measurements at once asks this for each sensor first, + so that none is taken where one of them cannot be. + """ + if sensor_id not in self._resolutions: + sensor = await self._fm_client.get_sensor(sensor_id) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + if resolution is None: + raise ValueError( + f"Sensor {sensor_id} states no resolution, so its steps are unknown." + ) + self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() + return self._resolutions[sensor_id] async def add( self, sensor_id: int, measured_at: datetime, watts: float, now: datetime ) -> None: - """Take one measurement, and record every step of this sensor that is over by now.""" - resolution = await self._resolution(sensor_id) - step = self._step_of(measured_at, resolution) + """Take one measurement, and record in the background every step of this sensor that is over by now.""" + resolution = await self.resolution(sensor_id) + step = step_of(measured_at, resolution) if step in self._recorded[sensor_id]: self._logger.debug( "Not recording %s W on sensor %s for %s again.", watts, sensor_id, step ) else: self._pending[sensor_id].setdefault(step, []).append(watts) - await self.record(sensor_id, now) + self._in_background(self.record(sensor_id, now)) async def record(self, sensor_id: int, now: datetime) -> None: - """Record every pending step of this sensor that is over by now, consecutive ones in one request each.""" - resolution = await self._resolution(sensor_id) - over = sorted( - step for step in self._pending[sensor_id] if step + resolution <= now - ) - for run in _consecutive(over, resolution): - values = [ - sum(samples) / len(samples) - for samples in (self._pending[sensor_id][step] for step in run) - ] - ended = run[-1] + resolution - try: - await self._fm_client.post_sensor_data( - sensor_id, - start=run[0], - duration=resolution * len(run) if resolution else timedelta(0), - values=values, - unit=S2_POWER_UNIT, - prior=ended, - ) - except ( - ValueError, - ConnectionError, - ContentTypeError, - JobFailedError, - ) as exc: - # Kept, and tried again with the next measurement. - self._logger.warning( - "Could not record measured power on sensor %s from %s to %s; trying again later: %s", - sensor_id, - run[0], - ended, - exc, - ) - continue - for step in run: + """Record every pending step of this sensor that is over by now, each in a request of its own. + + Each with its own end as the moment it became known, which one request for several steps cannot say. + One at a time per sensor, so a step is not posted twice, nor taken from under a post in flight. + """ + async with self._locks[sensor_id]: + resolution = await self.resolution(sensor_id) + over = sorted( + step for step in self._pending[sensor_id] if step + resolution <= now + ) + failed = False + for step in over: + samples = self._pending[sensor_id][step] + try: + await self._fm_client.post_sensor_data( + sensor_id, + start=step, + duration=resolution, + values=[sum(samples) / len(samples)], + unit=S2_POWER_UNIT, + prior=step + resolution, + ) + except POST_FAILURES as exc: + failed = True + attempts = self._attempts[sensor_id].get(step, 0) + 1 + if attempts < ATTEMPTS: + self._attempts[sensor_id][step] = attempts + self._logger.warning( + "Could not record measured power on sensor %s for %s; trying again later: %s", + sensor_id, + step, + exc, + ) + continue + self._logger.error( + "Giving up recording %s W of measured power on sensor %s for %s after %s tries: %s", + samples, + sensor_id, + step, + attempts, + exc, + ) del self._pending[sensor_id][step] - self._recorded[sensor_id].add(step) - self._recorded[sensor_id] = { - step for step in self._recorded[sensor_id] if step >= now - REMEMBERED_FOR - } + self._attempts[sensor_id].pop(step, None) + self._recorded[sensor_id][step] = None + self._forget_before(sensor_id, now - REMEMBERED_FOR) + if failed: + self._in_background(self._retry(sensor_id, now)) + + async def _retry(self, sensor_id: int, now: datetime) -> None: + await asyncio.sleep(RETRY_AFTER.total_seconds()) + await self.record(sensor_id, now) - async def _resolution(self, sensor_id: int) -> timedelta: - """The sensor's resolution, asked once, since it is a property of how the sensor was set up.""" - if sensor_id not in self._resolutions: - sensor = await self._fm_client.get_sensor(sensor_id) - resolution = ( - sensor.get("event_resolution") if isinstance(sensor, dict) else None + async def settle(self) -> None: + """Wait for every recording in the background to finish, retries included.""" + while self._tasks: + await asyncio.gather(*list(self._tasks), return_exceptions=True) + + def _in_background(self, recording) -> None: + task = asyncio.create_task(recording) + self._tasks.add(task) + task.add_done_callback(self._done) + + def _done(self, task: asyncio.Task) -> None: + self._tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + self._logger.error( + "Recording measured power failed.", exc_info=task.exception() ) - if resolution is None: - raise ValueError( - f"Sensor {sensor_id} states no resolution, so its steps are unknown." - ) - self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() - return self._resolutions[sensor_id] - @staticmethod - def _step_of(moment: datetime, resolution: timedelta) -> datetime: - """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it.""" - if not resolution: - return moment - return pd.Timestamp(moment).floor(resolution).to_pydatetime() + def _forget_before(self, sensor_id: int, cutoff: datetime) -> None: + """Forget recorded steps from before the cutoff, oldest first, so that this costs little when nothing is old.""" + recorded = self._recorded[sensor_id] + while recorded: + oldest = next(iter(recorded)) + if oldest >= cutoff: + break + del recorded[oldest] -def _consecutive(steps: list[datetime], resolution: timedelta) -> list[list[datetime]]: - """Sorted steps, in runs of steps that follow one another; with no resolution, each on its own.""" - runs: list[list[datetime]] = [] - for step in steps: - if runs and resolution and step == runs[-1][-1] + resolution: - runs[-1].append(step) - else: - runs.append([step]) - return runs +def step_of(moment: datetime, resolution: timedelta) -> datetime: + """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it. + + In the repeated hour of an autumn DST change a boundary exists twice, and the step is the later of the two + that is not after the moment; taken from the moment's pass alone, flooring cannot always be done at all. + """ + if not resolution: + return moment + at = pd.Timestamp(moment) + candidates = [ + at.floor(resolution, ambiguous=first_pass, nonexistent="shift_forward") + for first_pass in (True, False) + ] + return max(c for c in candidates if c <= at).to_pydatetime() diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py index 8a7ed3b9..12a64aa3 100644 --- a/tests/s2/test_example_end_to_end.py +++ b/tests/s2/test_example_end_to_end.py @@ -965,3 +965,36 @@ def capture(**kwargs): await cem_end._inbox.put(_CLOSED) await asyncio.wait_for(serving, timeout=5) assert made[0]["measurement_recorder"] is recorder + + +@pytest.mark.asyncio +@pytest.mark.parametrize("per_resource_manager", [False, True]) +async def test_a_connection_is_served_with_the_servers_recorder(per_resource_manager): + from unittest.mock import patch + + from aiohttp import web + from aiohttp.test_utils import TestClient, TestServer + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + app.add_routes([web.get("/ws", server.websocket_handler)]) + app[server.FM_CLIENT] = _flexmeasures() + app[server.RECORDER] = MeasurementRecorder(app[server.FM_CLIENT]) + app[server.SENSORS] = {"power": {"id": 1}} + app[server.CONFIGURATOR] = server.FRBCSiteConfigurator( + app[server.FM_CLIENT], asset_type_id=6 + ) + served = AsyncMock() + with ( + patch.object(server, "serve_resource_manager", served), + patch.object(server, "SITE_PER_RESOURCE_MANAGER", per_resource_manager), + ): + async with TestClient(TestServer(app)) as client: + async with client.ws_connect("/ws"): + await _until(lambda: served.await_count) + assert served.await_args.kwargs["recorder"] is app[server.RECORDER] + if per_resource_manager: + assert served.await_args.kwargs["configurator"] is app[server.CONFIGURATOR] + else: + assert served.await_args.args[2] is app[server.SENSORS] diff --git a/tests/s2/test_measurements.py b/tests/s2/test_measurements.py index 697bd7bc..1fb3182e 100644 --- a/tests/s2/test_measurements.py +++ b/tests/s2/test_measurements.py @@ -2,13 +2,16 @@ from __future__ import annotations +import asyncio from datetime import datetime, timedelta, timezone -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch +from zoneinfo import ZoneInfo import pytest from flexmeasures_client.client import FlexMeasuresClient -from flexmeasures_client.s2.measurements import MeasurementRecorder +from flexmeasures_client.s2 import measurements +from flexmeasures_client.s2.measurements import MeasurementRecorder, step_of T = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) Q = timedelta(minutes=15) @@ -26,8 +29,12 @@ def _posts(fm) -> list[dict]: ] +def _recorded(fm) -> dict[datetime, float]: + return {p["start"]: p["values"][0] for p in _posts(fm)} + + @pytest.mark.asyncio -async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_request(): +async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded(): """Its resource manager sends a window's measurements once the window has passed, each stamped at its quarter hour's start. Binned by the moment they arrived, none of them fell in the step the CEM's clock was in, and none was recorded. @@ -36,18 +43,16 @@ async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded_in_one_r window_end = T + 4 * Q for i, watts in enumerate([1000, 2000, 0, 500]): await recorder.add(7, T + i * Q, watts, now=window_end) - (post,) = _posts(fm)[-1:] - recorded = {} - for p in _posts(fm): - for k, v in enumerate(p["values"]): - recorded[p["start"] + k * Q] = v - assert recorded == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} - assert ( - post["unit"] == "W" - ) # as S2 states power; it used to go out labelled kW, a thousand times too large - assert ( - post["prior"] == post["start"] + len(post["values"]) * Q - ) # known at the end of what it measures + await recorder.settle() + assert _recorded(fm) == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} + for post in _posts(fm): + assert ( + post["unit"] == "W" + ) # as S2 states power; it went out labelled kW, a thousand times too large + assert post["duration"] == Q + assert ( + post["prior"] == post["start"] + Q + ) # each known at the end of what it measures @pytest.mark.asyncio @@ -55,12 +60,13 @@ async def test_a_live_resource_managers_samples_are_averaged_per_step_once_it_is recorder, fm = _recorder() await recorder.add(7, T + timedelta(minutes=1), 1000, now=T + timedelta(minutes=1)) await recorder.add(7, T + timedelta(minutes=8), 2000, now=T + timedelta(minutes=8)) + await recorder.settle() fm.post_sensor_data.assert_not_awaited() # the step is not over await recorder.add( 7, T + Q + timedelta(minutes=1), 800, now=T + Q + timedelta(minutes=1) ) - (post,) = _posts(fm) - assert (post["start"], post["values"], post["duration"]) == (T, [1500], Q) + await recorder.settle() + assert _recorded(fm) == {T: 1500} @pytest.mark.asyncio @@ -68,34 +74,67 @@ async def test_a_step_sent_again_is_not_recorded_again(): """A co-simulation's resource manager reconnects every round and sends its series again.""" recorder, fm = _recorder() await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() await recorder.add(7, T, 1000, now=T + Q) await recorder.add( 7, T, 999, now=T + Q ) # the first value recorded for a step stands + await recorder.settle() assert len(_posts(fm)) == 1 @pytest.mark.asyncio -async def test_a_step_that_could_not_be_recorded_is_tried_again(caplog): +async def test_a_step_that_could_not_be_recorded_is_tried_again_without_another_measurement( + caplog, +): + """The final window of a co-simulation has no measurement after it to try again with.""" recorder, fm = _recorder() fm.post_sensor_data.side_effect = [ConnectionError("down"), None] - await recorder.add(7, T, 1000, now=T + Q) + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() assert "trying again later" in caplog.text - await recorder.add(7, T + Q, 2000, now=T + 2 * Q) - retried = _posts(fm)[-1] - assert (retried["start"], retried["values"]) == (T, [1000, 2000]) + assert [p["start"] for p in _posts(fm)] == [T, T] @pytest.mark.asyncio -async def test_steps_that_do_not_follow_one_another_go_in_requests_of_their_own(): +async def test_a_step_that_keeps_failing_is_given_up_on_and_does_not_hold_up_the_rest( + caplog, +): + """Posted together, one step the server always refused failed every request after it, and nothing was recorded again.""" recorder, fm = _recorder() - await recorder.add(7, T, 1, now=T) - await recorder.add(7, T + 2 * Q, 3, now=T) - await recorder.record(7, now=T + 3 * Q) - assert [(p["start"], p["values"]) for p in _posts(fm)] == [ - (T, [1]), - (T + 2 * Q, [3]), - ] + + async def refuse_the_first_step(sensor_id, **kwargs): + if kwargs["start"] == T: + raise ValueError("422") + + fm.post_sensor_data.side_effect = refuse_the_first_step + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add(7, T + Q, 2000, now=T + 2 * Q) + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == measurements.ATTEMPTS + assert T + Q in [p["start"] for p in _posts(fm)] + assert "Giving up" in caplog.text + assert recorder._pending[7] == {} + + +@pytest.mark.asyncio +async def test_two_measurements_for_one_sensor_at_once_post_each_step_once(): + """Two CEMs on one sensor, or a sample arriving while its step's post is in flight, posted it twice and then raised.""" + recorder, fm = _recorder() + release = asyncio.Event() + + async def slow(sensor_id, **kwargs): + await release.wait() + + fm.post_sensor_data.side_effect = slow + await recorder.add(7, T, 100, now=T + Q) + await asyncio.sleep(0) + await recorder.add(7, T + timedelta(minutes=5), 200, now=T + Q) + release.set() + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 1 @pytest.mark.asyncio @@ -103,6 +142,7 @@ async def test_each_sensor_is_recorded_at_its_own_resolution(): recorder, fm = _recorder("PT1H") await recorder.add(7, T + timedelta(minutes=20), 10, now=T + timedelta(minutes=20)) await recorder.add(7, T + timedelta(minutes=40), 30, now=T + timedelta(hours=1)) + await recorder.settle() (post,) = _posts(fm) assert (post["start"], post["values"], post["duration"]) == ( T, @@ -117,6 +157,7 @@ async def test_on_an_instantaneous_sensor_each_measurement_is_its_own_reading(): recorder, fm = _recorder("PT0M") await recorder.add(7, T, 10, now=T) await recorder.add(7, T + timedelta(seconds=30), 20, now=T + timedelta(seconds=30)) + await recorder.settle() assert [(p["start"], p["values"], p["duration"]) for p in _posts(fm)] == [ (T, [10], timedelta(0)), (T + timedelta(seconds=30), [20], timedelta(0)), @@ -135,23 +176,56 @@ async def test_a_sensor_stating_no_resolution_cannot_take_measurements(): async def test_what_was_recorded_is_forgotten_after_a_week(): recorder, fm = _recorder() await recorder.add(7, T, 1000, now=T + Q) - await recorder.record(7, now=T + timedelta(days=8)) + await recorder.settle() + await recorder.add(7, T + timedelta(days=8), 1, now=T + timedelta(days=8)) + await recorder.settle() + assert T not in recorder._recorded[7] await recorder.add(7, T, 1000, now=T + timedelta(days=8)) - assert len(_posts(fm)) == 2 + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 2 -@pytest.mark.asyncio -async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where_it_cannot(): - from s2python.common import ( - CommodityQuantity, - PowerMeasurement, - PowerValue, - ReceptionStatusValues, +def test_a_step_in_the_repeated_autumn_hour_is_found_in_the_pass_the_moment_is_in(): + """pandas cannot floor such a moment by its pass alone, and the CEM answered every resend with a temporary error.""" + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 37, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + step = step_of(second_pass, Q) + assert step.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 30, tzinfo=timezone.utc + ) + + +def test_steps_are_on_the_moments_own_clock(): + kolkata = ZoneInfo("Asia/Kolkata") + assert step_of( + datetime(2026, 11, 5, 14, 40, tzinfo=kolkata), timedelta(hours=1) + ) == datetime(2026, 11, 5, 14, 0, tzinfo=kolkata) + + +def _measurement(*values: tuple[str, float]): + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + return PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=T, + values=[ + PowerValue(commodity_quantity=CommodityQuantity(q), value=v) + for q, v in values + ], ) + +@pytest.mark.asyncio +async def test_the_cem_hands_measured_power_to_its_recorder_on_its_own_clock(): + from s2python.common import ReceptionStatusValues + from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.clock import SimulatedClock - from flexmeasures_client.s2.utils import get_unique_id recorder, fm = _recorder() clock = SimulatedClock(T + timedelta(minutes=5)) @@ -161,35 +235,59 @@ async def test_the_cem_hands_measured_power_to_its_recorder_and_asks_again_where power_sensor_id={"ELECTRIC.POWER.L1": 7}, measurement_recorder=recorder, ) - - def measurement() -> PowerMeasurement: - return PowerMeasurement( - message_id=get_unique_id(), - measurement_timestamp=T, - values=[ - PowerValue( - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=750.0 - ) - ], - ) - - await cem.handle_message(measurement()) + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + await recorder.settle() fm.post_sensor_data.assert_not_awaited() # its step is not over by the simulated clock clock.set(T + Q) - await cem.handle_message(measurement()) - assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value - assert _posts(fm)[0]["values"] == [750.0] + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) + await cem.get_message() + await recorder.settle() + assert _recorded(fm) == {T: 750.0} + + +@pytest.mark.asyncio +async def test_a_message_the_cem_cannot_take_in_full_is_taken_not_at_all(): + """Asked for again, a message taken in part had the part taken twice, which skewed its average.""" + from s2python.common import ReceptionStatusValues + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + + recorder, fm = _recorder() + + async def readable(sensor_id, **kwargs): + if sensor_id == 8: + raise ConnectionError("down") + return {"event_resolution": "PT15M"} - fm.get_sensor = AsyncMock(side_effect=ConnectionError("down")) - recorder._resolutions.clear() - await cem.handle_message(measurement()) + fm.get_sensor = AsyncMock(side_effect=readable) + cem = CEM( + fm_client=fm, + clock=SimulatedClock(T + timedelta(minutes=5)), + power_sensor_id={"ELECTRIC.POWER.L1": 7, "ELECTRIC.POWER.L2": 8}, + measurement_recorder=recorder, + ) + await cem.handle_message( + _measurement(("ELECTRIC.POWER.L1", 750.0), ("ELECTRIC.POWER.L2", 10.0)) + ) assert (await cem.get_message())[ "status" ] == ReceptionStatusValues.TEMPORARY_ERROR.value + assert recorder._pending[7] == {} def test_a_cem_has_a_recorder_of_its_own_unless_given_one(): from flexmeasures_client.s2.cem import CEM assert isinstance(CEM(fm_client=None)._measurement_recorder, MeasurementRecorder) + + +@pytest.mark.asyncio +async def test_a_defect_while_recording_is_logged_rather_than_lost(caplog): + """An exception no post is expected to raise surfaces in the log, not as a task exception never retrieved.""" + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = TypeError("a malformed error payload") + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + assert "Recording measured power failed" in caplog.text From b8aff1dc53f37141490e04232d330ace4d9f73ea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:11:19 +0100 Subject: [PATCH 51/59] fix(examples): the co-simulation stack runs on a current FlexMeasures image Found by running the stack, from checkouts of its own, against FlexMeasures main (1.1.0) with the community scheduler plugin, and a two-home, one-day co-simulation through it (no community steering): every step ran, every home's measured power was recorded for every quarter hour, and no CEM error remained. It took four fixes to the overlay first. - The server and the workers installed this client with the system pip, while FlexMeasures' image runs FlexMeasures and its plugins from its own virtualenv, so the plugin could not import flexmeasures_client and the server never booted. They now install with that interpreter's pip, as the CEMs already did, and no longer try to install /app itself, which the current image has no project file in. - A checkout mounted without git history it can read, as a git worktree is, gives hatch-vcs no version, so the editable install failed. The install is given one, in the plain SETUPTOOLS_SCM_PRETEND_VERSION, since hatch-vcs does not match the per-distribution variable the CEMs had. - A development server trusts only loopback names, and the CEMs reach it as "server", so every request of theirs was refused as from an untrusted host. The server now trusts that name too. - The README says how to build FlexMeasures' image from a worktree (its version as a build argument) and how to move the ports where they are taken. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 12 ++++++--- .../S2-co-simulation/docker-compose.cem.yml | 27 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 9e1b8e0f..02d76fef 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -27,7 +27,13 @@ whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` inside the flexmeasures checkout it runs from. A compose project name of its own does not separate that. To keep a development database you care about out of it, -run it from a folder of its own holding checkouts (git worktrees will do) -named `flexmeasures` and `flexmeasures-client`: +run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`: the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. -Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too. +Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, +which it cannot through a worktree, so give it as a build argument, +`FLEXMEASURES_VERSION`, for each service the overlay builds. +Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too; +where they are not, an extra overlay can move them with compose's `!override`. + +It runs FlexMeasures as its image installs it, with the plugins and this client installed into the image's virtualenv, +and expects a development server, so the server trusts the name the CEMs reach it by, `server`, alongside the loopback names. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index 79a5b364..07a48049 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -37,8 +37,6 @@ x-cem: &cem S2_ACCEPT_SIMULATION_METADATA: "1" # And each home is a site of its own, rather than all of them sharing one. S2_SITE_PER_RESOURCE_MANAGER: "1" - # The checkout is mounted without its git history, which is where its version would come from. - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLEXMEASURES_CLIENT: "0.0.0" volumes: - ../flexmeasures-client:/app/flexmeasures-client:rw entrypoint: ["/bin/sh", "-c"] @@ -49,7 +47,9 @@ x-cem: &cem # which may come without pip, and cannot see what the system pip installs. python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade # s2-python as this repo locks it, and as the co-simulation's resource managers use it. - python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py # Shared definition for every RQ worker: one per home simulated at once, @@ -83,9 +83,12 @@ x-worker: &worker LOGGING_LEVEL: INFO command: - | - pip install --break-system-packages -e /app - pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt - pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" # Drop any lingering registration for the ingestion queue before this worker starts. # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. @@ -117,6 +120,9 @@ services: ports: - "6380:6379" server: + environment: + # The CEMs reach the server by its service name, which a development server does not trust by default. + TRUSTED_HOSTS: "localhost,.localhost,127.0.0.1,[::1],server" volumes: - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw - ./flexmeasures-instance/:/app/instance/:rw @@ -124,9 +130,12 @@ services: - ../flexmeasures-client:/app/flexmeasures-client:rw command: - | - pip install --break-system-packages -e /app - pip install --break-system-packages -r /usr/var/flexmeasures-instance/requirements.txt - pip install --break-system-packages -e /app/flexmeasures-client[s2] + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" flexmeasures db upgrade if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then flexmeasures add toy-account --name 'Docker Toy Account' From 40288e8b442ad87768c8e0ced865d634bcaf0332 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:11:48 +0100 Subject: [PATCH 52/59] docs(changelog): the co-simulation stack on FlexMeasures 1.1 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d0e74592..c2bf6885 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -29,7 +29,7 @@ Added * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] -* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets, run on FlexMeasures 1.1 [`#265 `__, `#267 `__] * ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed From 48073881ce143519af677e7e7aa1dc1571cb9e74 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:05 +0100 Subject: [PATCH 53/59] docs(examples): what the co-simulation stack runs, and how to build it from a worktree Review found the README saying the stack runs FlexMeasures as its image installs it, while the mounted checkout is what runs, and an s2-python pin on the server and workers, which speak no S2 and whose locked environment it should not shift. The README now shows the build-argument overlay a worktree needs, says the UI may lack its fetched libraries, and the changelog no longer claims a FlexMeasures version the overlay does not pin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- examples/S2-co-simulation/README.md | 28 +++++++++++++++---- .../S2-co-simulation/docker-compose.cem.yml | 6 ++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index c2bf6885..06623ba4 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -29,7 +29,7 @@ Added * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] -* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets, run on FlexMeasures 1.1 [`#265 `__, `#267 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__, `#267 `__] * ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 02d76fef..13b364dd 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -29,11 +29,29 @@ A compose project name of its own does not separate that. To keep a development database you care about out of it, run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`: the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. -Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, -which it cannot through a worktree, so give it as a build argument, -`FLEXMEASURES_VERSION`, for each service the overlay builds. +Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, which it cannot through a worktree. +Give it as the build argument `FLEXMEASURES_VERSION` to each service built from FlexMeasures' Dockerfile, in one more overlay: + +```yaml +x-version: &version + build: + context: . + dockerfile: Dockerfile + args: + FLEXMEASURES_VERSION: "1.1.0" + +services: + server: + <<: *version + worker: + <<: *version + # and likewise worker-2, worker-3, automation-runner and each cem +``` + Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too; where they are not, an extra overlay can move them with compose's `!override`. -It runs FlexMeasures as its image installs it, with the plugins and this client installed into the image's virtualenv, -and expects a development server, so the server trusts the name the CEMs reach it by, `server`, alongside the loopback names. +It runs FlexMeasures from the flexmeasures checkout, which is mounted over the code the image installed, +with this client installed into the image's virtualenv, which is what FlexMeasures and its plugins run from. +The UI's third-party libraries are fetched into the installed package when the image is built, so a UI served from the mounted checkout may lack them; the API does not need them. +The server trusts the name the CEMs reach it by, `server`, alongside the loopback names a development server trusts by default. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml index 07a48049..9304b049 100644 --- a/examples/S2-co-simulation/docker-compose.cem.yml +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -4,7 +4,7 @@ # such as the community simulations that steer several homes at once. # # It is an overlay on FlexMeasures' own docker-compose.yml, -# and runs the CEMs from this checkout. +# and runs FlexMeasures from the flexmeasures checkout and the CEMs from this one, both mounted. # With checkouts named flexmeasures and flexmeasures-client side by side, # run it from the flexmeasures folder, which holds the Dockerfile: # @@ -88,7 +88,7 @@ x-worker: &worker python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt # The checkout may be mounted without git history to read its version from, as a git worktree is, # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. - SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" # Drop any lingering registration for the ingestion queue before this worker starts. # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. @@ -135,7 +135,7 @@ services: python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt # The checkout may be mounted without git history to read its version from, as a git worktree is, # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. - SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" flexmeasures db upgrade if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then flexmeasures add toy-account --name 'Docker Toy Account' From f0c4d4e67a60d4a5ecfe8ad9ea51dcc7c05c11d7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:21 +0100 Subject: [PATCH 54/59] fix(FRBC): a state of charge is known as of the handler's clock Every other value FRBCSimple records says when it became known; the state of charge did not, so FlexMeasures took the moment the request reached it. In the co-simulation gate that recorded readings of 2022 as known in 2026, 1436 days after they were taken. It is now posted with the handler's time as its prior, which in a co-simulation is the simulated time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 7 ++++- tests/s2/test_clock.py | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ccd43d1f..908a314b 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -387,14 +387,19 @@ async def _soc_duration(self) -> timedelta: async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): + # Read by this handler's clock, and known as of then, which in a co-simulation is the simulated time: + # without a prior, FlexMeasures took the moment the request reached it, + # and a simulated reading was recorded as known years after it was taken. + now = self.now() await self._fm_client.post_measurements( self._soc_sensor_id, - start=self.now(), + start=now, # Scaled like every other fill level this handler records, or the level and the # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, duration=await self._soc_duration(), + prior=now, ) # The state of charge this status reports is what a schedule starts from, so this is diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py index 946bae72..dd3a4361 100644 --- a/tests/s2/test_clock.py +++ b/tests/s2/test_clock.py @@ -218,3 +218,32 @@ def test_a_tunes_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time( (start + timedelta(minutes=30)).astimezone().replace(tzinfo=None) ) assert not tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_known_as_of_the_simulated_time(): + """Without a prior, FlexMeasures stamped a simulated reading with the moment the request arrived, years later.""" + from unittest.mock import AsyncMock + + from s2python.frbc import FRBCStorageStatus + + from flexmeasures_client.client import FlexMeasuresClient + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + from flexmeasures_client.s2.utils import get_unique_id + + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": "PT0M"}) + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + ) + cem = CEM(fm_client=fm, clock=SimulatedClock(SIMULATED)) + cem.register_control_type(frbc) + await frbc.send_storage_status( + FRBCStorageStatus(message_id=get_unique_id(), present_fill_level=5.0) + ) + posted = fm.post_measurements.await_args.kwargs + assert posted["start"] == SIMULATED + assert posted["prior"] == SIMULATED From 2b5d48159d8bf71f73c6712399fdc598c58dd85b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:14:40 +0100 Subject: [PATCH 55/59] docs(changelog): the state of charge on the clock Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index d0e74592..057fa2ae 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -25,7 +25,7 @@ Added * ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] * ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] * An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] -* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers included [`#264 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers and the moment a state of charge is known as of included [`#264 `__, `#268 `__] * ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] * ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] * ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] From 00d3676be5e40205942bcb6a7462c019eeca0270 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:21:33 +0100 Subject: [PATCH 56/59] feat(FRBC): plan one schedule at a time, for the newest request Package D4 of the plan to finish #188. Each storage status is a planning request, and one arriving while a schedule was being planned started a second solve beside it, whose answer replaced the first's. A resource manager reporting faster than FlexMeasures plans kept several solves running on the server for nothing. FRBCSimple now plans one schedule at a time: a request waits for the one in flight and, when its turn comes, is dropped where a newer one arrived meanwhile, as a system description's plan already was where a status overtook it. A burst of statuses costs at most the solve in flight and one more, for the newest. No clock is involved, so a co-simulation's retrigger round, which shares a simulated time with the round before it, is planned as before, and the newest request is never left without a plan. The plan's revision 1 proposed a minimum replanning interval instead; its review showed that a deferred plan could strand the last status of a session, and that on a simulated clock a retrigger would always fall inside the interval. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 24 +++++++- tests/s2/test_frbc_simple_planning.py | 56 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ccd43d1f..7e359c74 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -263,6 +263,7 @@ def __init__( self._power_sensor_takes_consumption_as_positive = None self._power_recorded_at = {} self._soc_sensor_records_instants: bool | None = None + self._planning = asyncio.Lock() if hard_bound_factor < 1: # Below one, the hard upper bound sits inside the declared range, # and FlexMeasures clips the band into the hard bounds, @@ -849,12 +850,33 @@ def _warn_where_a_fallback_scheduler_answered( async def trigger_schedule( self, system_description_id: str, request_number: int | None = None ): - """Translates S2 System Description into FM API calls. + """Plan, one schedule at a time, and only for the newest request once it is this one's turn. + + Each storage status is a planning request, and one arriving while a schedule is being planned + used to start a second solve beside it, whose answer replaced the first's. + A resource manager reporting faster than FlexMeasures plans then kept several solves running for nothing. + Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile, + so a burst of statuses costs at most the solve in flight and one more, for the newest. :param system_description_id: system description to base the schedule on :param request_number: which planning request this schedule answers, which is checked once more before the instructions go out """ + async with self._planning: + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not planning for request %s: a newer one arrived while the schedule before it was being planned.", + request_number, + ) + return + await self._plan(system_description_id, request_number) + + async def _plan( + self, system_description_id: str, request_number: int | None = None + ): + """Translates S2 System Description into FM API calls.""" system_description: FRBCSystemDescription = self._system_description_history[ system_description_id diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 046c4fb2..9628410f 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -578,3 +578,59 @@ async def recording(*args, **kwargs): newest = max(n for n, _ in batches) assert sent[newest] > 0 assert all(count == 0 for n, count in batches if n != newest) + + +@pytest.mark.asyncio +async def test_a_burst_of_statuses_costs_the_solve_in_flight_and_one_more(): + """Each status used to start a solve beside the one in flight, all but the last for nothing.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + running = 0 + most = 0 + + async def solve(**kwargs): + nonlocal running, most + running += 1 + most = max(most, running) + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + running -= 1 + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + for _ in range(4): + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert most == 1 + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_overtaken_while_it_waits_for_the_solve_in_flight_is_dropped(): + """It was the newest when it began to wait, and is not when its turn comes.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # waits for the solve in flight + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # overtakes the one waiting + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 From 401599bd160049693c1505a9f66cda911ef18879 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:22:21 +0100 Subject: [PATCH 57/59] docs(changelog): one schedule at a time Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index b75342d6..6247fa97 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -35,6 +35,7 @@ Added Changed ------- +* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile. Each status used to start a solve of its own beside the one in flight [`#269 `__] * An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] From 29845ea71969ba0300b4a136960c6f24ea61773d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:37:19 +0100 Subject: [PATCH 58/59] fix(FRBC): a request out of date makes no further attempt, so the newest waits less Review found that a request overtaken while its schedule was being asked for kept the planning lock through every retry of a failed attempt, up to five attempts with a wait between each, and that the schedule they finally got was thrown away; only then was the newest request planned, where before this PR it would have started at once. Between attempts the request now checks whether a newer one has arrived, and stands down if so. The docstring, the PR's cost bound and the changelog line now say what holds: a burst costs at most the attempt in flight and one plan more, for the newest, which starts after that attempt rather than beside it. The docstring breaks after punctuation, and _plan documents its parameters again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- CHANGELOG-S2.rst | 2 +- .../s2/control_types/FRBC/frbc_simple.py | 48 +++++++++++++----- tests/s2/test_frbc_simple_planning.py | 50 +++++++++++++++++++ 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 6247fa97..cce3da96 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -35,7 +35,7 @@ Added Changed ------- -* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile. Each status used to start a solve of its own beside the one in flight [`#269 `__] +* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile, and a request out of date makes no further attempt after a failed one. Each status used to start a solve of its own beside the one in flight, so the newest is now planned after the attempt in flight rather than beside it [`#269 `__] * An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index dde08a0c..ffa491a6 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -770,8 +770,12 @@ def _is_worth_asking_again(self, failure: Exception) -> bool: return any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS) async def _ask_for_a_schedule( - self, start: datetime, flex_context: dict, flex_model: dict - ) -> dict | list[dict]: + self, + start: datetime, + flex_context: dict, + flex_model: dict, + request_number: int | None = None, + ) -> dict | list[dict] | None: """Ask FlexMeasures for a schedule, and ask again where the attempt is worth making again. One failed scheduling job otherwise leaves the storage without an instruction for its window. @@ -781,7 +785,9 @@ async def _ask_for_a_schedule( a worker crashed inside its solver, and the resource manager it belonged to polled until its round timed out, every round, for the rest of the run. - :param start: the window the schedule is asked for, which every attempt asks for again. + :param start: the window the schedule is asked for, which every attempt asks for again. + :param request_number: the planning request this answers; where a newer one arrives between attempts, + no further attempt is made and None is returned, so that the newer one is planned sooner. :raises JobFailedError: where the last attempt worth making failed inside the server. :raises ValueError: where the trigger itself was refused. """ @@ -823,6 +829,14 @@ async def _ask_for_a_schedule( failure, ) await asyncio.sleep(self._schedule_retry_interval.total_seconds()) + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not asking again for planning request %s: a newer one arrived, and is planned instead.", + request_number, + ) + return None def _warn_where_a_fallback_scheduler_answered( self, schedule: dict | list[dict], start: datetime @@ -857,15 +871,17 @@ async def trigger_schedule( ): """Plan, one schedule at a time, and only for the newest request once it is this one's turn. - Each storage status is a planning request, and one arriving while a schedule is being planned - used to start a second solve beside it, whose answer replaced the first's. - A resource manager reporting faster than FlexMeasures plans then kept several solves running for nothing. - Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile, - so a burst of statuses costs at most the solve in flight and one more, for the newest. + Each storage status is a planning request. + One arriving while a schedule was being planned used to start a second solve beside it, whose answer replaced the first's, + so a resource manager reporting faster than FlexMeasures plans kept several solves running for nothing. + Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile; + the one in flight, once out of date, makes no further attempt after a failed one. + So a burst of statuses costs at most the attempt in flight and one plan more, for the newest, + which starts once that attempt is over rather than beside it. :param system_description_id: system description to base the schedule on - :param request_number: which planning request this schedule answers, which is - checked once more before the instructions go out + :param request_number: which planning request this schedule answers, which is checked when its turn comes, + between attempts, and before the schedule is translated and sent """ async with self._planning: if request_number is not None and not self._is_latest_request( @@ -881,7 +897,11 @@ async def trigger_schedule( async def _plan( self, system_description_id: str, request_number: int | None = None ): - """Translates S2 System Description into FM API calls.""" + """Translates S2 System Description into FM API calls. + + :param system_description_id: system description to base the schedule on + :param request_number: which planning request this schedule answers + """ system_description: FRBCSystemDescription = self._system_description_history[ system_description_id @@ -1045,7 +1065,11 @@ async def _plan( if self._relax_site_capacity is not None: flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity - schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) + schedule = await self._ask_for_a_schedule( + start, flex_context, flex_model, request_number=request_number + ) + if schedule is None: + return self._warn_where_a_fallback_scheduler_answered(schedule, start) # send_instruction_batch checks this once more, but translating a schedule follows every step of it, diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index 9628410f..428c76ee 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -634,3 +634,53 @@ async def solve(**kwargs): release.set() await settle(frbc) assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_out_of_date_makes_no_further_attempt_while_the_newest_waits( + caplog, +): + """It kept the newest waiting through every one of its retries, and the schedule they got was thrown away.""" + import logging + from datetime import timedelta as td + + caplog.set_level(logging.INFO) + + from flexmeasures_client.exceptions import JobFailedError + + frbc = frbc_simple() + frbc._schedule_attempts = 5 + frbc._schedule_retry_interval = td(milliseconds=10) + frbc._is_worth_asking_again = lambda failure: True + first_attempt = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await first_attempt.wait() + raise JobFailedError("the worker crashed") + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + batches: list[int] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + batches.append(kwargs["request_number"]) + return await send(*args, **kwargs) + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # newer, while the first attempt is out + await asyncio.sleep(0.05) + first_attempt.set() + await settle(frbc) + # The failed attempt, then the newest. + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + assert "Not asking again for planning request" in caplog.text + assert not [ + r for r in caplog.records if r.levelname == "ERROR" + ] # nothing translated for it + assert batches == [2] # nothing sent for the request that stood down From 5564983963fa2644905ec5806f6c1d3a1636677a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 5 Nov 2026 23:37:50 +0100 Subject: [PATCH 59/59] docs(examples): the instance settings a co-simulation needs Found by running a steered two-home co-simulation through the stack: a steering round replans a window at the same simulated moment, so without FLEXMEASURES_ALLOW_DATA_OVERWRITE the scheduler could not save the second plan and the homes got no instructions for their window; and every home triggers a schedule each round from one account, far above the default trigger limit, so every trigger past it waited minutes. Neither is read from the environment, so the README says what to put in the instance config, with the solver time limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CcpawsJs74cSwFLXkz7ZwW Signed-off-by: F.N. Claessen --- examples/S2-co-simulation/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md index 13b364dd..6e95fa06 100644 --- a/examples/S2-co-simulation/README.md +++ b/examples/S2-co-simulation/README.md @@ -18,6 +18,20 @@ with two switches on: resource manager retries. Its schedules start from the simulated now and cover 24 hours. The site's prices are for whoever runs the simulation to post. +FlexMeasures reads three settings a co-simulation needs from its instance config +(`flexmeasures-instance/flexmeasures.cfg` in the flexmeasures checkout), not from the environment: + +```python +# A steering round replans a window at the same simulated moment, and that plan replaces the one before. +# Without this, the scheduler cannot save the second plan, and the home gets no instructions for its window. +FLEXMEASURES_ALLOW_DATA_OVERWRITE = True +# Every home triggers a schedule each round, all from one account, far above the default of 10 per 5 minutes, +# and every trigger past the limit waits minutes before it is retried. +FLEXMEASURES_API_TRIGGER_RATE_LIMIT = "1000 per minute" +# A bound on each solve, since a tail of degenerate problems otherwise proves optimality past the job timeout. +FLEXMEASURES_LP_SOLVER_OPTIONS = {"time_limit": "15"} +``` + A co-simulation reads the sites by sensor name: `price`, `production price`, `power` (the schedule), `measured-power` (what the home measured) and `state of charge`.